o grub 0.93 # grub --version || grub-install --version
o mcelog 0.6 # mcelog --version
o iptables 1.4.2 # iptables -V
-o openssl & libcrypto 1.0.1k # openssl version
+o openssl & libcrypto 1.0.0 # openssl version
Kernel compilation
genericirq.xml s390-drivers.xml uio-howto.xml scsi.xml \
80211.xml debugobjects.xml sh.xml regulator.xml \
alsa-driver-api.xml writing-an-alsa-driver.xml \
- tracepoint.xml drm.xml media_api.xml w1.xml \
+ tracepoint.xml gpu.xml media_api.xml w1.xml \
writing_musb_glue_layer.xml crypto-API.xml iio.xml
include Documentation/DocBook/media/Makefile
+++ /dev/null
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
- "http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd" []>
-
-<book id="drmDevelopersGuide">
- <bookinfo>
- <title>Linux DRM Developer's Guide</title>
-
- <authorgroup>
- <author>
- <firstname>Jesse</firstname>
- <surname>Barnes</surname>
- <contrib>Initial version</contrib>
- <affiliation>
- <orgname>Intel Corporation</orgname>
- <address>
- <email>jesse.barnes@intel.com</email>
- </address>
- </affiliation>
- </author>
- <author>
- <firstname>Laurent</firstname>
- <surname>Pinchart</surname>
- <contrib>Driver internals</contrib>
- <affiliation>
- <orgname>Ideas on board SPRL</orgname>
- <address>
- <email>laurent.pinchart@ideasonboard.com</email>
- </address>
- </affiliation>
- </author>
- <author>
- <firstname>Daniel</firstname>
- <surname>Vetter</surname>
- <contrib>Contributions all over the place</contrib>
- <affiliation>
- <orgname>Intel Corporation</orgname>
- <address>
- <email>daniel.vetter@ffwll.ch</email>
- </address>
- </affiliation>
- </author>
- </authorgroup>
-
- <copyright>
- <year>2008-2009</year>
- <year>2013-2014</year>
- <holder>Intel Corporation</holder>
- </copyright>
- <copyright>
- <year>2012</year>
- <holder>Laurent Pinchart</holder>
- </copyright>
-
- <legalnotice>
- <para>
- The contents of this file may be used under the terms of the GNU
- General Public License version 2 (the "GPL") as distributed in
- the kernel source COPYING file.
- </para>
- </legalnotice>
-
- <revhistory>
- <!-- Put document revisions here, newest first. -->
- <revision>
- <revnumber>1.0</revnumber>
- <date>2012-07-13</date>
- <authorinitials>LP</authorinitials>
- <revremark>Added extensive documentation about driver internals.
- </revremark>
- </revision>
- </revhistory>
- </bookinfo>
-
-<toc></toc>
-
-<part id="drmCore">
- <title>DRM Core</title>
- <partintro>
- <para>
- This first part of the DRM Developer's Guide documents core DRM code,
- helper libraries for writing drivers and generic userspace interfaces
- exposed by DRM drivers.
- </para>
- </partintro>
-
- <chapter id="drmIntroduction">
- <title>Introduction</title>
- <para>
- The Linux DRM layer contains code intended to support the needs
- of complex graphics devices, usually containing programmable
- pipelines well suited to 3D graphics acceleration. Graphics
- drivers in the kernel may make use of DRM functions to make
- tasks like memory management, interrupt handling and DMA easier,
- and provide a uniform interface to applications.
- </para>
- <para>
- A note on versions: this guide covers features found in the DRM
- tree, including the TTM memory manager, output configuration and
- mode setting, and the new vblank internals, in addition to all
- the regular features found in current kernels.
- </para>
- <para>
- [Insert diagram of typical DRM stack here]
- </para>
- </chapter>
-
- <!-- Internals -->
-
- <chapter id="drmInternals">
- <title>DRM Internals</title>
- <para>
- This chapter documents DRM internals relevant to driver authors
- and developers working to add support for the latest features to
- existing drivers.
- </para>
- <para>
- First, we go over some typical driver initialization
- requirements, like setting up command buffers, creating an
- initial output configuration, and initializing core services.
- Subsequent sections cover core internals in more detail,
- providing implementation notes and examples.
- </para>
- <para>
- The DRM layer provides several services to graphics drivers,
- many of them driven by the application interfaces it provides
- through libdrm, the library that wraps most of the DRM ioctls.
- These include vblank event handling, memory
- management, output management, framebuffer management, command
- submission & fencing, suspend/resume support, and DMA
- services.
- </para>
-
- <!-- Internals: driver init -->
-
- <sect1>
- <title>Driver Initialization</title>
- <para>
- At the core of every DRM driver is a <structname>drm_driver</structname>
- structure. Drivers typically statically initialize a drm_driver structure,
- and then pass it to one of the <function>drm_*_init()</function> functions
- to register it with the DRM subsystem.
- </para>
- <para>
- Newer drivers that no longer require a <structname>drm_bus</structname>
- structure can alternatively use the low-level device initialization and
- registration functions such as <function>drm_dev_alloc()</function> and
- <function>drm_dev_register()</function> directly.
- </para>
- <para>
- The <structname>drm_driver</structname> structure contains static
- information that describes the driver and features it supports, and
- pointers to methods that the DRM core will call to implement the DRM API.
- We will first go through the <structname>drm_driver</structname> static
- information fields, and will then describe individual operations in
- details as they get used in later sections.
- </para>
- <sect2>
- <title>Driver Information</title>
- <sect3>
- <title>Driver Features</title>
- <para>
- Drivers inform the DRM core about their requirements and supported
- features by setting appropriate flags in the
- <structfield>driver_features</structfield> field. Since those flags
- influence the DRM core behaviour since registration time, most of them
- must be set to registering the <structname>drm_driver</structname>
- instance.
- </para>
- <synopsis>u32 driver_features;</synopsis>
- <variablelist>
- <title>Driver Feature Flags</title>
- <varlistentry>
- <term>DRIVER_USE_AGP</term>
- <listitem><para>
- Driver uses AGP interface, the DRM core will manage AGP resources.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_REQUIRE_AGP</term>
- <listitem><para>
- Driver needs AGP interface to function. AGP initialization failure
- will become a fatal error.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_PCI_DMA</term>
- <listitem><para>
- Driver is capable of PCI DMA, mapping of PCI DMA buffers to
- userspace will be enabled. Deprecated.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_SG</term>
- <listitem><para>
- Driver can perform scatter/gather DMA, allocation and mapping of
- scatter/gather buffers will be enabled. Deprecated.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_HAVE_DMA</term>
- <listitem><para>
- Driver supports DMA, the userspace DMA API will be supported.
- Deprecated.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_HAVE_IRQ</term><term>DRIVER_IRQ_SHARED</term>
- <listitem><para>
- DRIVER_HAVE_IRQ indicates whether the driver has an IRQ handler
- managed by the DRM Core. The core will support simple IRQ handler
- installation when the flag is set. The installation process is
- described in <xref linkend="drm-irq-registration"/>.</para>
- <para>DRIVER_IRQ_SHARED indicates whether the device & handler
- support shared IRQs (note that this is required of PCI drivers).
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_GEM</term>
- <listitem><para>
- Driver use the GEM memory manager.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_MODESET</term>
- <listitem><para>
- Driver supports mode setting interfaces (KMS).
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_PRIME</term>
- <listitem><para>
- Driver implements DRM PRIME buffer sharing.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_RENDER</term>
- <listitem><para>
- Driver supports dedicated render nodes.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRIVER_ATOMIC</term>
- <listitem><para>
- Driver supports atomic properties. In this case the driver
- must implement appropriate obj->atomic_get_property() vfuncs
- for any modeset objects with driver specific properties.
- </para></listitem>
- </varlistentry>
- </variablelist>
- </sect3>
- <sect3>
- <title>Major, Minor and Patchlevel</title>
- <synopsis>int major;
-int minor;
-int patchlevel;</synopsis>
- <para>
- The DRM core identifies driver versions by a major, minor and patch
- level triplet. The information is printed to the kernel log at
- initialization time and passed to userspace through the
- DRM_IOCTL_VERSION ioctl.
- </para>
- <para>
- The major and minor numbers are also used to verify the requested driver
- API version passed to DRM_IOCTL_SET_VERSION. When the driver API changes
- between minor versions, applications can call DRM_IOCTL_SET_VERSION to
- select a specific version of the API. If the requested major isn't equal
- to the driver major, or the requested minor is larger than the driver
- minor, the DRM_IOCTL_SET_VERSION call will return an error. Otherwise
- the driver's set_version() method will be called with the requested
- version.
- </para>
- </sect3>
- <sect3>
- <title>Name, Description and Date</title>
- <synopsis>char *name;
-char *desc;
-char *date;</synopsis>
- <para>
- The driver name is printed to the kernel log at initialization time,
- used for IRQ registration and passed to userspace through
- DRM_IOCTL_VERSION.
- </para>
- <para>
- The driver description is a purely informative string passed to
- userspace through the DRM_IOCTL_VERSION ioctl and otherwise unused by
- the kernel.
- </para>
- <para>
- The driver date, formatted as YYYYMMDD, is meant to identify the date of
- the latest modification to the driver. However, as most drivers fail to
- update it, its value is mostly useless. The DRM core prints it to the
- kernel log at initialization time and passes it to userspace through the
- DRM_IOCTL_VERSION ioctl.
- </para>
- </sect3>
- </sect2>
- <sect2>
- <title>Device Registration</title>
- <para>
- A number of functions are provided to help with device registration.
- The functions deal with PCI and platform devices, respectively.
- </para>
-!Edrivers/gpu/drm/drm_pci.c
-!Edrivers/gpu/drm/drm_platform.c
- <para>
- New drivers that no longer rely on the services provided by the
- <structname>drm_bus</structname> structure can call the low-level
- device registration functions directly. The
- <function>drm_dev_alloc()</function> function can be used to allocate
- and initialize a new <structname>drm_device</structname> structure.
- Drivers will typically want to perform some additional setup on this
- structure, such as allocating driver-specific data and storing a
- pointer to it in the DRM device's <structfield>dev_private</structfield>
- field. Drivers should also set the device's unique name using the
- <function>drm_dev_set_unique()</function> function. After it has been
- set up a device can be registered with the DRM subsystem by calling
- <function>drm_dev_register()</function>. This will cause the device to
- be exposed to userspace and will call the driver's
- <structfield>.load()</structfield> implementation. When a device is
- removed, the DRM device can safely be unregistered and freed by calling
- <function>drm_dev_unregister()</function> followed by a call to
- <function>drm_dev_unref()</function>.
- </para>
-!Edrivers/gpu/drm/drm_drv.c
- </sect2>
- <sect2>
- <title>Driver Load</title>
- <para>
- The <methodname>load</methodname> method is the driver and device
- initialization entry point. The method is responsible for allocating and
- initializing driver private data, performing resource allocation and
- mapping (e.g. acquiring
- clocks, mapping registers or allocating command buffers), initializing
- the memory manager (<xref linkend="drm-memory-management"/>), installing
- the IRQ handler (<xref linkend="drm-irq-registration"/>), setting up
- vertical blanking handling (<xref linkend="drm-vertical-blank"/>), mode
- setting (<xref linkend="drm-mode-setting"/>) and initial output
- configuration (<xref linkend="drm-kms-init"/>).
- </para>
- <note><para>
- If compatibility is a concern (e.g. with drivers converted over from
- User Mode Setting to Kernel Mode Setting), care must be taken to prevent
- device initialization and control that is incompatible with currently
- active userspace drivers. For instance, if user level mode setting
- drivers are in use, it would be problematic to perform output discovery
- & configuration at load time. Likewise, if user-level drivers
- unaware of memory management are in use, memory management and command
- buffer setup may need to be omitted. These requirements are
- driver-specific, and care needs to be taken to keep both old and new
- applications and libraries working.
- </para></note>
- <synopsis>int (*load) (struct drm_device *, unsigned long flags);</synopsis>
- <para>
- The method takes two arguments, a pointer to the newly created
- <structname>drm_device</structname> and flags. The flags are used to
- pass the <structfield>driver_data</structfield> field of the device id
- corresponding to the device passed to <function>drm_*_init()</function>.
- Only PCI devices currently use this, USB and platform DRM drivers have
- their <methodname>load</methodname> method called with flags to 0.
- </para>
- <sect3>
- <title>Driver Private Data</title>
- <para>
- The driver private hangs off the main
- <structname>drm_device</structname> structure and can be used for
- tracking various device-specific bits of information, like register
- offsets, command buffer status, register state for suspend/resume, etc.
- At load time, a driver may simply allocate one and set
- <structname>drm_device</structname>.<structfield>dev_priv</structfield>
- appropriately; it should be freed and
- <structname>drm_device</structname>.<structfield>dev_priv</structfield>
- set to NULL when the driver is unloaded.
- </para>
- </sect3>
- <sect3 id="drm-irq-registration">
- <title>IRQ Registration</title>
- <para>
- The DRM core tries to facilitate IRQ handler registration and
- unregistration by providing <function>drm_irq_install</function> and
- <function>drm_irq_uninstall</function> functions. Those functions only
- support a single interrupt per device, devices that use more than one
- IRQs need to be handled manually.
- </para>
- <sect4>
- <title>Managed IRQ Registration</title>
- <para>
- <function>drm_irq_install</function> starts by calling the
- <methodname>irq_preinstall</methodname> driver operation. The operation
- is optional and must make sure that the interrupt will not get fired by
- clearing all pending interrupt flags or disabling the interrupt.
- </para>
- <para>
- The passed-in IRQ will then be requested by a call to
- <function>request_irq</function>. If the DRIVER_IRQ_SHARED driver
- feature flag is set, a shared (IRQF_SHARED) IRQ handler will be
- requested.
- </para>
- <para>
- The IRQ handler function must be provided as the mandatory irq_handler
- driver operation. It will get passed directly to
- <function>request_irq</function> and thus has the same prototype as all
- IRQ handlers. It will get called with a pointer to the DRM device as the
- second argument.
- </para>
- <para>
- Finally the function calls the optional
- <methodname>irq_postinstall</methodname> driver operation. The operation
- usually enables interrupts (excluding the vblank interrupt, which is
- enabled separately), but drivers may choose to enable/disable interrupts
- at a different time.
- </para>
- <para>
- <function>drm_irq_uninstall</function> is similarly used to uninstall an
- IRQ handler. It starts by waking up all processes waiting on a vblank
- interrupt to make sure they don't hang, and then calls the optional
- <methodname>irq_uninstall</methodname> driver operation. The operation
- must disable all hardware interrupts. Finally the function frees the IRQ
- by calling <function>free_irq</function>.
- </para>
- </sect4>
- <sect4>
- <title>Manual IRQ Registration</title>
- <para>
- Drivers that require multiple interrupt handlers can't use the managed
- IRQ registration functions. In that case IRQs must be registered and
- unregistered manually (usually with the <function>request_irq</function>
- and <function>free_irq</function> functions, or their devm_* equivalent).
- </para>
- <para>
- When manually registering IRQs, drivers must not set the DRIVER_HAVE_IRQ
- driver feature flag, and must not provide the
- <methodname>irq_handler</methodname> driver operation. They must set the
- <structname>drm_device</structname> <structfield>irq_enabled</structfield>
- field to 1 upon registration of the IRQs, and clear it to 0 after
- unregistering the IRQs.
- </para>
- </sect4>
- </sect3>
- <sect3>
- <title>Memory Manager Initialization</title>
- <para>
- Every DRM driver requires a memory manager which must be initialized at
- load time. DRM currently contains two memory managers, the Translation
- Table Manager (TTM) and the Graphics Execution Manager (GEM).
- This document describes the use of the GEM memory manager only. See
- <xref linkend="drm-memory-management"/> for details.
- </para>
- </sect3>
- <sect3>
- <title>Miscellaneous Device Configuration</title>
- <para>
- Another task that may be necessary for PCI devices during configuration
- is mapping the video BIOS. On many devices, the VBIOS describes device
- configuration, LCD panel timings (if any), and contains flags indicating
- device state. Mapping the BIOS can be done using the pci_map_rom() call,
- a convenience function that takes care of mapping the actual ROM,
- whether it has been shadowed into memory (typically at address 0xc0000)
- or exists on the PCI device in the ROM BAR. Note that after the ROM has
- been mapped and any necessary information has been extracted, it should
- be unmapped; on many devices, the ROM address decoder is shared with
- other BARs, so leaving it mapped could cause undesired behaviour like
- hangs or memory corruption.
- <!--!Fdrivers/pci/rom.c pci_map_rom-->
- </para>
- </sect3>
- </sect2>
- </sect1>
-
- <!-- Internals: memory management -->
-
- <sect1 id="drm-memory-management">
- <title>Memory management</title>
- <para>
- Modern Linux systems require large amount of graphics memory to store
- frame buffers, textures, vertices and other graphics-related data. Given
- the very dynamic nature of many of that data, managing graphics memory
- efficiently is thus crucial for the graphics stack and plays a central
- role in the DRM infrastructure.
- </para>
- <para>
- The DRM core includes two memory managers, namely Translation Table Maps
- (TTM) and Graphics Execution Manager (GEM). TTM was the first DRM memory
- manager to be developed and tried to be a one-size-fits-them all
- solution. It provides a single userspace API to accommodate the need of
- all hardware, supporting both Unified Memory Architecture (UMA) devices
- and devices with dedicated video RAM (i.e. most discrete video cards).
- This resulted in a large, complex piece of code that turned out to be
- hard to use for driver development.
- </para>
- <para>
- GEM started as an Intel-sponsored project in reaction to TTM's
- complexity. Its design philosophy is completely different: instead of
- providing a solution to every graphics memory-related problems, GEM
- identified common code between drivers and created a support library to
- share it. GEM has simpler initialization and execution requirements than
- TTM, but has no video RAM management capabilities and is thus limited to
- UMA devices.
- </para>
- <sect2>
- <title>The Translation Table Manager (TTM)</title>
- <para>
- TTM design background and information belongs here.
- </para>
- <sect3>
- <title>TTM initialization</title>
- <warning><para>This section is outdated.</para></warning>
- <para>
- Drivers wishing to support TTM must fill out a drm_bo_driver
- structure. The structure contains several fields with function
- pointers for initializing the TTM, allocating and freeing memory,
- waiting for command completion and fence synchronization, and memory
- migration. See the radeon_ttm.c file for an example of usage.
- </para>
- <para>
- The ttm_global_reference structure is made up of several fields:
- </para>
- <programlisting>
- struct ttm_global_reference {
- enum ttm_global_types global_type;
- size_t size;
- void *object;
- int (*init) (struct ttm_global_reference *);
- void (*release) (struct ttm_global_reference *);
- };
- </programlisting>
- <para>
- There should be one global reference structure for your memory
- manager as a whole, and there will be others for each object
- created by the memory manager at runtime. Your global TTM should
- have a type of TTM_GLOBAL_TTM_MEM. The size field for the global
- object should be sizeof(struct ttm_mem_global), and the init and
- release hooks should point at your driver-specific init and
- release routines, which probably eventually call
- ttm_mem_global_init and ttm_mem_global_release, respectively.
- </para>
- <para>
- Once your global TTM accounting structure is set up and initialized
- by calling ttm_global_item_ref() on it,
- you need to create a buffer object TTM to
- provide a pool for buffer object allocation by clients and the
- kernel itself. The type of this object should be TTM_GLOBAL_TTM_BO,
- and its size should be sizeof(struct ttm_bo_global). Again,
- driver-specific init and release functions may be provided,
- likely eventually calling ttm_bo_global_init() and
- ttm_bo_global_release(), respectively. Also, like the previous
- object, ttm_global_item_ref() is used to create an initial reference
- count for the TTM, which will call your initialization function.
- </para>
- </sect3>
- </sect2>
- <sect2 id="drm-gem">
- <title>The Graphics Execution Manager (GEM)</title>
- <para>
- The GEM design approach has resulted in a memory manager that doesn't
- provide full coverage of all (or even all common) use cases in its
- userspace or kernel API. GEM exposes a set of standard memory-related
- operations to userspace and a set of helper functions to drivers, and let
- drivers implement hardware-specific operations with their own private API.
- </para>
- <para>
- The GEM userspace API is described in the
- <ulink url="http://lwn.net/Articles/283798/"><citetitle>GEM - the Graphics
- Execution Manager</citetitle></ulink> article on LWN. While slightly
- outdated, the document provides a good overview of the GEM API principles.
- Buffer allocation and read and write operations, described as part of the
- common GEM API, are currently implemented using driver-specific ioctls.
- </para>
- <para>
- GEM is data-agnostic. It manages abstract buffer objects without knowing
- what individual buffers contain. APIs that require knowledge of buffer
- contents or purpose, such as buffer allocation or synchronization
- primitives, are thus outside of the scope of GEM and must be implemented
- using driver-specific ioctls.
- </para>
- <para>
- On a fundamental level, GEM involves several operations:
- <itemizedlist>
- <listitem>Memory allocation and freeing</listitem>
- <listitem>Command execution</listitem>
- <listitem>Aperture management at command execution time</listitem>
- </itemizedlist>
- Buffer object allocation is relatively straightforward and largely
- provided by Linux's shmem layer, which provides memory to back each
- object.
- </para>
- <para>
- Device-specific operations, such as command execution, pinning, buffer
- read & write, mapping, and domain ownership transfers are left to
- driver-specific ioctls.
- </para>
- <sect3>
- <title>GEM Initialization</title>
- <para>
- Drivers that use GEM must set the DRIVER_GEM bit in the struct
- <structname>drm_driver</structname>
- <structfield>driver_features</structfield> field. The DRM core will
- then automatically initialize the GEM core before calling the
- <methodname>load</methodname> operation. Behind the scene, this will
- create a DRM Memory Manager object which provides an address space
- pool for object allocation.
- </para>
- <para>
- In a KMS configuration, drivers need to allocate and initialize a
- command ring buffer following core GEM initialization if required by
- the hardware. UMA devices usually have what is called a "stolen"
- memory region, which provides space for the initial framebuffer and
- large, contiguous memory regions required by the device. This space is
- typically not managed by GEM, and must be initialized separately into
- its own DRM MM object.
- </para>
- </sect3>
- <sect3>
- <title>GEM Objects Creation</title>
- <para>
- GEM splits creation of GEM objects and allocation of the memory that
- backs them in two distinct operations.
- </para>
- <para>
- GEM objects are represented by an instance of struct
- <structname>drm_gem_object</structname>. Drivers usually need to extend
- GEM objects with private information and thus create a driver-specific
- GEM object structure type that embeds an instance of struct
- <structname>drm_gem_object</structname>.
- </para>
- <para>
- To create a GEM object, a driver allocates memory for an instance of its
- specific GEM object type and initializes the embedded struct
- <structname>drm_gem_object</structname> with a call to
- <function>drm_gem_object_init</function>. The function takes a pointer to
- the DRM device, a pointer to the GEM object and the buffer object size
- in bytes.
- </para>
- <para>
- GEM uses shmem to allocate anonymous pageable memory.
- <function>drm_gem_object_init</function> will create an shmfs file of
- the requested size and store it into the struct
- <structname>drm_gem_object</structname> <structfield>filp</structfield>
- field. The memory is used as either main storage for the object when the
- graphics hardware uses system memory directly or as a backing store
- otherwise.
- </para>
- <para>
- Drivers are responsible for the actual physical pages allocation by
- calling <function>shmem_read_mapping_page_gfp</function> for each page.
- Note that they can decide to allocate pages when initializing the GEM
- object, or to delay allocation until the memory is needed (for instance
- when a page fault occurs as a result of a userspace memory access or
- when the driver needs to start a DMA transfer involving the memory).
- </para>
- <para>
- Anonymous pageable memory allocation is not always desired, for instance
- when the hardware requires physically contiguous system memory as is
- often the case in embedded devices. Drivers can create GEM objects with
- no shmfs backing (called private GEM objects) by initializing them with
- a call to <function>drm_gem_private_object_init</function> instead of
- <function>drm_gem_object_init</function>. Storage for private GEM
- objects must be managed by drivers.
- </para>
- <para>
- Drivers that do not need to extend GEM objects with private information
- can call the <function>drm_gem_object_alloc</function> function to
- allocate and initialize a struct <structname>drm_gem_object</structname>
- instance. The GEM core will call the optional driver
- <methodname>gem_init_object</methodname> operation after initializing
- the GEM object with <function>drm_gem_object_init</function>.
- <synopsis>int (*gem_init_object) (struct drm_gem_object *obj);</synopsis>
- </para>
- <para>
- No alloc-and-init function exists for private GEM objects.
- </para>
- </sect3>
- <sect3>
- <title>GEM Objects Lifetime</title>
- <para>
- All GEM objects are reference-counted by the GEM core. References can be
- acquired and release by <function>calling drm_gem_object_reference</function>
- and <function>drm_gem_object_unreference</function> respectively. The
- caller must hold the <structname>drm_device</structname>
- <structfield>struct_mutex</structfield> lock. As a convenience, GEM
- provides the <function>drm_gem_object_reference_unlocked</function> and
- <function>drm_gem_object_unreference_unlocked</function> functions that
- can be called without holding the lock.
- </para>
- <para>
- When the last reference to a GEM object is released the GEM core calls
- the <structname>drm_driver</structname>
- <methodname>gem_free_object</methodname> operation. That operation is
- mandatory for GEM-enabled drivers and must free the GEM object and all
- associated resources.
- </para>
- <para>
- <synopsis>void (*gem_free_object) (struct drm_gem_object *obj);</synopsis>
- Drivers are responsible for freeing all GEM object resources, including
- the resources created by the GEM core. If an mmap offset has been
- created for the object (in which case
- <structname>drm_gem_object</structname>::<structfield>map_list</structfield>::<structfield>map</structfield>
- is not NULL) it must be freed by a call to
- <function>drm_gem_free_mmap_offset</function>. The shmfs backing store
- must be released by calling <function>drm_gem_object_release</function>
- (that function can safely be called if no shmfs backing store has been
- created).
- </para>
- </sect3>
- <sect3>
- <title>GEM Objects Naming</title>
- <para>
- Communication between userspace and the kernel refers to GEM objects
- using local handles, global names or, more recently, file descriptors.
- All of those are 32-bit integer values; the usual Linux kernel limits
- apply to the file descriptors.
- </para>
- <para>
- GEM handles are local to a DRM file. Applications get a handle to a GEM
- object through a driver-specific ioctl, and can use that handle to refer
- to the GEM object in other standard or driver-specific ioctls. Closing a
- DRM file handle frees all its GEM handles and dereferences the
- associated GEM objects.
- </para>
- <para>
- To create a handle for a GEM object drivers call
- <function>drm_gem_handle_create</function>. The function takes a pointer
- to the DRM file and the GEM object and returns a locally unique handle.
- When the handle is no longer needed drivers delete it with a call to
- <function>drm_gem_handle_delete</function>. Finally the GEM object
- associated with a handle can be retrieved by a call to
- <function>drm_gem_object_lookup</function>.
- </para>
- <para>
- Handles don't take ownership of GEM objects, they only take a reference
- to the object that will be dropped when the handle is destroyed. To
- avoid leaking GEM objects, drivers must make sure they drop the
- reference(s) they own (such as the initial reference taken at object
- creation time) as appropriate, without any special consideration for the
- handle. For example, in the particular case of combined GEM object and
- handle creation in the implementation of the
- <methodname>dumb_create</methodname> operation, drivers must drop the
- initial reference to the GEM object before returning the handle.
- </para>
- <para>
- GEM names are similar in purpose to handles but are not local to DRM
- files. They can be passed between processes to reference a GEM object
- globally. Names can't be used directly to refer to objects in the DRM
- API, applications must convert handles to names and names to handles
- using the DRM_IOCTL_GEM_FLINK and DRM_IOCTL_GEM_OPEN ioctls
- respectively. The conversion is handled by the DRM core without any
- driver-specific support.
- </para>
- <para>
- GEM also supports buffer sharing with dma-buf file descriptors through
- PRIME. GEM-based drivers must use the provided helpers functions to
- implement the exporting and importing correctly. See <xref linkend="drm-prime-support" />.
- Since sharing file descriptors is inherently more secure than the
- easily guessable and global GEM names it is the preferred buffer
- sharing mechanism. Sharing buffers through GEM names is only supported
- for legacy userspace. Furthermore PRIME also allows cross-device
- buffer sharing since it is based on dma-bufs.
- </para>
- </sect3>
- <sect3 id="drm-gem-objects-mapping">
- <title>GEM Objects Mapping</title>
- <para>
- Because mapping operations are fairly heavyweight GEM favours
- read/write-like access to buffers, implemented through driver-specific
- ioctls, over mapping buffers to userspace. However, when random access
- to the buffer is needed (to perform software rendering for instance),
- direct access to the object can be more efficient.
- </para>
- <para>
- The mmap system call can't be used directly to map GEM objects, as they
- don't have their own file handle. Two alternative methods currently
- co-exist to map GEM objects to userspace. The first method uses a
- driver-specific ioctl to perform the mapping operation, calling
- <function>do_mmap</function> under the hood. This is often considered
- dubious, seems to be discouraged for new GEM-enabled drivers, and will
- thus not be described here.
- </para>
- <para>
- The second method uses the mmap system call on the DRM file handle.
- <synopsis>void *mmap(void *addr, size_t length, int prot, int flags, int fd,
- off_t offset);</synopsis>
- DRM identifies the GEM object to be mapped by a fake offset passed
- through the mmap offset argument. Prior to being mapped, a GEM object
- must thus be associated with a fake offset. To do so, drivers must call
- <function>drm_gem_create_mmap_offset</function> on the object. The
- function allocates a fake offset range from a pool and stores the
- offset divided by PAGE_SIZE in
- <literal>obj->map_list.hash.key</literal>. Care must be taken not to
- call <function>drm_gem_create_mmap_offset</function> if a fake offset
- has already been allocated for the object. This can be tested by
- <literal>obj->map_list.map</literal> being non-NULL.
- </para>
- <para>
- Once allocated, the fake offset value
- (<literal>obj->map_list.hash.key << PAGE_SHIFT</literal>)
- must be passed to the application in a driver-specific way and can then
- be used as the mmap offset argument.
- </para>
- <para>
- The GEM core provides a helper method <function>drm_gem_mmap</function>
- to handle object mapping. The method can be set directly as the mmap
- file operation handler. It will look up the GEM object based on the
- offset value and set the VMA operations to the
- <structname>drm_driver</structname> <structfield>gem_vm_ops</structfield>
- field. Note that <function>drm_gem_mmap</function> doesn't map memory to
- userspace, but relies on the driver-provided fault handler to map pages
- individually.
- </para>
- <para>
- To use <function>drm_gem_mmap</function>, drivers must fill the struct
- <structname>drm_driver</structname> <structfield>gem_vm_ops</structfield>
- field with a pointer to VM operations.
- </para>
- <para>
- <synopsis>struct vm_operations_struct *gem_vm_ops
-
- struct vm_operations_struct {
- void (*open)(struct vm_area_struct * area);
- void (*close)(struct vm_area_struct * area);
- int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
- };</synopsis>
- </para>
- <para>
- The <methodname>open</methodname> and <methodname>close</methodname>
- operations must update the GEM object reference count. Drivers can use
- the <function>drm_gem_vm_open</function> and
- <function>drm_gem_vm_close</function> helper functions directly as open
- and close handlers.
- </para>
- <para>
- The fault operation handler is responsible for mapping individual pages
- to userspace when a page fault occurs. Depending on the memory
- allocation scheme, drivers can allocate pages at fault time, or can
- decide to allocate memory for the GEM object at the time the object is
- created.
- </para>
- <para>
- Drivers that want to map the GEM object upfront instead of handling page
- faults can implement their own mmap file operation handler.
- </para>
- </sect3>
- <sect3>
- <title>Memory Coherency</title>
- <para>
- When mapped to the device or used in a command buffer, backing pages
- for an object are flushed to memory and marked write combined so as to
- be coherent with the GPU. Likewise, if the CPU accesses an object
- after the GPU has finished rendering to the object, then the object
- must be made coherent with the CPU's view of memory, usually involving
- GPU cache flushing of various kinds. This core CPU<->GPU
- coherency management is provided by a device-specific ioctl, which
- evaluates an object's current domain and performs any necessary
- flushing or synchronization to put the object into the desired
- coherency domain (note that the object may be busy, i.e. an active
- render target; in that case, setting the domain blocks the client and
- waits for rendering to complete before performing any necessary
- flushing operations).
- </para>
- </sect3>
- <sect3>
- <title>Command Execution</title>
- <para>
- Perhaps the most important GEM function for GPU devices is providing a
- command execution interface to clients. Client programs construct
- command buffers containing references to previously allocated memory
- objects, and then submit them to GEM. At that point, GEM takes care to
- bind all the objects into the GTT, execute the buffer, and provide
- necessary synchronization between clients accessing the same buffers.
- This often involves evicting some objects from the GTT and re-binding
- others (a fairly expensive operation), and providing relocation
- support which hides fixed GTT offsets from clients. Clients must take
- care not to submit command buffers that reference more objects than
- can fit in the GTT; otherwise, GEM will reject them and no rendering
- will occur. Similarly, if several objects in the buffer require fence
- registers to be allocated for correct rendering (e.g. 2D blits on
- pre-965 chips), care must be taken not to require more fence registers
- than are available to the client. Such resource management should be
- abstracted from the client in libdrm.
- </para>
- </sect3>
- <sect3>
- <title>GEM Function Reference</title>
-!Edrivers/gpu/drm/drm_gem.c
- </sect3>
- </sect2>
- <sect2>
- <title>VMA Offset Manager</title>
-!Pdrivers/gpu/drm/drm_vma_manager.c vma offset manager
-!Edrivers/gpu/drm/drm_vma_manager.c
-!Iinclude/drm/drm_vma_manager.h
- </sect2>
- <sect2 id="drm-prime-support">
- <title>PRIME Buffer Sharing</title>
- <para>
- PRIME is the cross device buffer sharing framework in drm, originally
- created for the OPTIMUS range of multi-gpu platforms. To userspace
- PRIME buffers are dma-buf based file descriptors.
- </para>
- <sect3>
- <title>Overview and Driver Interface</title>
- <para>
- Similar to GEM global names, PRIME file descriptors are
- also used to share buffer objects across processes. They offer
- additional security: as file descriptors must be explicitly sent over
- UNIX domain sockets to be shared between applications, they can't be
- guessed like the globally unique GEM names.
- </para>
- <para>
- Drivers that support the PRIME
- API must set the DRIVER_PRIME bit in the struct
- <structname>drm_driver</structname>
- <structfield>driver_features</structfield> field, and implement the
- <methodname>prime_handle_to_fd</methodname> and
- <methodname>prime_fd_to_handle</methodname> operations.
- </para>
- <para>
- <synopsis>int (*prime_handle_to_fd)(struct drm_device *dev,
- struct drm_file *file_priv, uint32_t handle,
- uint32_t flags, int *prime_fd);
-int (*prime_fd_to_handle)(struct drm_device *dev,
- struct drm_file *file_priv, int prime_fd,
- uint32_t *handle);</synopsis>
- Those two operations convert a handle to a PRIME file descriptor and
- vice versa. Drivers must use the kernel dma-buf buffer sharing framework
- to manage the PRIME file descriptors. Similar to the mode setting
- API PRIME is agnostic to the underlying buffer object manager, as
- long as handles are 32bit unsigned integers.
- </para>
- <para>
- While non-GEM drivers must implement the operations themselves, GEM
- drivers must use the <function>drm_gem_prime_handle_to_fd</function>
- and <function>drm_gem_prime_fd_to_handle</function> helper functions.
- Those helpers rely on the driver
- <methodname>gem_prime_export</methodname> and
- <methodname>gem_prime_import</methodname> operations to create a dma-buf
- instance from a GEM object (dma-buf exporter role) and to create a GEM
- object from a dma-buf instance (dma-buf importer role).
- </para>
- <para>
- <synopsis>struct dma_buf * (*gem_prime_export)(struct drm_device *dev,
- struct drm_gem_object *obj,
- int flags);
-struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev,
- struct dma_buf *dma_buf);</synopsis>
- These two operations are mandatory for GEM drivers that support
- PRIME.
- </para>
- </sect3>
- <sect3>
- <title>PRIME Helper Functions</title>
-!Pdrivers/gpu/drm/drm_prime.c PRIME Helpers
- </sect3>
- </sect2>
- <sect2>
- <title>PRIME Function References</title>
-!Edrivers/gpu/drm/drm_prime.c
- </sect2>
- <sect2>
- <title>DRM MM Range Allocator</title>
- <sect3>
- <title>Overview</title>
-!Pdrivers/gpu/drm/drm_mm.c Overview
- </sect3>
- <sect3>
- <title>LRU Scan/Eviction Support</title>
-!Pdrivers/gpu/drm/drm_mm.c lru scan roaster
- </sect3>
- </sect2>
- <sect2>
- <title>DRM MM Range Allocator Function References</title>
-!Edrivers/gpu/drm/drm_mm.c
-!Iinclude/drm/drm_mm.h
- </sect2>
- <sect2>
- <title>CMA Helper Functions Reference</title>
-!Pdrivers/gpu/drm/drm_gem_cma_helper.c cma helpers
-!Edrivers/gpu/drm/drm_gem_cma_helper.c
-!Iinclude/drm/drm_gem_cma_helper.h
- </sect2>
- </sect1>
-
- <!-- Internals: mode setting -->
-
- <sect1 id="drm-mode-setting">
- <title>Mode Setting</title>
- <para>
- Drivers must initialize the mode setting core by calling
- <function>drm_mode_config_init</function> on the DRM device. The function
- initializes the <structname>drm_device</structname>
- <structfield>mode_config</structfield> field and never fails. Once done,
- mode configuration must be setup by initializing the following fields.
- </para>
- <itemizedlist>
- <listitem>
- <synopsis>int min_width, min_height;
-int max_width, max_height;</synopsis>
- <para>
- Minimum and maximum width and height of the frame buffers in pixel
- units.
- </para>
- </listitem>
- <listitem>
- <synopsis>struct drm_mode_config_funcs *funcs;</synopsis>
- <para>Mode setting functions.</para>
- </listitem>
- </itemizedlist>
- <sect2>
- <title>Display Modes Function Reference</title>
-!Iinclude/drm/drm_modes.h
-!Edrivers/gpu/drm/drm_modes.c
- </sect2>
- <sect2>
- <title>Atomic Mode Setting Function Reference</title>
-!Edrivers/gpu/drm/drm_atomic.c
- </sect2>
- <sect2>
- <title>Frame Buffer Creation</title>
- <synopsis>struct drm_framebuffer *(*fb_create)(struct drm_device *dev,
- struct drm_file *file_priv,
- struct drm_mode_fb_cmd2 *mode_cmd);</synopsis>
- <para>
- Frame buffers are abstract memory objects that provide a source of
- pixels to scanout to a CRTC. Applications explicitly request the
- creation of frame buffers through the DRM_IOCTL_MODE_ADDFB(2) ioctls and
- receive an opaque handle that can be passed to the KMS CRTC control,
- plane configuration and page flip functions.
- </para>
- <para>
- Frame buffers rely on the underneath memory manager for low-level memory
- operations. When creating a frame buffer applications pass a memory
- handle (or a list of memory handles for multi-planar formats) through
- the <parameter>drm_mode_fb_cmd2</parameter> argument. For drivers using
- GEM as their userspace buffer management interface this would be a GEM
- handle. Drivers are however free to use their own backing storage object
- handles, e.g. vmwgfx directly exposes special TTM handles to userspace
- and so expects TTM handles in the create ioctl and not GEM handles.
- </para>
- <para>
- Drivers must first validate the requested frame buffer parameters passed
- through the mode_cmd argument. In particular this is where invalid
- sizes, pixel formats or pitches can be caught.
- </para>
- <para>
- If the parameters are deemed valid, drivers then create, initialize and
- return an instance of struct <structname>drm_framebuffer</structname>.
- If desired the instance can be embedded in a larger driver-specific
- structure. Drivers must fill its <structfield>width</structfield>,
- <structfield>height</structfield>, <structfield>pitches</structfield>,
- <structfield>offsets</structfield>, <structfield>depth</structfield>,
- <structfield>bits_per_pixel</structfield> and
- <structfield>pixel_format</structfield> fields from the values passed
- through the <parameter>drm_mode_fb_cmd2</parameter> argument. They
- should call the <function>drm_helper_mode_fill_fb_struct</function>
- helper function to do so.
- </para>
-
- <para>
- The initialization of the new framebuffer instance is finalized with a
- call to <function>drm_framebuffer_init</function> which takes a pointer
- to DRM frame buffer operations (struct
- <structname>drm_framebuffer_funcs</structname>). Note that this function
- publishes the framebuffer and so from this point on it can be accessed
- concurrently from other threads. Hence it must be the last step in the
- driver's framebuffer initialization sequence. Frame buffer operations
- are
- <itemizedlist>
- <listitem>
- <synopsis>int (*create_handle)(struct drm_framebuffer *fb,
- struct drm_file *file_priv, unsigned int *handle);</synopsis>
- <para>
- Create a handle to the frame buffer underlying memory object. If
- the frame buffer uses a multi-plane format, the handle will
- reference the memory object associated with the first plane.
- </para>
- <para>
- Drivers call <function>drm_gem_handle_create</function> to create
- the handle.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*destroy)(struct drm_framebuffer *framebuffer);</synopsis>
- <para>
- Destroy the frame buffer object and frees all associated
- resources. Drivers must call
- <function>drm_framebuffer_cleanup</function> to free resources
- allocated by the DRM core for the frame buffer object, and must
- make sure to unreference all memory objects associated with the
- frame buffer. Handles created by the
- <methodname>create_handle</methodname> operation are released by
- the DRM core.
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*dirty)(struct drm_framebuffer *framebuffer,
- struct drm_file *file_priv, unsigned flags, unsigned color,
- struct drm_clip_rect *clips, unsigned num_clips);</synopsis>
- <para>
- This optional operation notifies the driver that a region of the
- frame buffer has changed in response to a DRM_IOCTL_MODE_DIRTYFB
- ioctl call.
- </para>
- </listitem>
- </itemizedlist>
- </para>
- <para>
- The lifetime of a drm framebuffer is controlled with a reference count,
- drivers can grab additional references with
- <function>drm_framebuffer_reference</function>and drop them
- again with <function>drm_framebuffer_unreference</function>. For
- driver-private framebuffers for which the last reference is never
- dropped (e.g. for the fbdev framebuffer when the struct
- <structname>drm_framebuffer</structname> is embedded into the fbdev
- helper struct) drivers can manually clean up a framebuffer at module
- unload time with
- <function>drm_framebuffer_unregister_private</function>.
- </para>
- </sect2>
- <sect2>
- <title>Dumb Buffer Objects</title>
- <para>
- The KMS API doesn't standardize backing storage object creation and
- leaves it to driver-specific ioctls. Furthermore actually creating a
- buffer object even for GEM-based drivers is done through a
- driver-specific ioctl - GEM only has a common userspace interface for
- sharing and destroying objects. While not an issue for full-fledged
- graphics stacks that include device-specific userspace components (in
- libdrm for instance), this limit makes DRM-based early boot graphics
- unnecessarily complex.
- </para>
- <para>
- Dumb objects partly alleviate the problem by providing a standard
- API to create dumb buffers suitable for scanout, which can then be used
- to create KMS frame buffers.
- </para>
- <para>
- To support dumb objects drivers must implement the
- <methodname>dumb_create</methodname>,
- <methodname>dumb_destroy</methodname> and
- <methodname>dumb_map_offset</methodname> operations.
- </para>
- <itemizedlist>
- <listitem>
- <synopsis>int (*dumb_create)(struct drm_file *file_priv, struct drm_device *dev,
- struct drm_mode_create_dumb *args);</synopsis>
- <para>
- The <methodname>dumb_create</methodname> operation creates a driver
- object (GEM or TTM handle) suitable for scanout based on the
- width, height and depth from the struct
- <structname>drm_mode_create_dumb</structname> argument. It fills the
- argument's <structfield>handle</structfield>,
- <structfield>pitch</structfield> and <structfield>size</structfield>
- fields with a handle for the newly created object and its line
- pitch and size in bytes.
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*dumb_destroy)(struct drm_file *file_priv, struct drm_device *dev,
- uint32_t handle);</synopsis>
- <para>
- The <methodname>dumb_destroy</methodname> operation destroys a dumb
- object created by <methodname>dumb_create</methodname>.
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*dumb_map_offset)(struct drm_file *file_priv, struct drm_device *dev,
- uint32_t handle, uint64_t *offset);</synopsis>
- <para>
- The <methodname>dumb_map_offset</methodname> operation associates an
- mmap fake offset with the object given by the handle and returns
- it. Drivers must use the
- <function>drm_gem_create_mmap_offset</function> function to
- associate the fake offset as described in
- <xref linkend="drm-gem-objects-mapping"/>.
- </para>
- </listitem>
- </itemizedlist>
- <para>
- Note that dumb objects may not be used for gpu acceleration, as has been
- attempted on some ARM embedded platforms. Such drivers really must have
- a hardware-specific ioctl to allocate suitable buffer objects.
- </para>
- </sect2>
- <sect2>
- <title>Output Polling</title>
- <synopsis>void (*output_poll_changed)(struct drm_device *dev);</synopsis>
- <para>
- This operation notifies the driver that the status of one or more
- connectors has changed. Drivers that use the fb helper can just call the
- <function>drm_fb_helper_hotplug_event</function> function to handle this
- operation.
- </para>
- </sect2>
- <sect2>
- <title>Locking</title>
- <para>
- Beside some lookup structures with their own locking (which is hidden
- behind the interface functions) most of the modeset state is protected
- by the <code>dev-<mode_config.lock</code> mutex and additionally
- per-crtc locks to allow cursor updates, pageflips and similar operations
- to occur concurrently with background tasks like output detection.
- Operations which cross domains like a full modeset always grab all
- locks. Drivers there need to protect resources shared between crtcs with
- additional locking. They also need to be careful to always grab the
- relevant crtc locks if a modset functions touches crtc state, e.g. for
- load detection (which does only grab the <code>mode_config.lock</code>
- to allow concurrent screen updates on live crtcs).
- </para>
- </sect2>
- </sect1>
-
- <!-- Internals: kms initialization and cleanup -->
-
- <sect1 id="drm-kms-init">
- <title>KMS Initialization and Cleanup</title>
- <para>
- A KMS device is abstracted and exposed as a set of planes, CRTCs, encoders
- and connectors. KMS drivers must thus create and initialize all those
- objects at load time after initializing mode setting.
- </para>
- <sect2>
- <title>CRTCs (struct <structname>drm_crtc</structname>)</title>
- <para>
- A CRTC is an abstraction representing a part of the chip that contains a
- pointer to a scanout buffer. Therefore, the number of CRTCs available
- determines how many independent scanout buffers can be active at any
- given time. The CRTC structure contains several fields to support this:
- a pointer to some video memory (abstracted as a frame buffer object), a
- display mode, and an (x, y) offset into the video memory to support
- panning or configurations where one piece of video memory spans multiple
- CRTCs.
- </para>
- <sect3>
- <title>CRTC Initialization</title>
- <para>
- A KMS device must create and register at least one struct
- <structname>drm_crtc</structname> instance. The instance is allocated
- and zeroed by the driver, possibly as part of a larger structure, and
- registered with a call to <function>drm_crtc_init</function> with a
- pointer to CRTC functions.
- </para>
- </sect3>
- <sect3 id="drm-kms-crtcops">
- <title>CRTC Operations</title>
- <sect4>
- <title>Set Configuration</title>
- <synopsis>int (*set_config)(struct drm_mode_set *set);</synopsis>
- <para>
- Apply a new CRTC configuration to the device. The configuration
- specifies a CRTC, a frame buffer to scan out from, a (x,y) position in
- the frame buffer, a display mode and an array of connectors to drive
- with the CRTC if possible.
- </para>
- <para>
- If the frame buffer specified in the configuration is NULL, the driver
- must detach all encoders connected to the CRTC and all connectors
- attached to those encoders and disable them.
- </para>
- <para>
- This operation is called with the mode config lock held.
- </para>
- <note><para>
- Note that the drm core has no notion of restoring the mode setting
- state after resume, since all resume handling is in the full
- responsibility of the driver. The common mode setting helper library
- though provides a helper which can be used for this:
- <function>drm_helper_resume_force_mode</function>.
- </para></note>
- </sect4>
- <sect4>
- <title>Page Flipping</title>
- <synopsis>int (*page_flip)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
- struct drm_pending_vblank_event *event);</synopsis>
- <para>
- Schedule a page flip to the given frame buffer for the CRTC. This
- operation is called with the mode config mutex held.
- </para>
- <para>
- Page flipping is a synchronization mechanism that replaces the frame
- buffer being scanned out by the CRTC with a new frame buffer during
- vertical blanking, avoiding tearing. When an application requests a page
- flip the DRM core verifies that the new frame buffer is large enough to
- be scanned out by the CRTC in the currently configured mode and then
- calls the CRTC <methodname>page_flip</methodname> operation with a
- pointer to the new frame buffer.
- </para>
- <para>
- The <methodname>page_flip</methodname> operation schedules a page flip.
- Once any pending rendering targeting the new frame buffer has
- completed, the CRTC will be reprogrammed to display that frame buffer
- after the next vertical refresh. The operation must return immediately
- without waiting for rendering or page flip to complete and must block
- any new rendering to the frame buffer until the page flip completes.
- </para>
- <para>
- If a page flip can be successfully scheduled the driver must set the
- <code>drm_crtc->fb</code> field to the new framebuffer pointed to
- by <code>fb</code>. This is important so that the reference counting
- on framebuffers stays balanced.
- </para>
- <para>
- If a page flip is already pending, the
- <methodname>page_flip</methodname> operation must return
- -<errorname>EBUSY</errorname>.
- </para>
- <para>
- To synchronize page flip to vertical blanking the driver will likely
- need to enable vertical blanking interrupts. It should call
- <function>drm_vblank_get</function> for that purpose, and call
- <function>drm_vblank_put</function> after the page flip completes.
- </para>
- <para>
- If the application has requested to be notified when page flip completes
- the <methodname>page_flip</methodname> operation will be called with a
- non-NULL <parameter>event</parameter> argument pointing to a
- <structname>drm_pending_vblank_event</structname> instance. Upon page
- flip completion the driver must call <methodname>drm_send_vblank_event</methodname>
- to fill in the event and send to wake up any waiting processes.
- This can be performed with
- <programlisting><![CDATA[
- spin_lock_irqsave(&dev->event_lock, flags);
- ...
- drm_send_vblank_event(dev, pipe, event);
- spin_unlock_irqrestore(&dev->event_lock, flags);
- ]]></programlisting>
- </para>
- <note><para>
- FIXME: Could drivers that don't need to wait for rendering to complete
- just add the event to <literal>dev->vblank_event_list</literal> and
- let the DRM core handle everything, as for "normal" vertical blanking
- events?
- </para></note>
- <para>
- While waiting for the page flip to complete, the
- <literal>event->base.link</literal> list head can be used freely by
- the driver to store the pending event in a driver-specific list.
- </para>
- <para>
- If the file handle is closed before the event is signaled, drivers must
- take care to destroy the event in their
- <methodname>preclose</methodname> operation (and, if needed, call
- <function>drm_vblank_put</function>).
- </para>
- </sect4>
- <sect4>
- <title>Miscellaneous</title>
- <itemizedlist>
- <listitem>
- <synopsis>void (*set_property)(struct drm_crtc *crtc,
- struct drm_property *property, uint64_t value);</synopsis>
- <para>
- Set the value of the given CRTC property to
- <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
- for more information about properties.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*gamma_set)(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
- uint32_t start, uint32_t size);</synopsis>
- <para>
- Apply a gamma table to the device. The operation is optional.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*destroy)(struct drm_crtc *crtc);</synopsis>
- <para>
- Destroy the CRTC when not needed anymore. See
- <xref linkend="drm-kms-init"/>.
- </para>
- </listitem>
- </itemizedlist>
- </sect4>
- </sect3>
- </sect2>
- <sect2>
- <title>Planes (struct <structname>drm_plane</structname>)</title>
- <para>
- A plane represents an image source that can be blended with or overlayed
- on top of a CRTC during the scanout process. Planes are associated with
- a frame buffer to crop a portion of the image memory (source) and
- optionally scale it to a destination size. The result is then blended
- with or overlayed on top of a CRTC.
- </para>
- <para>
- The DRM core recognizes three types of planes:
- <itemizedlist>
- <listitem>
- DRM_PLANE_TYPE_PRIMARY represents a "main" plane for a CRTC. Primary
- planes are the planes operated upon by CRTC modesetting and flipping
- operations described in <xref linkend="drm-kms-crtcops"/>.
- </listitem>
- <listitem>
- DRM_PLANE_TYPE_CURSOR represents a "cursor" plane for a CRTC. Cursor
- planes are the planes operated upon by the DRM_IOCTL_MODE_CURSOR and
- DRM_IOCTL_MODE_CURSOR2 ioctls.
- </listitem>
- <listitem>
- DRM_PLANE_TYPE_OVERLAY represents all non-primary, non-cursor planes.
- Some drivers refer to these types of planes as "sprites" internally.
- </listitem>
- </itemizedlist>
- For compatibility with legacy userspace, only overlay planes are made
- available to userspace by default. Userspace clients may set the
- DRM_CLIENT_CAP_UNIVERSAL_PLANES client capability bit to indicate that
- they wish to receive a universal plane list containing all plane types.
- </para>
- <sect3>
- <title>Plane Initialization</title>
- <para>
- To create a plane, a KMS drivers allocates and
- zeroes an instances of struct <structname>drm_plane</structname>
- (possibly as part of a larger structure) and registers it with a call
- to <function>drm_universal_plane_init</function>. The function takes a bitmask
- of the CRTCs that can be associated with the plane, a pointer to the
- plane functions, a list of format supported formats, and the type of
- plane (primary, cursor, or overlay) being initialized.
- </para>
- <para>
- Cursor and overlay planes are optional. All drivers should provide
- one primary plane per CRTC (although this requirement may change in
- the future); drivers that do not wish to provide special handling for
- primary planes may make use of the helper functions described in
- <xref linkend="drm-kms-planehelpers"/> to create and register a
- primary plane with standard capabilities.
- </para>
- </sect3>
- <sect3>
- <title>Plane Operations</title>
- <itemizedlist>
- <listitem>
- <synopsis>int (*update_plane)(struct drm_plane *plane, struct drm_crtc *crtc,
- struct drm_framebuffer *fb, int crtc_x, int crtc_y,
- unsigned int crtc_w, unsigned int crtc_h,
- uint32_t src_x, uint32_t src_y,
- uint32_t src_w, uint32_t src_h);</synopsis>
- <para>
- Enable and configure the plane to use the given CRTC and frame buffer.
- </para>
- <para>
- The source rectangle in frame buffer memory coordinates is given by
- the <parameter>src_x</parameter>, <parameter>src_y</parameter>,
- <parameter>src_w</parameter> and <parameter>src_h</parameter>
- parameters (as 16.16 fixed point values). Devices that don't support
- subpixel plane coordinates can ignore the fractional part.
- </para>
- <para>
- The destination rectangle in CRTC coordinates is given by the
- <parameter>crtc_x</parameter>, <parameter>crtc_y</parameter>,
- <parameter>crtc_w</parameter> and <parameter>crtc_h</parameter>
- parameters (as integer values). Devices scale the source rectangle to
- the destination rectangle. If scaling is not supported, and the source
- rectangle size doesn't match the destination rectangle size, the
- driver must return a -<errorname>EINVAL</errorname> error.
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*disable_plane)(struct drm_plane *plane);</synopsis>
- <para>
- Disable the plane. The DRM core calls this method in response to a
- DRM_IOCTL_MODE_SETPLANE ioctl call with the frame buffer ID set to 0.
- Disabled planes must not be processed by the CRTC.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*destroy)(struct drm_plane *plane);</synopsis>
- <para>
- Destroy the plane when not needed anymore. See
- <xref linkend="drm-kms-init"/>.
- </para>
- </listitem>
- </itemizedlist>
- </sect3>
- </sect2>
- <sect2>
- <title>Encoders (struct <structname>drm_encoder</structname>)</title>
- <para>
- An encoder takes pixel data from a CRTC and converts it to a format
- suitable for any attached connectors. On some devices, it may be
- possible to have a CRTC send data to more than one encoder. In that
- case, both encoders would receive data from the same scanout buffer,
- resulting in a "cloned" display configuration across the connectors
- attached to each encoder.
- </para>
- <sect3>
- <title>Encoder Initialization</title>
- <para>
- As for CRTCs, a KMS driver must create, initialize and register at
- least one struct <structname>drm_encoder</structname> instance. The
- instance is allocated and zeroed by the driver, possibly as part of a
- larger structure.
- </para>
- <para>
- Drivers must initialize the struct <structname>drm_encoder</structname>
- <structfield>possible_crtcs</structfield> and
- <structfield>possible_clones</structfield> fields before registering the
- encoder. Both fields are bitmasks of respectively the CRTCs that the
- encoder can be connected to, and sibling encoders candidate for cloning.
- </para>
- <para>
- After being initialized, the encoder must be registered with a call to
- <function>drm_encoder_init</function>. The function takes a pointer to
- the encoder functions and an encoder type. Supported types are
- <itemizedlist>
- <listitem>
- DRM_MODE_ENCODER_DAC for VGA and analog on DVI-I/DVI-A
- </listitem>
- <listitem>
- DRM_MODE_ENCODER_TMDS for DVI, HDMI and (embedded) DisplayPort
- </listitem>
- <listitem>
- DRM_MODE_ENCODER_LVDS for display panels
- </listitem>
- <listitem>
- DRM_MODE_ENCODER_TVDAC for TV output (Composite, S-Video, Component,
- SCART)
- </listitem>
- <listitem>
- DRM_MODE_ENCODER_VIRTUAL for virtual machine displays
- </listitem>
- </itemizedlist>
- </para>
- <para>
- Encoders must be attached to a CRTC to be used. DRM drivers leave
- encoders unattached at initialization time. Applications (or the fbdev
- compatibility layer when implemented) are responsible for attaching the
- encoders they want to use to a CRTC.
- </para>
- </sect3>
- <sect3>
- <title>Encoder Operations</title>
- <itemizedlist>
- <listitem>
- <synopsis>void (*destroy)(struct drm_encoder *encoder);</synopsis>
- <para>
- Called to destroy the encoder when not needed anymore. See
- <xref linkend="drm-kms-init"/>.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*set_property)(struct drm_plane *plane,
- struct drm_property *property, uint64_t value);</synopsis>
- <para>
- Set the value of the given plane property to
- <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
- for more information about properties.
- </para>
- </listitem>
- </itemizedlist>
- </sect3>
- </sect2>
- <sect2>
- <title>Connectors (struct <structname>drm_connector</structname>)</title>
- <para>
- A connector is the final destination for pixel data on a device, and
- usually connects directly to an external display device like a monitor
- or laptop panel. A connector can only be attached to one encoder at a
- time. The connector is also the structure where information about the
- attached display is kept, so it contains fields for display data, EDID
- data, DPMS & connection status, and information about modes
- supported on the attached displays.
- </para>
- <sect3>
- <title>Connector Initialization</title>
- <para>
- Finally a KMS driver must create, initialize, register and attach at
- least one struct <structname>drm_connector</structname> instance. The
- instance is created as other KMS objects and initialized by setting the
- following fields.
- </para>
- <variablelist>
- <varlistentry>
- <term><structfield>interlace_allowed</structfield></term>
- <listitem><para>
- Whether the connector can handle interlaced modes.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term><structfield>doublescan_allowed</structfield></term>
- <listitem><para>
- Whether the connector can handle doublescan.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term><structfield>display_info
- </structfield></term>
- <listitem><para>
- Display information is filled from EDID information when a display
- is detected. For non hot-pluggable displays such as flat panels in
- embedded systems, the driver should initialize the
- <structfield>display_info</structfield>.<structfield>width_mm</structfield>
- and
- <structfield>display_info</structfield>.<structfield>height_mm</structfield>
- fields with the physical size of the display.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term id="drm-kms-connector-polled"><structfield>polled</structfield></term>
- <listitem><para>
- Connector polling mode, a combination of
- <variablelist>
- <varlistentry>
- <term>DRM_CONNECTOR_POLL_HPD</term>
- <listitem><para>
- The connector generates hotplug events and doesn't need to be
- periodically polled. The CONNECT and DISCONNECT flags must not
- be set together with the HPD flag.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_CONNECTOR_POLL_CONNECT</term>
- <listitem><para>
- Periodically poll the connector for connection.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_CONNECTOR_POLL_DISCONNECT</term>
- <listitem><para>
- Periodically poll the connector for disconnection.
- </para></listitem>
- </varlistentry>
- </variablelist>
- Set to 0 for connectors that don't support connection status
- discovery.
- </para></listitem>
- </varlistentry>
- </variablelist>
- <para>
- The connector is then registered with a call to
- <function>drm_connector_init</function> with a pointer to the connector
- functions and a connector type, and exposed through sysfs with a call to
- <function>drm_connector_register</function>.
- </para>
- <para>
- Supported connector types are
- <itemizedlist>
- <listitem>DRM_MODE_CONNECTOR_VGA</listitem>
- <listitem>DRM_MODE_CONNECTOR_DVII</listitem>
- <listitem>DRM_MODE_CONNECTOR_DVID</listitem>
- <listitem>DRM_MODE_CONNECTOR_DVIA</listitem>
- <listitem>DRM_MODE_CONNECTOR_Composite</listitem>
- <listitem>DRM_MODE_CONNECTOR_SVIDEO</listitem>
- <listitem>DRM_MODE_CONNECTOR_LVDS</listitem>
- <listitem>DRM_MODE_CONNECTOR_Component</listitem>
- <listitem>DRM_MODE_CONNECTOR_9PinDIN</listitem>
- <listitem>DRM_MODE_CONNECTOR_DisplayPort</listitem>
- <listitem>DRM_MODE_CONNECTOR_HDMIA</listitem>
- <listitem>DRM_MODE_CONNECTOR_HDMIB</listitem>
- <listitem>DRM_MODE_CONNECTOR_TV</listitem>
- <listitem>DRM_MODE_CONNECTOR_eDP</listitem>
- <listitem>DRM_MODE_CONNECTOR_VIRTUAL</listitem>
- </itemizedlist>
- </para>
- <para>
- Connectors must be attached to an encoder to be used. For devices that
- map connectors to encoders 1:1, the connector should be attached at
- initialization time with a call to
- <function>drm_mode_connector_attach_encoder</function>. The driver must
- also set the <structname>drm_connector</structname>
- <structfield>encoder</structfield> field to point to the attached
- encoder.
- </para>
- <para>
- Finally, drivers must initialize the connectors state change detection
- with a call to <function>drm_kms_helper_poll_init</function>. If at
- least one connector is pollable but can't generate hotplug interrupts
- (indicated by the DRM_CONNECTOR_POLL_CONNECT and
- DRM_CONNECTOR_POLL_DISCONNECT connector flags), a delayed work will
- automatically be queued to periodically poll for changes. Connectors
- that can generate hotplug interrupts must be marked with the
- DRM_CONNECTOR_POLL_HPD flag instead, and their interrupt handler must
- call <function>drm_helper_hpd_irq_event</function>. The function will
- queue a delayed work to check the state of all connectors, but no
- periodic polling will be done.
- </para>
- </sect3>
- <sect3>
- <title>Connector Operations</title>
- <note><para>
- Unless otherwise state, all operations are mandatory.
- </para></note>
- <sect4>
- <title>DPMS</title>
- <synopsis>void (*dpms)(struct drm_connector *connector, int mode);</synopsis>
- <para>
- The DPMS operation sets the power state of a connector. The mode
- argument is one of
- <itemizedlist>
- <listitem><para>DRM_MODE_DPMS_ON</para></listitem>
- <listitem><para>DRM_MODE_DPMS_STANDBY</para></listitem>
- <listitem><para>DRM_MODE_DPMS_SUSPEND</para></listitem>
- <listitem><para>DRM_MODE_DPMS_OFF</para></listitem>
- </itemizedlist>
- </para>
- <para>
- In all but DPMS_ON mode the encoder to which the connector is attached
- should put the display in low-power mode by driving its signals
- appropriately. If more than one connector is attached to the encoder
- care should be taken not to change the power state of other displays as
- a side effect. Low-power mode should be propagated to the encoders and
- CRTCs when all related connectors are put in low-power mode.
- </para>
- </sect4>
- <sect4>
- <title>Modes</title>
- <synopsis>int (*fill_modes)(struct drm_connector *connector, uint32_t max_width,
- uint32_t max_height);</synopsis>
- <para>
- Fill the mode list with all supported modes for the connector. If the
- <parameter>max_width</parameter> and <parameter>max_height</parameter>
- arguments are non-zero, the implementation must ignore all modes wider
- than <parameter>max_width</parameter> or higher than
- <parameter>max_height</parameter>.
- </para>
- <para>
- The connector must also fill in this operation its
- <structfield>display_info</structfield>
- <structfield>width_mm</structfield> and
- <structfield>height_mm</structfield> fields with the connected display
- physical size in millimeters. The fields should be set to 0 if the value
- isn't known or is not applicable (for instance for projector devices).
- </para>
- </sect4>
- <sect4>
- <title>Connection Status</title>
- <para>
- The connection status is updated through polling or hotplug events when
- supported (see <xref linkend="drm-kms-connector-polled"/>). The status
- value is reported to userspace through ioctls and must not be used
- inside the driver, as it only gets initialized by a call to
- <function>drm_mode_getconnector</function> from userspace.
- </para>
- <synopsis>enum drm_connector_status (*detect)(struct drm_connector *connector,
- bool force);</synopsis>
- <para>
- Check to see if anything is attached to the connector. The
- <parameter>force</parameter> parameter is set to false whilst polling or
- to true when checking the connector due to user request.
- <parameter>force</parameter> can be used by the driver to avoid
- expensive, destructive operations during automated probing.
- </para>
- <para>
- Return connector_status_connected if something is connected to the
- connector, connector_status_disconnected if nothing is connected and
- connector_status_unknown if the connection state isn't known.
- </para>
- <para>
- Drivers should only return connector_status_connected if the connection
- status has really been probed as connected. Connectors that can't detect
- the connection status, or failed connection status probes, should return
- connector_status_unknown.
- </para>
- </sect4>
- <sect4>
- <title>Miscellaneous</title>
- <itemizedlist>
- <listitem>
- <synopsis>void (*set_property)(struct drm_connector *connector,
- struct drm_property *property, uint64_t value);</synopsis>
- <para>
- Set the value of the given connector property to
- <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
- for more information about properties.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*destroy)(struct drm_connector *connector);</synopsis>
- <para>
- Destroy the connector when not needed anymore. See
- <xref linkend="drm-kms-init"/>.
- </para>
- </listitem>
- </itemizedlist>
- </sect4>
- </sect3>
- </sect2>
- <sect2>
- <title>Cleanup</title>
- <para>
- The DRM core manages its objects' lifetime. When an object is not needed
- anymore the core calls its destroy function, which must clean up and
- free every resource allocated for the object. Every
- <function>drm_*_init</function> call must be matched with a
- corresponding <function>drm_*_cleanup</function> call to cleanup CRTCs
- (<function>drm_crtc_cleanup</function>), planes
- (<function>drm_plane_cleanup</function>), encoders
- (<function>drm_encoder_cleanup</function>) and connectors
- (<function>drm_connector_cleanup</function>). Furthermore, connectors
- that have been added to sysfs must be removed by a call to
- <function>drm_connector_unregister</function> before calling
- <function>drm_connector_cleanup</function>.
- </para>
- <para>
- Connectors state change detection must be cleanup up with a call to
- <function>drm_kms_helper_poll_fini</function>.
- </para>
- </sect2>
- <sect2>
- <title>Output discovery and initialization example</title>
- <programlisting><![CDATA[
-void intel_crt_init(struct drm_device *dev)
-{
- struct drm_connector *connector;
- struct intel_output *intel_output;
-
- intel_output = kzalloc(sizeof(struct intel_output), GFP_KERNEL);
- if (!intel_output)
- return;
-
- connector = &intel_output->base;
- drm_connector_init(dev, &intel_output->base,
- &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA);
-
- drm_encoder_init(dev, &intel_output->enc, &intel_crt_enc_funcs,
- DRM_MODE_ENCODER_DAC);
-
- drm_mode_connector_attach_encoder(&intel_output->base,
- &intel_output->enc);
-
- /* Set up the DDC bus. */
- intel_output->ddc_bus = intel_i2c_create(dev, GPIOA, "CRTDDC_A");
- if (!intel_output->ddc_bus) {
- dev_printk(KERN_ERR, &dev->pdev->dev, "DDC bus registration "
- "failed.\n");
- return;
- }
-
- intel_output->type = INTEL_OUTPUT_ANALOG;
- connector->interlace_allowed = 0;
- connector->doublescan_allowed = 0;
-
- drm_encoder_helper_add(&intel_output->enc, &intel_crt_helper_funcs);
- drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
-
- drm_connector_register(connector);
-}]]></programlisting>
- <para>
- In the example above (taken from the i915 driver), a CRTC, connector and
- encoder combination is created. A device-specific i2c bus is also
- created for fetching EDID data and performing monitor detection. Once
- the process is complete, the new connector is registered with sysfs to
- make its properties available to applications.
- </para>
- </sect2>
- <sect2>
- <title>KMS API Functions</title>
-!Edrivers/gpu/drm/drm_crtc.c
- </sect2>
- <sect2>
- <title>KMS Data Structures</title>
-!Iinclude/drm/drm_crtc.h
- </sect2>
- <sect2>
- <title>KMS Locking</title>
-!Pdrivers/gpu/drm/drm_modeset_lock.c kms locking
-!Iinclude/drm/drm_modeset_lock.h
-!Edrivers/gpu/drm/drm_modeset_lock.c
- </sect2>
- </sect1>
-
- <!-- Internals: kms helper functions -->
-
- <sect1>
- <title>Mode Setting Helper Functions</title>
- <para>
- The plane, CRTC, encoder and connector functions provided by the drivers
- implement the DRM API. They're called by the DRM core and ioctl handlers
- to handle device state changes and configuration request. As implementing
- those functions often requires logic not specific to drivers, mid-layer
- helper functions are available to avoid duplicating boilerplate code.
- </para>
- <para>
- The DRM core contains one mid-layer implementation. The mid-layer provides
- implementations of several plane, CRTC, encoder and connector functions
- (called from the top of the mid-layer) that pre-process requests and call
- lower-level functions provided by the driver (at the bottom of the
- mid-layer). For instance, the
- <function>drm_crtc_helper_set_config</function> function can be used to
- fill the struct <structname>drm_crtc_funcs</structname>
- <structfield>set_config</structfield> field. When called, it will split
- the <methodname>set_config</methodname> operation in smaller, simpler
- operations and call the driver to handle them.
- </para>
- <para>
- To use the mid-layer, drivers call <function>drm_crtc_helper_add</function>,
- <function>drm_encoder_helper_add</function> and
- <function>drm_connector_helper_add</function> functions to install their
- mid-layer bottom operations handlers, and fill the
- <structname>drm_crtc_funcs</structname>,
- <structname>drm_encoder_funcs</structname> and
- <structname>drm_connector_funcs</structname> structures with pointers to
- the mid-layer top API functions. Installing the mid-layer bottom operation
- handlers is best done right after registering the corresponding KMS object.
- </para>
- <para>
- The mid-layer is not split between CRTC, encoder and connector operations.
- To use it, a driver must provide bottom functions for all of the three KMS
- entities.
- </para>
- <sect2>
- <title>Helper Functions</title>
- <itemizedlist>
- <listitem>
- <synopsis>int drm_crtc_helper_set_config(struct drm_mode_set *set);</synopsis>
- <para>
- The <function>drm_crtc_helper_set_config</function> helper function
- is a CRTC <methodname>set_config</methodname> implementation. It
- first tries to locate the best encoder for each connector by calling
- the connector <methodname>best_encoder</methodname> helper
- operation.
- </para>
- <para>
- After locating the appropriate encoders, the helper function will
- call the <methodname>mode_fixup</methodname> encoder and CRTC helper
- operations to adjust the requested mode, or reject it completely in
- which case an error will be returned to the application. If the new
- configuration after mode adjustment is identical to the current
- configuration the helper function will return without performing any
- other operation.
- </para>
- <para>
- If the adjusted mode is identical to the current mode but changes to
- the frame buffer need to be applied, the
- <function>drm_crtc_helper_set_config</function> function will call
- the CRTC <methodname>mode_set_base</methodname> helper operation. If
- the adjusted mode differs from the current mode, or if the
- <methodname>mode_set_base</methodname> helper operation is not
- provided, the helper function performs a full mode set sequence by
- calling the <methodname>prepare</methodname>,
- <methodname>mode_set</methodname> and
- <methodname>commit</methodname> CRTC and encoder helper operations,
- in that order.
- </para>
- </listitem>
- <listitem>
- <synopsis>void drm_helper_connector_dpms(struct drm_connector *connector, int mode);</synopsis>
- <para>
- The <function>drm_helper_connector_dpms</function> helper function
- is a connector <methodname>dpms</methodname> implementation that
- tracks power state of connectors. To use the function, drivers must
- provide <methodname>dpms</methodname> helper operations for CRTCs
- and encoders to apply the DPMS state to the device.
- </para>
- <para>
- The mid-layer doesn't track the power state of CRTCs and encoders.
- The <methodname>dpms</methodname> helper operations can thus be
- called with a mode identical to the currently active mode.
- </para>
- </listitem>
- <listitem>
- <synopsis>int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
- uint32_t maxX, uint32_t maxY);</synopsis>
- <para>
- The <function>drm_helper_probe_single_connector_modes</function> helper
- function is a connector <methodname>fill_modes</methodname>
- implementation that updates the connection status for the connector
- and then retrieves a list of modes by calling the connector
- <methodname>get_modes</methodname> helper operation.
- </para>
- <para>
- If the helper operation returns no mode, and if the connector status
- is connector_status_connected, standard VESA DMT modes up to
- 1024x768 are automatically added to the modes list by a call to
- <function>drm_add_modes_noedid</function>.
- </para>
- <para>
- The function then filters out modes larger than
- <parameter>max_width</parameter> and <parameter>max_height</parameter>
- if specified. It finally calls the optional connector
- <methodname>mode_valid</methodname> helper operation for each mode in
- the probed list to check whether the mode is valid for the connector.
- </para>
- </listitem>
- </itemizedlist>
- </sect2>
- <sect2>
- <title>CRTC Helper Operations</title>
- <itemizedlist>
- <listitem id="drm-helper-crtc-mode-fixup">
- <synopsis>bool (*mode_fixup)(struct drm_crtc *crtc,
- const struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode);</synopsis>
- <para>
- Let CRTCs adjust the requested mode or reject it completely. This
- operation returns true if the mode is accepted (possibly after being
- adjusted) or false if it is rejected.
- </para>
- <para>
- The <methodname>mode_fixup</methodname> operation should reject the
- mode if it can't reasonably use it. The definition of "reasonable"
- is currently fuzzy in this context. One possible behaviour would be
- to set the adjusted mode to the panel timings when a fixed-mode
- panel is used with hardware capable of scaling. Another behaviour
- would be to accept any input mode and adjust it to the closest mode
- supported by the hardware (FIXME: This needs to be clarified).
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*mode_set_base)(struct drm_crtc *crtc, int x, int y,
- struct drm_framebuffer *old_fb)</synopsis>
- <para>
- Move the CRTC on the current frame buffer (stored in
- <literal>crtc->fb</literal>) to position (x,y). Any of the frame
- buffer, x position or y position may have been modified.
- </para>
- <para>
- This helper operation is optional. If not provided, the
- <function>drm_crtc_helper_set_config</function> function will fall
- back to the <methodname>mode_set</methodname> helper operation.
- </para>
- <note><para>
- FIXME: Why are x and y passed as arguments, as they can be accessed
- through <literal>crtc->x</literal> and
- <literal>crtc->y</literal>?
- </para></note>
- </listitem>
- <listitem>
- <synopsis>void (*prepare)(struct drm_crtc *crtc);</synopsis>
- <para>
- Prepare the CRTC for mode setting. This operation is called after
- validating the requested mode. Drivers use it to perform
- device-specific operations required before setting the new mode.
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*mode_set)(struct drm_crtc *crtc, struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode, int x, int y,
- struct drm_framebuffer *old_fb);</synopsis>
- <para>
- Set a new mode, position and frame buffer. Depending on the device
- requirements, the mode can be stored internally by the driver and
- applied in the <methodname>commit</methodname> operation, or
- programmed to the hardware immediately.
- </para>
- <para>
- The <methodname>mode_set</methodname> operation returns 0 on success
- or a negative error code if an error occurs.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*commit)(struct drm_crtc *crtc);</synopsis>
- <para>
- Commit a mode. This operation is called after setting the new mode.
- Upon return the device must use the new mode and be fully
- operational.
- </para>
- </listitem>
- </itemizedlist>
- </sect2>
- <sect2>
- <title>Encoder Helper Operations</title>
- <itemizedlist>
- <listitem>
- <synopsis>bool (*mode_fixup)(struct drm_encoder *encoder,
- const struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode);</synopsis>
- <para>
- Let encoders adjust the requested mode or reject it completely. This
- operation returns true if the mode is accepted (possibly after being
- adjusted) or false if it is rejected. See the
- <link linkend="drm-helper-crtc-mode-fixup">mode_fixup CRTC helper
- operation</link> for an explanation of the allowed adjustments.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*prepare)(struct drm_encoder *encoder);</synopsis>
- <para>
- Prepare the encoder for mode setting. This operation is called after
- validating the requested mode. Drivers use it to perform
- device-specific operations required before setting the new mode.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*mode_set)(struct drm_encoder *encoder,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode);</synopsis>
- <para>
- Set a new mode. Depending on the device requirements, the mode can
- be stored internally by the driver and applied in the
- <methodname>commit</methodname> operation, or programmed to the
- hardware immediately.
- </para>
- </listitem>
- <listitem>
- <synopsis>void (*commit)(struct drm_encoder *encoder);</synopsis>
- <para>
- Commit a mode. This operation is called after setting the new mode.
- Upon return the device must use the new mode and be fully
- operational.
- </para>
- </listitem>
- </itemizedlist>
- </sect2>
- <sect2>
- <title>Connector Helper Operations</title>
- <itemizedlist>
- <listitem>
- <synopsis>struct drm_encoder *(*best_encoder)(struct drm_connector *connector);</synopsis>
- <para>
- Return a pointer to the best encoder for the connecter. Device that
- map connectors to encoders 1:1 simply return the pointer to the
- associated encoder. This operation is mandatory.
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*get_modes)(struct drm_connector *connector);</synopsis>
- <para>
- Fill the connector's <structfield>probed_modes</structfield> list
- by parsing EDID data with <function>drm_add_edid_modes</function>,
- adding standard VESA DMT modes with <function>drm_add_modes_noedid</function>,
- or calling <function>drm_mode_probed_add</function> directly for every
- supported mode and return the number of modes it has detected. This
- operation is mandatory.
- </para>
- <para>
- Note that the caller function will automatically add standard VESA
- DMT modes up to 1024x768 if the <methodname>get_modes</methodname>
- helper operation returns no mode and if the connector status is
- connector_status_connected. There is no need to call
- <function>drm_add_edid_modes</function> manually in that case.
- </para>
- <para>
- When adding modes manually the driver creates each mode with a call to
- <function>drm_mode_create</function> and must fill the following fields.
- <itemizedlist>
- <listitem>
- <synopsis>__u32 type;</synopsis>
- <para>
- Mode type bitmask, a combination of
- <variablelist>
- <varlistentry>
- <term>DRM_MODE_TYPE_BUILTIN</term>
- <listitem><para>not used?</para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_TYPE_CLOCK_C</term>
- <listitem><para>not used?</para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_TYPE_CRTC_C</term>
- <listitem><para>not used?</para></listitem>
- </varlistentry>
- <varlistentry>
- <term>
- DRM_MODE_TYPE_PREFERRED - The preferred mode for the connector
- </term>
- <listitem>
- <para>not used?</para>
- </listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_TYPE_DEFAULT</term>
- <listitem><para>not used?</para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_TYPE_USERDEF</term>
- <listitem><para>not used?</para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_TYPE_DRIVER</term>
- <listitem>
- <para>
- The mode has been created by the driver (as opposed to
- to user-created modes).
- </para>
- </listitem>
- </varlistentry>
- </variablelist>
- Drivers must set the DRM_MODE_TYPE_DRIVER bit for all modes they
- create, and set the DRM_MODE_TYPE_PREFERRED bit for the preferred
- mode.
- </para>
- </listitem>
- <listitem>
- <synopsis>__u32 clock;</synopsis>
- <para>Pixel clock frequency in kHz unit</para>
- </listitem>
- <listitem>
- <synopsis>__u16 hdisplay, hsync_start, hsync_end, htotal;
- __u16 vdisplay, vsync_start, vsync_end, vtotal;</synopsis>
- <para>Horizontal and vertical timing information</para>
- <screen><![CDATA[
- Active Front Sync Back
- Region Porch Porch
- <-----------------------><----------------><-------------><-------------->
-
- //////////////////////|
- ////////////////////// |
- ////////////////////// |.................. ................
- _______________
-
- <----- [hv]display ----->
- <------------- [hv]sync_start ------------>
- <--------------------- [hv]sync_end --------------------->
- <-------------------------------- [hv]total ----------------------------->
-]]></screen>
- </listitem>
- <listitem>
- <synopsis>__u16 hskew;
- __u16 vscan;</synopsis>
- <para>Unknown</para>
- </listitem>
- <listitem>
- <synopsis>__u32 flags;</synopsis>
- <para>
- Mode flags, a combination of
- <variablelist>
- <varlistentry>
- <term>DRM_MODE_FLAG_PHSYNC</term>
- <listitem><para>
- Horizontal sync is active high
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_NHSYNC</term>
- <listitem><para>
- Horizontal sync is active low
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_PVSYNC</term>
- <listitem><para>
- Vertical sync is active high
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_NVSYNC</term>
- <listitem><para>
- Vertical sync is active low
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_INTERLACE</term>
- <listitem><para>
- Mode is interlaced
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_DBLSCAN</term>
- <listitem><para>
- Mode uses doublescan
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_CSYNC</term>
- <listitem><para>
- Mode uses composite sync
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_PCSYNC</term>
- <listitem><para>
- Composite sync is active high
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_NCSYNC</term>
- <listitem><para>
- Composite sync is active low
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_HSKEW</term>
- <listitem><para>
- hskew provided (not used?)
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_BCAST</term>
- <listitem><para>
- not used?
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_PIXMUX</term>
- <listitem><para>
- not used?
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_DBLCLK</term>
- <listitem><para>
- not used?
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_FLAG_CLKDIV2</term>
- <listitem><para>
- ?
- </para></listitem>
- </varlistentry>
- </variablelist>
- </para>
- <para>
- Note that modes marked with the INTERLACE or DBLSCAN flags will be
- filtered out by
- <function>drm_helper_probe_single_connector_modes</function> if
- the connector's <structfield>interlace_allowed</structfield> or
- <structfield>doublescan_allowed</structfield> field is set to 0.
- </para>
- </listitem>
- <listitem>
- <synopsis>char name[DRM_DISPLAY_MODE_LEN];</synopsis>
- <para>
- Mode name. The driver must call
- <function>drm_mode_set_name</function> to fill the mode name from
- <structfield>hdisplay</structfield>,
- <structfield>vdisplay</structfield> and interlace flag after
- filling the corresponding fields.
- </para>
- </listitem>
- </itemizedlist>
- </para>
- <para>
- The <structfield>vrefresh</structfield> value is computed by
- <function>drm_helper_probe_single_connector_modes</function>.
- </para>
- <para>
- When parsing EDID data, <function>drm_add_edid_modes</function> fills the
- connector <structfield>display_info</structfield>
- <structfield>width_mm</structfield> and
- <structfield>height_mm</structfield> fields. When creating modes
- manually the <methodname>get_modes</methodname> helper operation must
- set the <structfield>display_info</structfield>
- <structfield>width_mm</structfield> and
- <structfield>height_mm</structfield> fields if they haven't been set
- already (for instance at initialization time when a fixed-size panel is
- attached to the connector). The mode <structfield>width_mm</structfield>
- and <structfield>height_mm</structfield> fields are only used internally
- during EDID parsing and should not be set when creating modes manually.
- </para>
- </listitem>
- <listitem>
- <synopsis>int (*mode_valid)(struct drm_connector *connector,
- struct drm_display_mode *mode);</synopsis>
- <para>
- Verify whether a mode is valid for the connector. Return MODE_OK for
- supported modes and one of the enum drm_mode_status values (MODE_*)
- for unsupported modes. This operation is optional.
- </para>
- <para>
- As the mode rejection reason is currently not used beside for
- immediately removing the unsupported mode, an implementation can
- return MODE_BAD regardless of the exact reason why the mode is not
- valid.
- </para>
- <note><para>
- Note that the <methodname>mode_valid</methodname> helper operation is
- only called for modes detected by the device, and
- <emphasis>not</emphasis> for modes set by the user through the CRTC
- <methodname>set_config</methodname> operation.
- </para></note>
- </listitem>
- </itemizedlist>
- </sect2>
- <sect2>
- <title>Atomic Modeset Helper Functions Reference</title>
- <sect3>
- <title>Overview</title>
-!Pdrivers/gpu/drm/drm_atomic_helper.c overview
- </sect3>
- <sect3>
- <title>Implementing Asynchronous Atomic Commit</title>
-!Pdrivers/gpu/drm/drm_atomic_helper.c implementing async commit
- </sect3>
- <sect3>
- <title>Atomic State Reset and Initialization</title>
-!Pdrivers/gpu/drm/drm_atomic_helper.c atomic state reset and initialization
- </sect3>
-!Iinclude/drm/drm_atomic_helper.h
-!Edrivers/gpu/drm/drm_atomic_helper.c
- </sect2>
- <sect2>
- <title>Modeset Helper Functions Reference</title>
-!Iinclude/drm/drm_crtc_helper.h
-!Edrivers/gpu/drm/drm_crtc_helper.c
-!Pdrivers/gpu/drm/drm_crtc_helper.c overview
- </sect2>
- <sect2>
- <title>Output Probing Helper Functions Reference</title>
-!Pdrivers/gpu/drm/drm_probe_helper.c output probing helper overview
-!Edrivers/gpu/drm/drm_probe_helper.c
- </sect2>
- <sect2>
- <title>fbdev Helper Functions Reference</title>
-!Pdrivers/gpu/drm/drm_fb_helper.c fbdev helpers
-!Edrivers/gpu/drm/drm_fb_helper.c
-!Iinclude/drm/drm_fb_helper.h
- </sect2>
- <sect2>
- <title>Display Port Helper Functions Reference</title>
-!Pdrivers/gpu/drm/drm_dp_helper.c dp helpers
-!Iinclude/drm/drm_dp_helper.h
-!Edrivers/gpu/drm/drm_dp_helper.c
- </sect2>
- <sect2>
- <title>Display Port MST Helper Functions Reference</title>
-!Pdrivers/gpu/drm/drm_dp_mst_topology.c dp mst helper
-!Iinclude/drm/drm_dp_mst_helper.h
-!Edrivers/gpu/drm/drm_dp_mst_topology.c
- </sect2>
- <sect2>
- <title>MIPI DSI Helper Functions Reference</title>
-!Pdrivers/gpu/drm/drm_mipi_dsi.c dsi helpers
-!Iinclude/drm/drm_mipi_dsi.h
-!Edrivers/gpu/drm/drm_mipi_dsi.c
- </sect2>
- <sect2>
- <title>EDID Helper Functions Reference</title>
-!Edrivers/gpu/drm/drm_edid.c
- </sect2>
- <sect2>
- <title>Rectangle Utilities Reference</title>
-!Pinclude/drm/drm_rect.h rect utils
-!Iinclude/drm/drm_rect.h
-!Edrivers/gpu/drm/drm_rect.c
- </sect2>
- <sect2>
- <title>Flip-work Helper Reference</title>
-!Pinclude/drm/drm_flip_work.h flip utils
-!Iinclude/drm/drm_flip_work.h
-!Edrivers/gpu/drm/drm_flip_work.c
- </sect2>
- <sect2>
- <title>HDMI Infoframes Helper Reference</title>
- <para>
- Strictly speaking this is not a DRM helper library but generally useable
- by any driver interfacing with HDMI outputs like v4l or alsa drivers.
- But it nicely fits into the overall topic of mode setting helper
- libraries and hence is also included here.
- </para>
-!Iinclude/linux/hdmi.h
-!Edrivers/video/hdmi.c
- </sect2>
- <sect2>
- <title id="drm-kms-planehelpers">Plane Helper Reference</title>
-!Edrivers/gpu/drm/drm_plane_helper.c
-!Pdrivers/gpu/drm/drm_plane_helper.c overview
- </sect2>
- <sect2>
- <title>Tile group</title>
-!Pdrivers/gpu/drm/drm_crtc.c Tile group
- </sect2>
- <sect2>
- <title>Bridges</title>
- <sect3>
- <title>Overview</title>
-!Pdrivers/gpu/drm/drm_bridge.c overview
- </sect3>
- <sect3>
- <title>Default bridge callback sequence</title>
-!Pdrivers/gpu/drm/drm_bridge.c bridge callbacks
- </sect3>
-!Edrivers/gpu/drm/drm_bridge.c
- </sect2>
- </sect1>
-
- <!-- Internals: kms properties -->
-
- <sect1 id="drm-kms-properties">
- <title>KMS Properties</title>
- <para>
- Drivers may need to expose additional parameters to applications than
- those described in the previous sections. KMS supports attaching
- properties to CRTCs, connectors and planes and offers a userspace API to
- list, get and set the property values.
- </para>
- <para>
- Properties are identified by a name that uniquely defines the property
- purpose, and store an associated value. For all property types except blob
- properties the value is a 64-bit unsigned integer.
- </para>
- <para>
- KMS differentiates between properties and property instances. Drivers
- first create properties and then create and associate individual instances
- of those properties to objects. A property can be instantiated multiple
- times and associated with different objects. Values are stored in property
- instances, and all other property information are stored in the property
- and shared between all instances of the property.
- </para>
- <para>
- Every property is created with a type that influences how the KMS core
- handles the property. Supported property types are
- <variablelist>
- <varlistentry>
- <term>DRM_MODE_PROP_RANGE</term>
- <listitem><para>Range properties report their minimum and maximum
- admissible values. The KMS core verifies that values set by
- application fit in that range.</para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_PROP_ENUM</term>
- <listitem><para>Enumerated properties take a numerical value that
- ranges from 0 to the number of enumerated values defined by the
- property minus one, and associate a free-formed string name to each
- value. Applications can retrieve the list of defined value-name pairs
- and use the numerical value to get and set property instance values.
- </para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_PROP_BITMASK</term>
- <listitem><para>Bitmask properties are enumeration properties that
- additionally restrict all enumerated values to the 0..63 range.
- Bitmask property instance values combine one or more of the
- enumerated bits defined by the property.</para></listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_MODE_PROP_BLOB</term>
- <listitem><para>Blob properties store a binary blob without any format
- restriction. The binary blobs are created as KMS standalone objects,
- and blob property instance values store the ID of their associated
- blob object.</para>
- <para>Blob properties are only used for the connector EDID property
- and cannot be created by drivers.</para></listitem>
- </varlistentry>
- </variablelist>
- </para>
- <para>
- To create a property drivers call one of the following functions depending
- on the property type. All property creation functions take property flags
- and name, as well as type-specific arguments.
- <itemizedlist>
- <listitem>
- <synopsis>struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
- const char *name,
- uint64_t min, uint64_t max);</synopsis>
- <para>Create a range property with the given minimum and maximum
- values.</para>
- </listitem>
- <listitem>
- <synopsis>struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
- const char *name,
- const struct drm_prop_enum_list *props,
- int num_values);</synopsis>
- <para>Create an enumerated property. The <parameter>props</parameter>
- argument points to an array of <parameter>num_values</parameter>
- value-name pairs.</para>
- </listitem>
- <listitem>
- <synopsis>struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
- int flags, const char *name,
- const struct drm_prop_enum_list *props,
- int num_values);</synopsis>
- <para>Create a bitmask property. The <parameter>props</parameter>
- argument points to an array of <parameter>num_values</parameter>
- value-name pairs.</para>
- </listitem>
- </itemizedlist>
- </para>
- <para>
- Properties can additionally be created as immutable, in which case they
- will be read-only for applications but can be modified by the driver. To
- create an immutable property drivers must set the DRM_MODE_PROP_IMMUTABLE
- flag at property creation time.
- </para>
- <para>
- When no array of value-name pairs is readily available at property
- creation time for enumerated or range properties, drivers can create
- the property using the <function>drm_property_create</function> function
- and manually add enumeration value-name pairs by calling the
- <function>drm_property_add_enum</function> function. Care must be taken to
- properly specify the property type through the <parameter>flags</parameter>
- argument.
- </para>
- <para>
- After creating properties drivers can attach property instances to CRTC,
- connector and plane objects by calling the
- <function>drm_object_attach_property</function>. The function takes a
- pointer to the target object, a pointer to the previously created property
- and an initial instance value.
- </para>
- <sect2>
- <title>Existing KMS Properties</title>
- <para>
- The following table gives description of drm properties exposed by various
- modules/drivers.
- </para>
- <table border="1" cellpadding="0" cellspacing="0">
- <tbody>
- <tr style="font-weight: bold;">
- <td valign="top" >Owner Module/Drivers</td>
- <td valign="top" >Group</td>
- <td valign="top" >Property Name</td>
- <td valign="top" >Type</td>
- <td valign="top" >Property Values</td>
- <td valign="top" >Object attached</td>
- <td valign="top" >Description/Restrictions</td>
- </tr>
- <tr>
- <td rowspan="37" valign="top" >DRM</td>
- <td valign="top" >Generic</td>
- <td valign="top" >“rotation”</td>
- <td valign="top" >BITMASK</td>
- <td valign="top" >{ 0, "rotate-0" },
- { 1, "rotate-90" },
- { 2, "rotate-180" },
- { 3, "rotate-270" },
- { 4, "reflect-x" },
- { 5, "reflect-y" }</td>
- <td valign="top" >CRTC, Plane</td>
- <td valign="top" >rotate-(degrees) rotates the image by the specified amount in degrees
- in counter clockwise direction. reflect-x and reflect-y reflects the
- image along the specified axis prior to rotation</td>
- </tr>
- <tr>
- <td rowspan="5" valign="top" >Connector</td>
- <td valign="top" >“EDID”</td>
- <td valign="top" >BLOB | IMMUTABLE</td>
- <td valign="top" >0</td>
- <td valign="top" >Connector</td>
- <td valign="top" >Contains id of edid blob ptr object.</td>
- </tr>
- <tr>
- <td valign="top" >“DPMS”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ “On”, “Standby”, “Suspend”, “Off” }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >Contains DPMS operation mode value.</td>
- </tr>
- <tr>
- <td valign="top" >“PATH”</td>
- <td valign="top" >BLOB | IMMUTABLE</td>
- <td valign="top" >0</td>
- <td valign="top" >Connector</td>
- <td valign="top" >Contains topology path to a connector.</td>
- </tr>
- <tr>
- <td valign="top" >“TILE”</td>
- <td valign="top" >BLOB | IMMUTABLE</td>
- <td valign="top" >0</td>
- <td valign="top" >Connector</td>
- <td valign="top" >Contains tiling information for a connector.</td>
- </tr>
- <tr>
- <td valign="top" >“CRTC_ID”</td>
- <td valign="top" >OBJECT</td>
- <td valign="top" >DRM_MODE_OBJECT_CRTC</td>
- <td valign="top" >Connector</td>
- <td valign="top" >CRTC that connector is attached to (atomic)</td>
- </tr>
- <tr>
- <td rowspan="11" valign="top" >Plane</td>
- <td valign="top" >“type”</td>
- <td valign="top" >ENUM | IMMUTABLE</td>
- <td valign="top" >{ "Overlay", "Primary", "Cursor" }</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Plane type</td>
- </tr>
- <tr>
- <td valign="top" >“SRC_X”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=UINT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout source x coordinate in 16.16 fixed point (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“SRC_Y”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=UINT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout source y coordinate in 16.16 fixed point (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“SRC_W”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=UINT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout source width in 16.16 fixed point (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“SRC_H”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=UINT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout source height in 16.16 fixed point (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“CRTC_X”</td>
- <td valign="top" >SIGNED_RANGE</td>
- <td valign="top" >Min=INT_MIN, Max=INT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout CRTC (destination) x coordinate (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“CRTC_Y”</td>
- <td valign="top" >SIGNED_RANGE</td>
- <td valign="top" >Min=INT_MIN, Max=INT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout CRTC (destination) y coordinate (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“CRTC_W”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=UINT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout CRTC (destination) width (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“CRTC_H”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=UINT_MAX</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout CRTC (destination) height (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“FB_ID”</td>
- <td valign="top" >OBJECT</td>
- <td valign="top" >DRM_MODE_OBJECT_FB</td>
- <td valign="top" >Plane</td>
- <td valign="top" >Scanout framebuffer (atomic)</td>
- </tr>
- <tr>
- <td valign="top" >“CRTC_ID”</td>
- <td valign="top" >OBJECT</td>
- <td valign="top" >DRM_MODE_OBJECT_CRTC</td>
- <td valign="top" >Plane</td>
- <td valign="top" >CRTC that plane is attached to (atomic)</td>
- </tr>
- <tr>
- <td rowspan="2" valign="top" >DVI-I</td>
- <td valign="top" >“subconnector”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ “Unknown”, “DVI-D”, “DVI-A” }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“select subconnector”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ “Automatic”, “DVI-D”, “DVI-A” }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="13" valign="top" >TV</td>
- <td valign="top" >“subconnector”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "Unknown", "Composite", "SVIDEO", "Component", "SCART" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“select subconnector”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "Automatic", "Composite", "SVIDEO", "Component", "SCART" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“mode”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "NTSC_M", "NTSC_J", "NTSC_443", "PAL_B" } etc.</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“left margin”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“right margin”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“top margin”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“bottom margin”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“brightness”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“contrast”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“flicker reduction”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“overscan”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“saturation”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“hue”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="2" valign="top" >Virtual GPU</td>
- <td valign="top" >“suggested X”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0xffffffff</td>
- <td valign="top" >Connector</td>
- <td valign="top" >property to suggest an X offset for a connector</td>
- </tr>
- <tr>
- <td valign="top" >“suggested Y”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0xffffffff</td>
- <td valign="top" >Connector</td>
- <td valign="top" >property to suggest an Y offset for a connector</td>
- </tr>
- <tr>
- <td rowspan="3" valign="top" >Optional</td>
- <td valign="top" >“scaling mode”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "None", "Full", "Center", "Full aspect" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"aspect ratio"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "None", "4:3", "16:9" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >DRM property to set aspect ratio from user space app.
- This enum is made generic to allow addition of custom aspect
- ratios.</td>
- </tr>
- <tr>
- <td valign="top" >“dirty”</td>
- <td valign="top" >ENUM | IMMUTABLE</td>
- <td valign="top" >{ "Off", "On", "Annotate" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="20" valign="top" >i915</td>
- <td rowspan="2" valign="top" >Generic</td>
- <td valign="top" >"Broadcast RGB"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "Automatic", "Full", "Limited 16:235" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“audio”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "force-dvi", "off", "auto", "on" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="17" valign="top" >SDVO-TV</td>
- <td valign="top" >“mode”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "NTSC_M", "NTSC_J", "NTSC_443", "PAL_B" } etc.</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"left_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"right_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"top_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"bottom_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“hpos”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“vpos”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“contrast”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“saturation”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“hue”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“sharpness”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“flicker_filter”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“flicker_filter_adaptive”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“flicker_filter_2d”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“tv_chroma_filter”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“tv_luma_filter”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“dot_crawl”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >SDVO-TV/LVDS</td>
- <td valign="top" >“brightness”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="2" valign="top" >CDV gma-500</td>
- <td rowspan="2" valign="top" >Generic</td>
- <td valign="top" >"Broadcast RGB"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ “Full”, “Limited 16:235” }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"Broadcast RGB"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ “off”, “auto”, “on” }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="19" valign="top" >Poulsbo</td>
- <td rowspan="1" valign="top" >Generic</td>
- <td valign="top" >“backlight”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=100</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="17" valign="top" >SDVO-TV</td>
- <td valign="top" >“mode”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "NTSC_M", "NTSC_J", "NTSC_443", "PAL_B" } etc.</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"left_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"right_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"top_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"bottom_margin"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“hpos”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“vpos”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“contrast”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“saturation”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“hue”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“sharpness”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“flicker_filter”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“flicker_filter_adaptive”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“flicker_filter_2d”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“tv_chroma_filter”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“tv_luma_filter”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“dot_crawl”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >SDVO-TV/LVDS</td>
- <td valign="top" >“brightness”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max= SDVO dependent</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="11" valign="top" >armada</td>
- <td rowspan="2" valign="top" >CRTC</td>
- <td valign="top" >"CSC_YUV"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "Auto" , "CCIR601", "CCIR709" }</td>
- <td valign="top" >CRTC</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"CSC_RGB"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "Auto", "Computer system", "Studio" }</td>
- <td valign="top" >CRTC</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="9" valign="top" >Overlay</td>
- <td valign="top" >"colorkey"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0xffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"colorkey_min"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0xffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"colorkey_max"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0xffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"colorkey_val"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0xffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"colorkey_alpha"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0xffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"colorkey_mode"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "disabled", "Y component", "U component"
- , "V component", "RGB", “R component", "G component", "B component" }</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"brightness"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=256 + 255</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"contrast"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0x7fff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"saturation"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0x7fff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="2" valign="top" >exynos</td>
- <td valign="top" >CRTC</td>
- <td valign="top" >“mode”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "normal", "blank" }</td>
- <td valign="top" >CRTC</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >Overlay</td>
- <td valign="top" >“zpos”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=MAX_PLANE-1</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="2" valign="top" >i2c/ch7006_drv</td>
- <td valign="top" >Generic</td>
- <td valign="top" >“scale”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=2</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="1" valign="top" >TV</td>
- <td valign="top" >“mode”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "PAL", "PAL-M","PAL-N"}, ”PAL-Nc"
- , "PAL-60", "NTSC-M", "NTSC-J" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="15" valign="top" >nouveau</td>
- <td rowspan="6" valign="top" >NV10 Overlay</td>
- <td valign="top" >"colorkey"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0x01ffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“contrast”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=8192-1</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“brightness”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1024</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“hue”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=359</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“saturation”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=8192-1</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“iturbt_709”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="2" valign="top" >Nv04 Overlay</td>
- <td valign="top" >“colorkey”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0x01ffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“brightness”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1024</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="7" valign="top" >Display</td>
- <td valign="top" >“dithering mode”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "auto", "off", "on" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“dithering depth”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "auto", "off", "on", "static 2x2", "dynamic 2x2", "temporal" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“underscan”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "auto", "6 bpc", "8 bpc" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“underscan hborder”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=128</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“underscan vborder”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=128</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“vibrant hue”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=180</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >“color vibrance”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=200</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >omap</td>
- <td valign="top" >Generic</td>
- <td valign="top" >“zorder”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=3</td>
- <td valign="top" >CRTC, Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >qxl</td>
- <td valign="top" >Generic</td>
- <td valign="top" >“hotplug_mode_update"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="9" valign="top" >radeon</td>
- <td valign="top" >DVI-I</td>
- <td valign="top" >“coherent”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >DAC enable load detect</td>
- <td valign="top" >“load detection”</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=1</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >TV Standard</td>
- <td valign="top" >"tv standard"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "ntsc", "pal", "pal-m", "pal-60", "ntsc-j"
- , "scart-pal", "pal-cn", "secam" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >legacy TMDS PLL detect</td>
- <td valign="top" >"tmds_pll"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "driver", "bios" }</td>
- <td valign="top" >-</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="3" valign="top" >Underscan</td>
- <td valign="top" >"underscan"</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "off", "on", "auto" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"underscan hborder"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=128</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"underscan vborder"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=128</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >Audio</td>
- <td valign="top" >“audio”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "off", "on", "auto" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >FMT Dithering</td>
- <td valign="top" >“dither”</td>
- <td valign="top" >ENUM</td>
- <td valign="top" >{ "off", "on" }</td>
- <td valign="top" >Connector</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td rowspan="3" valign="top" >rcar-du</td>
- <td rowspan="3" valign="top" >Generic</td>
- <td valign="top" >"alpha"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=255</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"colorkey"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=0, Max=0x01ffffff</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- <tr>
- <td valign="top" >"zpos"</td>
- <td valign="top" >RANGE</td>
- <td valign="top" >Min=1, Max=7</td>
- <td valign="top" >Plane</td>
- <td valign="top" >TBD</td>
- </tr>
- </tbody>
- </table>
- </sect2>
- </sect1>
-
- <!-- Internals: vertical blanking -->
-
- <sect1 id="drm-vertical-blank">
- <title>Vertical Blanking</title>
- <para>
- Vertical blanking plays a major role in graphics rendering. To achieve
- tear-free display, users must synchronize page flips and/or rendering to
- vertical blanking. The DRM API offers ioctls to perform page flips
- synchronized to vertical blanking and wait for vertical blanking.
- </para>
- <para>
- The DRM core handles most of the vertical blanking management logic, which
- involves filtering out spurious interrupts, keeping race-free blanking
- counters, coping with counter wrap-around and resets and keeping use
- counts. It relies on the driver to generate vertical blanking interrupts
- and optionally provide a hardware vertical blanking counter. Drivers must
- implement the following operations.
- </para>
- <itemizedlist>
- <listitem>
- <synopsis>int (*enable_vblank) (struct drm_device *dev, int crtc);
-void (*disable_vblank) (struct drm_device *dev, int crtc);</synopsis>
- <para>
- Enable or disable vertical blanking interrupts for the given CRTC.
- </para>
- </listitem>
- <listitem>
- <synopsis>u32 (*get_vblank_counter) (struct drm_device *dev, int crtc);</synopsis>
- <para>
- Retrieve the value of the vertical blanking counter for the given
- CRTC. If the hardware maintains a vertical blanking counter its value
- should be returned. Otherwise drivers can use the
- <function>drm_vblank_count</function> helper function to handle this
- operation.
- </para>
- </listitem>
- </itemizedlist>
- <para>
- Drivers must initialize the vertical blanking handling core with a call to
- <function>drm_vblank_init</function> in their
- <methodname>load</methodname> operation. The function will set the struct
- <structname>drm_device</structname>
- <structfield>vblank_disable_allowed</structfield> field to 0. This will
- keep vertical blanking interrupts enabled permanently until the first mode
- set operation, where <structfield>vblank_disable_allowed</structfield> is
- set to 1. The reason behind this is not clear. Drivers can set the field
- to 1 after <function>calling drm_vblank_init</function> to make vertical
- blanking interrupts dynamically managed from the beginning.
- </para>
- <para>
- Vertical blanking interrupts can be enabled by the DRM core or by drivers
- themselves (for instance to handle page flipping operations). The DRM core
- maintains a vertical blanking use count to ensure that the interrupts are
- not disabled while a user still needs them. To increment the use count,
- drivers call <function>drm_vblank_get</function>. Upon return vertical
- blanking interrupts are guaranteed to be enabled.
- </para>
- <para>
- To decrement the use count drivers call
- <function>drm_vblank_put</function>. Only when the use count drops to zero
- will the DRM core disable the vertical blanking interrupts after a delay
- by scheduling a timer. The delay is accessible through the vblankoffdelay
- module parameter or the <varname>drm_vblank_offdelay</varname> global
- variable and expressed in milliseconds. Its default value is 5000 ms.
- Zero means never disable, and a negative value means disable immediately.
- Drivers may override the behaviour by setting the
- <structname>drm_device</structname>
- <structfield>vblank_disable_immediate</structfield> flag, which when set
- causes vblank interrupts to be disabled immediately regardless of the
- drm_vblank_offdelay value. The flag should only be set if there's a
- properly working hardware vblank counter present.
- </para>
- <para>
- When a vertical blanking interrupt occurs drivers only need to call the
- <function>drm_handle_vblank</function> function to account for the
- interrupt.
- </para>
- <para>
- Resources allocated by <function>drm_vblank_init</function> must be freed
- with a call to <function>drm_vblank_cleanup</function> in the driver
- <methodname>unload</methodname> operation handler.
- </para>
- <sect2>
- <title>Vertical Blanking and Interrupt Handling Functions Reference</title>
-!Edrivers/gpu/drm/drm_irq.c
-!Finclude/drm/drmP.h drm_crtc_vblank_waitqueue
- </sect2>
- </sect1>
-
- <!-- Internals: open/close, file operations and ioctls -->
-
- <sect1>
- <title>Open/Close, File Operations and IOCTLs</title>
- <sect2>
- <title>Open and Close</title>
- <synopsis>int (*firstopen) (struct drm_device *);
-void (*lastclose) (struct drm_device *);
-int (*open) (struct drm_device *, struct drm_file *);
-void (*preclose) (struct drm_device *, struct drm_file *);
-void (*postclose) (struct drm_device *, struct drm_file *);</synopsis>
- <abstract>Open and close handlers. None of those methods are mandatory.
- </abstract>
- <para>
- The <methodname>firstopen</methodname> method is called by the DRM core
- for legacy UMS (User Mode Setting) drivers only when an application
- opens a device that has no other opened file handle. UMS drivers can
- implement it to acquire device resources. KMS drivers can't use the
- method and must acquire resources in the <methodname>load</methodname>
- method instead.
- </para>
- <para>
- Similarly the <methodname>lastclose</methodname> method is called when
- the last application holding a file handle opened on the device closes
- it, for both UMS and KMS drivers. Additionally, the method is also
- called at module unload time or, for hot-pluggable devices, when the
- device is unplugged. The <methodname>firstopen</methodname> and
- <methodname>lastclose</methodname> calls can thus be unbalanced.
- </para>
- <para>
- The <methodname>open</methodname> method is called every time the device
- is opened by an application. Drivers can allocate per-file private data
- in this method and store them in the struct
- <structname>drm_file</structname> <structfield>driver_priv</structfield>
- field. Note that the <methodname>open</methodname> method is called
- before <methodname>firstopen</methodname>.
- </para>
- <para>
- The close operation is split into <methodname>preclose</methodname> and
- <methodname>postclose</methodname> methods. Drivers must stop and
- cleanup all per-file operations in the <methodname>preclose</methodname>
- method. For instance pending vertical blanking and page flip events must
- be cancelled. No per-file operation is allowed on the file handle after
- returning from the <methodname>preclose</methodname> method.
- </para>
- <para>
- Finally the <methodname>postclose</methodname> method is called as the
- last step of the close operation, right before calling the
- <methodname>lastclose</methodname> method if no other open file handle
- exists for the device. Drivers that have allocated per-file private data
- in the <methodname>open</methodname> method should free it here.
- </para>
- <para>
- The <methodname>lastclose</methodname> method should restore CRTC and
- plane properties to default value, so that a subsequent open of the
- device will not inherit state from the previous user. It can also be
- used to execute delayed power switching state changes, e.g. in
- conjunction with the vga-switcheroo infrastructure. Beyond that KMS
- drivers should not do any further cleanup. Only legacy UMS drivers might
- need to clean up device state so that the vga console or an independent
- fbdev driver could take over.
- </para>
- </sect2>
- <sect2>
- <title>File Operations</title>
- <synopsis>const struct file_operations *fops</synopsis>
- <abstract>File operations for the DRM device node.</abstract>
- <para>
- Drivers must define the file operations structure that forms the DRM
- userspace API entry point, even though most of those operations are
- implemented in the DRM core. The <methodname>open</methodname>,
- <methodname>release</methodname> and <methodname>ioctl</methodname>
- operations are handled by
- <programlisting>
- .owner = THIS_MODULE,
- .open = drm_open,
- .release = drm_release,
- .unlocked_ioctl = drm_ioctl,
- #ifdef CONFIG_COMPAT
- .compat_ioctl = drm_compat_ioctl,
- #endif
- </programlisting>
- </para>
- <para>
- Drivers that implement private ioctls that requires 32/64bit
- compatibility support must provide their own
- <methodname>compat_ioctl</methodname> handler that processes private
- ioctls and calls <function>drm_compat_ioctl</function> for core ioctls.
- </para>
- <para>
- The <methodname>read</methodname> and <methodname>poll</methodname>
- operations provide support for reading DRM events and polling them. They
- are implemented by
- <programlisting>
- .poll = drm_poll,
- .read = drm_read,
- .llseek = no_llseek,
- </programlisting>
- </para>
- <para>
- The memory mapping implementation varies depending on how the driver
- manages memory. Pre-GEM drivers will use <function>drm_mmap</function>,
- while GEM-aware drivers will use <function>drm_gem_mmap</function>. See
- <xref linkend="drm-gem"/>.
- <programlisting>
- .mmap = drm_gem_mmap,
- </programlisting>
- </para>
- <para>
- No other file operation is supported by the DRM API.
- </para>
- </sect2>
- <sect2>
- <title>IOCTLs</title>
- <synopsis>struct drm_ioctl_desc *ioctls;
-int num_ioctls;</synopsis>
- <abstract>Driver-specific ioctls descriptors table.</abstract>
- <para>
- Driver-specific ioctls numbers start at DRM_COMMAND_BASE. The ioctls
- descriptors table is indexed by the ioctl number offset from the base
- value. Drivers can use the DRM_IOCTL_DEF_DRV() macro to initialize the
- table entries.
- </para>
- <para>
- <programlisting>DRM_IOCTL_DEF_DRV(ioctl, func, flags)</programlisting>
- <para>
- <parameter>ioctl</parameter> is the ioctl name. Drivers must define
- the DRM_##ioctl and DRM_IOCTL_##ioctl macros to the ioctl number
- offset from DRM_COMMAND_BASE and the ioctl number respectively. The
- first macro is private to the device while the second must be exposed
- to userspace in a public header.
- </para>
- <para>
- <parameter>func</parameter> is a pointer to the ioctl handler function
- compatible with the <type>drm_ioctl_t</type> type.
- <programlisting>typedef int drm_ioctl_t(struct drm_device *dev, void *data,
- struct drm_file *file_priv);</programlisting>
- </para>
- <para>
- <parameter>flags</parameter> is a bitmask combination of the following
- values. It restricts how the ioctl is allowed to be called.
- <itemizedlist>
- <listitem><para>
- DRM_AUTH - Only authenticated callers allowed
- </para></listitem>
- <listitem><para>
- DRM_MASTER - The ioctl can only be called on the master file
- handle
- </para></listitem>
- <listitem><para>
- DRM_ROOT_ONLY - Only callers with the SYSADMIN capability allowed
- </para></listitem>
- <listitem><para>
- DRM_CONTROL_ALLOW - The ioctl can only be called on a control
- device
- </para></listitem>
- <listitem><para>
- DRM_UNLOCKED - The ioctl handler will be called without locking
- the DRM global mutex
- </para></listitem>
- </itemizedlist>
- </para>
- </para>
- </sect2>
- </sect1>
- <sect1>
- <title>Legacy Support Code</title>
- <para>
- The section very briefly covers some of the old legacy support code which
- is only used by old DRM drivers which have done a so-called shadow-attach
- to the underlying device instead of registering as a real driver. This
- also includes some of the old generic buffer management and command
- submission code. Do not use any of this in new and modern drivers.
- </para>
-
- <sect2>
- <title>Legacy Suspend/Resume</title>
- <para>
- The DRM core provides some suspend/resume code, but drivers wanting full
- suspend/resume support should provide save() and restore() functions.
- These are called at suspend, hibernate, or resume time, and should perform
- any state save or restore required by your device across suspend or
- hibernate states.
- </para>
- <synopsis>int (*suspend) (struct drm_device *, pm_message_t state);
- int (*resume) (struct drm_device *);</synopsis>
- <para>
- Those are legacy suspend and resume methods which
- <emphasis>only</emphasis> work with the legacy shadow-attach driver
- registration functions. New driver should use the power management
- interface provided by their bus type (usually through
- the struct <structname>device_driver</structname> dev_pm_ops) and set
- these methods to NULL.
- </para>
- </sect2>
-
- <sect2>
- <title>Legacy DMA Services</title>
- <para>
- This should cover how DMA mapping etc. is supported by the core.
- These functions are deprecated and should not be used.
- </para>
- </sect2>
- </sect1>
- </chapter>
-
-<!-- TODO
-
-- Add a glossary
-- Document the struct_mutex catch-all lock
-- Document connector properties
-
-- Why is the load method optional?
-- What are drivers supposed to set the initial display state to, and how?
- Connector's DPMS states are not initialized and are thus equal to
- DRM_MODE_DPMS_ON. The fbcon compatibility layer calls
- drm_helper_disable_unused_functions(), which disables unused encoders and
- CRTCs, but doesn't touch the connectors' DPMS state, and
- drm_helper_connector_dpms() in reaction to fbdev blanking events. Do drivers
- that don't implement (or just don't use) fbcon compatibility need to call
- those functions themselves?
-- KMS drivers must call drm_vblank_pre_modeset() and drm_vblank_post_modeset()
- around mode setting. Should this be done in the DRM core?
-- vblank_disable_allowed is set to 1 in the first drm_vblank_post_modeset()
- call and never set back to 0. It seems to be safe to permanently set it to 1
- in drm_vblank_init() for KMS driver, and it might be safe for UMS drivers as
- well. This should be investigated.
-- crtc and connector .save and .restore operations are only used internally in
- drivers, should they be removed from the core?
-- encoder mid-layer .save and .restore operations are only used internally in
- drivers, should they be removed from the core?
-- encoder mid-layer .detect operation is only used internally in drivers,
- should it be removed from the core?
--->
-
- <!-- External interfaces -->
-
- <chapter id="drmExternals">
- <title>Userland interfaces</title>
- <para>
- The DRM core exports several interfaces to applications,
- generally intended to be used through corresponding libdrm
- wrapper functions. In addition, drivers export device-specific
- interfaces for use by userspace drivers & device-aware
- applications through ioctls and sysfs files.
- </para>
- <para>
- External interfaces include: memory mapping, context management,
- DMA operations, AGP management, vblank control, fence
- management, memory management, and output management.
- </para>
- <para>
- Cover generic ioctls and sysfs layout here. We only need high-level
- info, since man pages should cover the rest.
- </para>
-
- <!-- External: render nodes -->
-
- <sect1>
- <title>Render nodes</title>
- <para>
- DRM core provides multiple character-devices for user-space to use.
- Depending on which device is opened, user-space can perform a different
- set of operations (mainly ioctls). The primary node is always created
- and called card<num>. Additionally, a currently
- unused control node, called controlD<num> is also
- created. The primary node provides all legacy operations and
- historically was the only interface used by userspace. With KMS, the
- control node was introduced. However, the planned KMS control interface
- has never been written and so the control node stays unused to date.
- </para>
- <para>
- With the increased use of offscreen renderers and GPGPU applications,
- clients no longer require running compositors or graphics servers to
- make use of a GPU. But the DRM API required unprivileged clients to
- authenticate to a DRM-Master prior to getting GPU access. To avoid this
- step and to grant clients GPU access without authenticating, render
- nodes were introduced. Render nodes solely serve render clients, that
- is, no modesetting or privileged ioctls can be issued on render nodes.
- Only non-global rendering commands are allowed. If a driver supports
- render nodes, it must advertise it via the DRIVER_RENDER
- DRM driver capability. If not supported, the primary node must be used
- for render clients together with the legacy drmAuth authentication
- procedure.
- </para>
- <para>
- If a driver advertises render node support, DRM core will create a
- separate render node called renderD<num>. There will
- be one render node per device. No ioctls except PRIME-related ioctls
- will be allowed on this node. Especially GEM_OPEN will be
- explicitly prohibited. Render nodes are designed to avoid the
- buffer-leaks, which occur if clients guess the flink names or mmap
- offsets on the legacy interface. Additionally to this basic interface,
- drivers must mark their driver-dependent render-only ioctls as
- DRM_RENDER_ALLOW so render clients can use them. Driver
- authors must be careful not to allow any privileged ioctls on render
- nodes.
- </para>
- <para>
- With render nodes, user-space can now control access to the render node
- via basic file-system access-modes. A running graphics server which
- authenticates clients on the privileged primary/legacy node is no longer
- required. Instead, a client can open the render node and is immediately
- granted GPU access. Communication between clients (or servers) is done
- via PRIME. FLINK from render node to legacy node is not supported. New
- clients must not use the insecure FLINK interface.
- </para>
- <para>
- Besides dropping all modeset/global ioctls, render nodes also drop the
- DRM-Master concept. There is no reason to associate render clients with
- a DRM-Master as they are independent of any graphics server. Besides,
- they must work without any running master, anyway.
- Drivers must be able to run without a master object if they support
- render nodes. If, on the other hand, a driver requires shared state
- between clients which is visible to user-space and accessible beyond
- open-file boundaries, they cannot support render nodes.
- </para>
- </sect1>
-
- <!-- External: vblank handling -->
-
- <sect1>
- <title>VBlank event handling</title>
- <para>
- The DRM core exposes two vertical blank related ioctls:
- <variablelist>
- <varlistentry>
- <term>DRM_IOCTL_WAIT_VBLANK</term>
- <listitem>
- <para>
- This takes a struct drm_wait_vblank structure as its argument,
- and it is used to block or request a signal when a specified
- vblank event occurs.
- </para>
- </listitem>
- </varlistentry>
- <varlistentry>
- <term>DRM_IOCTL_MODESET_CTL</term>
- <listitem>
- <para>
- This was only used for user-mode-settind drivers around
- modesetting changes to allow the kernel to update the vblank
- interrupt after mode setting, since on many devices the vertical
- blank counter is reset to 0 at some point during modeset. Modern
- drivers should not call this any more since with kernel mode
- setting it is a no-op.
- </para>
- </listitem>
- </varlistentry>
- </variablelist>
- </para>
- </sect1>
-
- </chapter>
-</part>
-<part id="drmDrivers">
- <title>DRM Drivers</title>
-
- <partintro>
- <para>
- This second part of the DRM Developer's Guide documents driver code,
- implementation details and also all the driver-specific userspace
- interfaces. Especially since all hardware-acceleration interfaces to
- userspace are driver specific for efficiency and other reasons these
- interfaces can be rather substantial. Hence every driver has its own
- chapter.
- </para>
- </partintro>
-
- <chapter id="drmI915">
- <title>drm/i915 Intel GFX Driver</title>
- <para>
- The drm/i915 driver supports all (with the exception of some very early
- models) integrated GFX chipsets with both Intel display and rendering
- blocks. This excludes a set of SoC platforms with an SGX rendering unit,
- those have basic support through the gma500 drm driver.
- </para>
- <sect1>
- <title>Core Driver Infrastructure</title>
- <para>
- This section covers core driver infrastructure used by both the display
- and the GEM parts of the driver.
- </para>
- <sect2>
- <title>Runtime Power Management</title>
-!Pdrivers/gpu/drm/i915/intel_runtime_pm.c runtime pm
-!Idrivers/gpu/drm/i915/intel_runtime_pm.c
-!Idrivers/gpu/drm/i915/intel_uncore.c
- </sect2>
- <sect2>
- <title>Interrupt Handling</title>
-!Pdrivers/gpu/drm/i915/i915_irq.c interrupt handling
-!Fdrivers/gpu/drm/i915/i915_irq.c intel_irq_init intel_irq_init_hw intel_hpd_init
-!Fdrivers/gpu/drm/i915/i915_irq.c intel_runtime_pm_disable_interrupts
-!Fdrivers/gpu/drm/i915/i915_irq.c intel_runtime_pm_enable_interrupts
- </sect2>
- <sect2>
- <title>Intel GVT-g Guest Support(vGPU)</title>
-!Pdrivers/gpu/drm/i915/i915_vgpu.c Intel GVT-g guest support
-!Idrivers/gpu/drm/i915/i915_vgpu.c
- </sect2>
- </sect1>
- <sect1>
- <title>Display Hardware Handling</title>
- <para>
- This section covers everything related to the display hardware including
- the mode setting infrastructure, plane, sprite and cursor handling and
- display, output probing and related topics.
- </para>
- <sect2>
- <title>Mode Setting Infrastructure</title>
- <para>
- The i915 driver is thus far the only DRM driver which doesn't use the
- common DRM helper code to implement mode setting sequences. Thus it
- has its own tailor-made infrastructure for executing a display
- configuration change.
- </para>
- </sect2>
- <sect2>
- <title>Frontbuffer Tracking</title>
-!Pdrivers/gpu/drm/i915/intel_frontbuffer.c frontbuffer tracking
-!Idrivers/gpu/drm/i915/intel_frontbuffer.c
-!Fdrivers/gpu/drm/i915/i915_gem.c i915_gem_track_fb
- </sect2>
- <sect2>
- <title>Display FIFO Underrun Reporting</title>
-!Pdrivers/gpu/drm/i915/intel_fifo_underrun.c fifo underrun handling
-!Idrivers/gpu/drm/i915/intel_fifo_underrun.c
- </sect2>
- <sect2>
- <title>Plane Configuration</title>
- <para>
- This section covers plane configuration and composition with the
- primary plane, sprites, cursors and overlays. This includes the
- infrastructure to do atomic vsync'ed updates of all this state and
- also tightly coupled topics like watermark setup and computation,
- framebuffer compression and panel self refresh.
- </para>
- </sect2>
- <sect2>
- <title>Atomic Plane Helpers</title>
-!Pdrivers/gpu/drm/i915/intel_atomic_plane.c atomic plane helpers
-!Idrivers/gpu/drm/i915/intel_atomic_plane.c
- </sect2>
- <sect2>
- <title>Output Probing</title>
- <para>
- This section covers output probing and related infrastructure like the
- hotplug interrupt storm detection and mitigation code. Note that the
- i915 driver still uses most of the common DRM helper code for output
- probing, so those sections fully apply.
- </para>
- </sect2>
- <sect2>
- <title>Hotplug</title>
-!Pdrivers/gpu/drm/i915/intel_hotplug.c Hotplug
-!Idrivers/gpu/drm/i915/intel_hotplug.c
- </sect2>
- <sect2>
- <title>High Definition Audio</title>
-!Pdrivers/gpu/drm/i915/intel_audio.c High Definition Audio over HDMI and Display Port
-!Idrivers/gpu/drm/i915/intel_audio.c
- </sect2>
- <sect2>
- <title>Panel Self Refresh PSR (PSR/SRD)</title>
-!Pdrivers/gpu/drm/i915/intel_psr.c Panel Self Refresh (PSR/SRD)
-!Idrivers/gpu/drm/i915/intel_psr.c
- </sect2>
- <sect2>
- <title>Frame Buffer Compression (FBC)</title>
-!Pdrivers/gpu/drm/i915/intel_fbc.c Frame Buffer Compression (FBC)
-!Idrivers/gpu/drm/i915/intel_fbc.c
- </sect2>
- <sect2>
- <title>Display Refresh Rate Switching (DRRS)</title>
-!Pdrivers/gpu/drm/i915/intel_dp.c Display Refresh Rate Switching (DRRS)
-!Fdrivers/gpu/drm/i915/intel_dp.c intel_dp_set_drrs_state
-!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_enable
-!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_disable
-!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_invalidate
-!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_flush
-!Fdrivers/gpu/drm/i915/intel_dp.c intel_dp_drrs_init
-
- </sect2>
- <sect2>
- <title>DPIO</title>
-!Pdrivers/gpu/drm/i915/i915_reg.h DPIO
- <table id="dpiox2">
- <title>Dual channel PHY (VLV/CHV/BXT)</title>
- <tgroup cols="8">
- <colspec colname="c0" />
- <colspec colname="c1" />
- <colspec colname="c2" />
- <colspec colname="c3" />
- <colspec colname="c4" />
- <colspec colname="c5" />
- <colspec colname="c6" />
- <colspec colname="c7" />
- <spanspec spanname="ch0" namest="c0" nameend="c3" />
- <spanspec spanname="ch1" namest="c4" nameend="c7" />
- <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
- <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
- <spanspec spanname="ch1pcs01" namest="c4" nameend="c5" />
- <spanspec spanname="ch1pcs23" namest="c6" nameend="c7" />
- <thead>
- <row>
- <entry spanname="ch0">CH0</entry>
- <entry spanname="ch1">CH1</entry>
- </row>
- </thead>
- <tbody valign="top" align="center">
- <row>
- <entry spanname="ch0">CMN/PLL/REF</entry>
- <entry spanname="ch1">CMN/PLL/REF</entry>
- </row>
- <row>
- <entry spanname="ch0pcs01">PCS01</entry>
- <entry spanname="ch0pcs23">PCS23</entry>
- <entry spanname="ch1pcs01">PCS01</entry>
- <entry spanname="ch1pcs23">PCS23</entry>
- </row>
- <row>
- <entry>TX0</entry>
- <entry>TX1</entry>
- <entry>TX2</entry>
- <entry>TX3</entry>
- <entry>TX0</entry>
- <entry>TX1</entry>
- <entry>TX2</entry>
- <entry>TX3</entry>
- </row>
- <row>
- <entry spanname="ch0">DDI0</entry>
- <entry spanname="ch1">DDI1</entry>
- </row>
- </tbody>
- </tgroup>
- </table>
- <table id="dpiox1">
- <title>Single channel PHY (CHV/BXT)</title>
- <tgroup cols="4">
- <colspec colname="c0" />
- <colspec colname="c1" />
- <colspec colname="c2" />
- <colspec colname="c3" />
- <spanspec spanname="ch0" namest="c0" nameend="c3" />
- <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
- <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
- <thead>
- <row>
- <entry spanname="ch0">CH0</entry>
- </row>
- </thead>
- <tbody valign="top" align="center">
- <row>
- <entry spanname="ch0">CMN/PLL/REF</entry>
- </row>
- <row>
- <entry spanname="ch0pcs01">PCS01</entry>
- <entry spanname="ch0pcs23">PCS23</entry>
- </row>
- <row>
- <entry>TX0</entry>
- <entry>TX1</entry>
- <entry>TX2</entry>
- <entry>TX3</entry>
- </row>
- <row>
- <entry spanname="ch0">DDI2</entry>
- </row>
- </tbody>
- </tgroup>
- </table>
- </sect2>
-
- <sect2>
- <title>CSR firmware support for DMC</title>
-!Pdrivers/gpu/drm/i915/intel_csr.c csr support for dmc
-!Idrivers/gpu/drm/i915/intel_csr.c
- </sect2>
- </sect1>
-
- <sect1>
- <title>Memory Management and Command Submission</title>
- <para>
- This sections covers all things related to the GEM implementation in the
- i915 driver.
- </para>
- <sect2>
- <title>Batchbuffer Parsing</title>
-!Pdrivers/gpu/drm/i915/i915_cmd_parser.c batch buffer command parser
-!Idrivers/gpu/drm/i915/i915_cmd_parser.c
- </sect2>
- <sect2>
- <title>Batchbuffer Pools</title>
-!Pdrivers/gpu/drm/i915/i915_gem_batch_pool.c batch pool
-!Idrivers/gpu/drm/i915/i915_gem_batch_pool.c
- </sect2>
- <sect2>
- <title>Logical Rings, Logical Ring Contexts and Execlists</title>
-!Pdrivers/gpu/drm/i915/intel_lrc.c Logical Rings, Logical Ring Contexts and Execlists
-!Idrivers/gpu/drm/i915/intel_lrc.c
- </sect2>
- <sect2>
- <title>Global GTT views</title>
-!Pdrivers/gpu/drm/i915/i915_gem_gtt.c Global GTT views
-!Idrivers/gpu/drm/i915/i915_gem_gtt.c
- </sect2>
- <sect2>
- <title>GTT Fences and Swizzling</title>
-!Idrivers/gpu/drm/i915/i915_gem_fence.c
- <sect3>
- <title>Global GTT Fence Handling</title>
-!Pdrivers/gpu/drm/i915/i915_gem_fence.c fence register handling
- </sect3>
- <sect3>
- <title>Hardware Tiling and Swizzling Details</title>
-!Pdrivers/gpu/drm/i915/i915_gem_fence.c tiling swizzling details
- </sect3>
- </sect2>
- <sect2>
- <title>Object Tiling IOCTLs</title>
-!Idrivers/gpu/drm/i915/i915_gem_tiling.c
-!Pdrivers/gpu/drm/i915/i915_gem_tiling.c buffer object tiling
- </sect2>
- <sect2>
- <title>Buffer Object Eviction</title>
- <para>
- This section documents the interface functions for evicting buffer
- objects to make space available in the virtual gpu address spaces.
- Note that this is mostly orthogonal to shrinking buffer objects
- caches, which has the goal to make main memory (shared with the gpu
- through the unified memory architecture) available.
- </para>
-!Idrivers/gpu/drm/i915/i915_gem_evict.c
- </sect2>
- <sect2>
- <title>Buffer Object Memory Shrinking</title>
- <para>
- This section documents the interface function for shrinking memory
- usage of buffer object caches. Shrinking is used to make main memory
- available. Note that this is mostly orthogonal to evicting buffer
- objects, which has the goal to make space in gpu virtual address
- spaces.
- </para>
-!Idrivers/gpu/drm/i915/i915_gem_shrinker.c
- </sect2>
- </sect1>
- <sect1>
- <title> Tracing </title>
- <para>
- This sections covers all things related to the tracepoints implemented in
- the i915 driver.
- </para>
- <sect2>
- <title> i915_ppgtt_create and i915_ppgtt_release </title>
-!Pdrivers/gpu/drm/i915/i915_trace.h i915_ppgtt_create and i915_ppgtt_release tracepoints
- </sect2>
- <sect2>
- <title> i915_context_create and i915_context_free </title>
-!Pdrivers/gpu/drm/i915/i915_trace.h i915_context_create and i915_context_free tracepoints
- </sect2>
- <sect2>
- <title> switch_mm </title>
-!Pdrivers/gpu/drm/i915/i915_trace.h switch_mm tracepoint
- </sect2>
- </sect1>
-
- </chapter>
-!Cdrivers/gpu/drm/i915/i915_irq.c
-</part>
-</book>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
+ "http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd" []>
+
+<book id="gpuDevelopersGuide">
+ <bookinfo>
+ <title>Linux GPU Driver Developer's Guide</title>
+
+ <authorgroup>
+ <author>
+ <firstname>Jesse</firstname>
+ <surname>Barnes</surname>
+ <contrib>Initial version</contrib>
+ <affiliation>
+ <orgname>Intel Corporation</orgname>
+ <address>
+ <email>jesse.barnes@intel.com</email>
+ </address>
+ </affiliation>
+ </author>
+ <author>
+ <firstname>Laurent</firstname>
+ <surname>Pinchart</surname>
+ <contrib>Driver internals</contrib>
+ <affiliation>
+ <orgname>Ideas on board SPRL</orgname>
+ <address>
+ <email>laurent.pinchart@ideasonboard.com</email>
+ </address>
+ </affiliation>
+ </author>
+ <author>
+ <firstname>Daniel</firstname>
+ <surname>Vetter</surname>
+ <contrib>Contributions all over the place</contrib>
+ <affiliation>
+ <orgname>Intel Corporation</orgname>
+ <address>
+ <email>daniel.vetter@ffwll.ch</email>
+ </address>
+ </affiliation>
+ </author>
+ <author>
+ <firstname>Lukas</firstname>
+ <surname>Wunner</surname>
+ <contrib>vga_switcheroo documentation</contrib>
+ <affiliation>
+ <address>
+ <email>lukas@wunner.de</email>
+ </address>
+ </affiliation>
+ </author>
+ </authorgroup>
+
+ <copyright>
+ <year>2008-2009</year>
+ <year>2013-2014</year>
+ <holder>Intel Corporation</holder>
+ </copyright>
+ <copyright>
+ <year>2012</year>
+ <holder>Laurent Pinchart</holder>
+ </copyright>
+ <copyright>
+ <year>2015</year>
+ <holder>Lukas Wunner</holder>
+ </copyright>
+
+ <legalnotice>
+ <para>
+ The contents of this file may be used under the terms of the GNU
+ General Public License version 2 (the "GPL") as distributed in
+ the kernel source COPYING file.
+ </para>
+ </legalnotice>
+
+ <revhistory>
+ <!-- Put document revisions here, newest first. -->
+ <revision>
+ <revnumber>1.0</revnumber>
+ <date>2012-07-13</date>
+ <authorinitials>LP</authorinitials>
+ <revremark>Added extensive documentation about driver internals.
+ </revremark>
+ </revision>
+ <revision>
+ <revnumber>1.1</revnumber>
+ <date>2015-10-11</date>
+ <authorinitials>LW</authorinitials>
+ <revremark>Added vga_switcheroo documentation.
+ </revremark>
+ </revision>
+ </revhistory>
+ </bookinfo>
+
+<toc></toc>
+
+<part id="drmCore">
+ <title>DRM Core</title>
+ <partintro>
+ <para>
+ This first part of the GPU Driver Developer's Guide documents core DRM
+ code, helper libraries for writing drivers and generic userspace
+ interfaces exposed by DRM drivers.
+ </para>
+ </partintro>
+
+ <chapter id="drmIntroduction">
+ <title>Introduction</title>
+ <para>
+ The Linux DRM layer contains code intended to support the needs
+ of complex graphics devices, usually containing programmable
+ pipelines well suited to 3D graphics acceleration. Graphics
+ drivers in the kernel may make use of DRM functions to make
+ tasks like memory management, interrupt handling and DMA easier,
+ and provide a uniform interface to applications.
+ </para>
+ <para>
+ A note on versions: this guide covers features found in the DRM
+ tree, including the TTM memory manager, output configuration and
+ mode setting, and the new vblank internals, in addition to all
+ the regular features found in current kernels.
+ </para>
+ <para>
+ [Insert diagram of typical DRM stack here]
+ </para>
+ </chapter>
+
+ <!-- Internals -->
+
+ <chapter id="drmInternals">
+ <title>DRM Internals</title>
+ <para>
+ This chapter documents DRM internals relevant to driver authors
+ and developers working to add support for the latest features to
+ existing drivers.
+ </para>
+ <para>
+ First, we go over some typical driver initialization
+ requirements, like setting up command buffers, creating an
+ initial output configuration, and initializing core services.
+ Subsequent sections cover core internals in more detail,
+ providing implementation notes and examples.
+ </para>
+ <para>
+ The DRM layer provides several services to graphics drivers,
+ many of them driven by the application interfaces it provides
+ through libdrm, the library that wraps most of the DRM ioctls.
+ These include vblank event handling, memory
+ management, output management, framebuffer management, command
+ submission & fencing, suspend/resume support, and DMA
+ services.
+ </para>
+
+ <!-- Internals: driver init -->
+
+ <sect1>
+ <title>Driver Initialization</title>
+ <para>
+ At the core of every DRM driver is a <structname>drm_driver</structname>
+ structure. Drivers typically statically initialize a drm_driver structure,
+ and then pass it to <function>drm_dev_alloc()</function> to allocate a
+ device instance. After the device instance is fully initialized it can be
+ registered (which makes it accessible from userspace) using
+ <function>drm_dev_register()</function>.
+ </para>
+ <para>
+ The <structname>drm_driver</structname> structure contains static
+ information that describes the driver and features it supports, and
+ pointers to methods that the DRM core will call to implement the DRM API.
+ We will first go through the <structname>drm_driver</structname> static
+ information fields, and will then describe individual operations in
+ details as they get used in later sections.
+ </para>
+ <sect2>
+ <title>Driver Information</title>
+ <sect3>
+ <title>Driver Features</title>
+ <para>
+ Drivers inform the DRM core about their requirements and supported
+ features by setting appropriate flags in the
+ <structfield>driver_features</structfield> field. Since those flags
+ influence the DRM core behaviour since registration time, most of them
+ must be set to registering the <structname>drm_driver</structname>
+ instance.
+ </para>
+ <synopsis>u32 driver_features;</synopsis>
+ <variablelist>
+ <title>Driver Feature Flags</title>
+ <varlistentry>
+ <term>DRIVER_USE_AGP</term>
+ <listitem><para>
+ Driver uses AGP interface, the DRM core will manage AGP resources.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_REQUIRE_AGP</term>
+ <listitem><para>
+ Driver needs AGP interface to function. AGP initialization failure
+ will become a fatal error.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_PCI_DMA</term>
+ <listitem><para>
+ Driver is capable of PCI DMA, mapping of PCI DMA buffers to
+ userspace will be enabled. Deprecated.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_SG</term>
+ <listitem><para>
+ Driver can perform scatter/gather DMA, allocation and mapping of
+ scatter/gather buffers will be enabled. Deprecated.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_HAVE_DMA</term>
+ <listitem><para>
+ Driver supports DMA, the userspace DMA API will be supported.
+ Deprecated.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_HAVE_IRQ</term><term>DRIVER_IRQ_SHARED</term>
+ <listitem><para>
+ DRIVER_HAVE_IRQ indicates whether the driver has an IRQ handler
+ managed by the DRM Core. The core will support simple IRQ handler
+ installation when the flag is set. The installation process is
+ described in <xref linkend="drm-irq-registration"/>.</para>
+ <para>DRIVER_IRQ_SHARED indicates whether the device & handler
+ support shared IRQs (note that this is required of PCI drivers).
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_GEM</term>
+ <listitem><para>
+ Driver use the GEM memory manager.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_MODESET</term>
+ <listitem><para>
+ Driver supports mode setting interfaces (KMS).
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_PRIME</term>
+ <listitem><para>
+ Driver implements DRM PRIME buffer sharing.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_RENDER</term>
+ <listitem><para>
+ Driver supports dedicated render nodes.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRIVER_ATOMIC</term>
+ <listitem><para>
+ Driver supports atomic properties. In this case the driver
+ must implement appropriate obj->atomic_get_property() vfuncs
+ for any modeset objects with driver specific properties.
+ </para></listitem>
+ </varlistentry>
+ </variablelist>
+ </sect3>
+ <sect3>
+ <title>Major, Minor and Patchlevel</title>
+ <synopsis>int major;
+int minor;
+int patchlevel;</synopsis>
+ <para>
+ The DRM core identifies driver versions by a major, minor and patch
+ level triplet. The information is printed to the kernel log at
+ initialization time and passed to userspace through the
+ DRM_IOCTL_VERSION ioctl.
+ </para>
+ <para>
+ The major and minor numbers are also used to verify the requested driver
+ API version passed to DRM_IOCTL_SET_VERSION. When the driver API changes
+ between minor versions, applications can call DRM_IOCTL_SET_VERSION to
+ select a specific version of the API. If the requested major isn't equal
+ to the driver major, or the requested minor is larger than the driver
+ minor, the DRM_IOCTL_SET_VERSION call will return an error. Otherwise
+ the driver's set_version() method will be called with the requested
+ version.
+ </para>
+ </sect3>
+ <sect3>
+ <title>Name, Description and Date</title>
+ <synopsis>char *name;
+char *desc;
+char *date;</synopsis>
+ <para>
+ The driver name is printed to the kernel log at initialization time,
+ used for IRQ registration and passed to userspace through
+ DRM_IOCTL_VERSION.
+ </para>
+ <para>
+ The driver description is a purely informative string passed to
+ userspace through the DRM_IOCTL_VERSION ioctl and otherwise unused by
+ the kernel.
+ </para>
+ <para>
+ The driver date, formatted as YYYYMMDD, is meant to identify the date of
+ the latest modification to the driver. However, as most drivers fail to
+ update it, its value is mostly useless. The DRM core prints it to the
+ kernel log at initialization time and passes it to userspace through the
+ DRM_IOCTL_VERSION ioctl.
+ </para>
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>Device Instance and Driver Handling</title>
+!Pdrivers/gpu/drm/drm_drv.c driver instance overview
+!Edrivers/gpu/drm/drm_drv.c
+ </sect2>
+ <sect2>
+ <title>Driver Load</title>
+ <sect3 id="drm-irq-registration">
+ <title>IRQ Registration</title>
+ <para>
+ The DRM core tries to facilitate IRQ handler registration and
+ unregistration by providing <function>drm_irq_install</function> and
+ <function>drm_irq_uninstall</function> functions. Those functions only
+ support a single interrupt per device, devices that use more than one
+ IRQs need to be handled manually.
+ </para>
+ <sect4>
+ <title>Managed IRQ Registration</title>
+ <para>
+ <function>drm_irq_install</function> starts by calling the
+ <methodname>irq_preinstall</methodname> driver operation. The operation
+ is optional and must make sure that the interrupt will not get fired by
+ clearing all pending interrupt flags or disabling the interrupt.
+ </para>
+ <para>
+ The passed-in IRQ will then be requested by a call to
+ <function>request_irq</function>. If the DRIVER_IRQ_SHARED driver
+ feature flag is set, a shared (IRQF_SHARED) IRQ handler will be
+ requested.
+ </para>
+ <para>
+ The IRQ handler function must be provided as the mandatory irq_handler
+ driver operation. It will get passed directly to
+ <function>request_irq</function> and thus has the same prototype as all
+ IRQ handlers. It will get called with a pointer to the DRM device as the
+ second argument.
+ </para>
+ <para>
+ Finally the function calls the optional
+ <methodname>irq_postinstall</methodname> driver operation. The operation
+ usually enables interrupts (excluding the vblank interrupt, which is
+ enabled separately), but drivers may choose to enable/disable interrupts
+ at a different time.
+ </para>
+ <para>
+ <function>drm_irq_uninstall</function> is similarly used to uninstall an
+ IRQ handler. It starts by waking up all processes waiting on a vblank
+ interrupt to make sure they don't hang, and then calls the optional
+ <methodname>irq_uninstall</methodname> driver operation. The operation
+ must disable all hardware interrupts. Finally the function frees the IRQ
+ by calling <function>free_irq</function>.
+ </para>
+ </sect4>
+ <sect4>
+ <title>Manual IRQ Registration</title>
+ <para>
+ Drivers that require multiple interrupt handlers can't use the managed
+ IRQ registration functions. In that case IRQs must be registered and
+ unregistered manually (usually with the <function>request_irq</function>
+ and <function>free_irq</function> functions, or their devm_* equivalent).
+ </para>
+ <para>
+ When manually registering IRQs, drivers must not set the DRIVER_HAVE_IRQ
+ driver feature flag, and must not provide the
+ <methodname>irq_handler</methodname> driver operation. They must set the
+ <structname>drm_device</structname> <structfield>irq_enabled</structfield>
+ field to 1 upon registration of the IRQs, and clear it to 0 after
+ unregistering the IRQs.
+ </para>
+ </sect4>
+ </sect3>
+ <sect3>
+ <title>Memory Manager Initialization</title>
+ <para>
+ Every DRM driver requires a memory manager which must be initialized at
+ load time. DRM currently contains two memory managers, the Translation
+ Table Manager (TTM) and the Graphics Execution Manager (GEM).
+ This document describes the use of the GEM memory manager only. See
+ <xref linkend="drm-memory-management"/> for details.
+ </para>
+ </sect3>
+ <sect3>
+ <title>Miscellaneous Device Configuration</title>
+ <para>
+ Another task that may be necessary for PCI devices during configuration
+ is mapping the video BIOS. On many devices, the VBIOS describes device
+ configuration, LCD panel timings (if any), and contains flags indicating
+ device state. Mapping the BIOS can be done using the pci_map_rom() call,
+ a convenience function that takes care of mapping the actual ROM,
+ whether it has been shadowed into memory (typically at address 0xc0000)
+ or exists on the PCI device in the ROM BAR. Note that after the ROM has
+ been mapped and any necessary information has been extracted, it should
+ be unmapped; on many devices, the ROM address decoder is shared with
+ other BARs, so leaving it mapped could cause undesired behaviour like
+ hangs or memory corruption.
+ <!--!Fdrivers/pci/rom.c pci_map_rom-->
+ </para>
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>Bus-specific Device Registration and PCI Support</title>
+ <para>
+ A number of functions are provided to help with device registration.
+ The functions deal with PCI and platform devices respectively and are
+ only provided for historical reasons. These are all deprecated and
+ shouldn't be used in new drivers. Besides that there's a few
+ helpers for pci drivers.
+ </para>
+!Edrivers/gpu/drm/drm_pci.c
+!Edrivers/gpu/drm/drm_platform.c
+ </sect2>
+ </sect1>
+
+ <!-- Internals: memory management -->
+
+ <sect1 id="drm-memory-management">
+ <title>Memory management</title>
+ <para>
+ Modern Linux systems require large amount of graphics memory to store
+ frame buffers, textures, vertices and other graphics-related data. Given
+ the very dynamic nature of many of that data, managing graphics memory
+ efficiently is thus crucial for the graphics stack and plays a central
+ role in the DRM infrastructure.
+ </para>
+ <para>
+ The DRM core includes two memory managers, namely Translation Table Maps
+ (TTM) and Graphics Execution Manager (GEM). TTM was the first DRM memory
+ manager to be developed and tried to be a one-size-fits-them all
+ solution. It provides a single userspace API to accommodate the need of
+ all hardware, supporting both Unified Memory Architecture (UMA) devices
+ and devices with dedicated video RAM (i.e. most discrete video cards).
+ This resulted in a large, complex piece of code that turned out to be
+ hard to use for driver development.
+ </para>
+ <para>
+ GEM started as an Intel-sponsored project in reaction to TTM's
+ complexity. Its design philosophy is completely different: instead of
+ providing a solution to every graphics memory-related problems, GEM
+ identified common code between drivers and created a support library to
+ share it. GEM has simpler initialization and execution requirements than
+ TTM, but has no video RAM management capabilities and is thus limited to
+ UMA devices.
+ </para>
+ <sect2>
+ <title>The Translation Table Manager (TTM)</title>
+ <para>
+ TTM design background and information belongs here.
+ </para>
+ <sect3>
+ <title>TTM initialization</title>
+ <warning><para>This section is outdated.</para></warning>
+ <para>
+ Drivers wishing to support TTM must fill out a drm_bo_driver
+ structure. The structure contains several fields with function
+ pointers for initializing the TTM, allocating and freeing memory,
+ waiting for command completion and fence synchronization, and memory
+ migration. See the radeon_ttm.c file for an example of usage.
+ </para>
+ <para>
+ The ttm_global_reference structure is made up of several fields:
+ </para>
+ <programlisting>
+ struct ttm_global_reference {
+ enum ttm_global_types global_type;
+ size_t size;
+ void *object;
+ int (*init) (struct ttm_global_reference *);
+ void (*release) (struct ttm_global_reference *);
+ };
+ </programlisting>
+ <para>
+ There should be one global reference structure for your memory
+ manager as a whole, and there will be others for each object
+ created by the memory manager at runtime. Your global TTM should
+ have a type of TTM_GLOBAL_TTM_MEM. The size field for the global
+ object should be sizeof(struct ttm_mem_global), and the init and
+ release hooks should point at your driver-specific init and
+ release routines, which probably eventually call
+ ttm_mem_global_init and ttm_mem_global_release, respectively.
+ </para>
+ <para>
+ Once your global TTM accounting structure is set up and initialized
+ by calling ttm_global_item_ref() on it,
+ you need to create a buffer object TTM to
+ provide a pool for buffer object allocation by clients and the
+ kernel itself. The type of this object should be TTM_GLOBAL_TTM_BO,
+ and its size should be sizeof(struct ttm_bo_global). Again,
+ driver-specific init and release functions may be provided,
+ likely eventually calling ttm_bo_global_init() and
+ ttm_bo_global_release(), respectively. Also, like the previous
+ object, ttm_global_item_ref() is used to create an initial reference
+ count for the TTM, which will call your initialization function.
+ </para>
+ </sect3>
+ </sect2>
+ <sect2 id="drm-gem">
+ <title>The Graphics Execution Manager (GEM)</title>
+ <para>
+ The GEM design approach has resulted in a memory manager that doesn't
+ provide full coverage of all (or even all common) use cases in its
+ userspace or kernel API. GEM exposes a set of standard memory-related
+ operations to userspace and a set of helper functions to drivers, and let
+ drivers implement hardware-specific operations with their own private API.
+ </para>
+ <para>
+ The GEM userspace API is described in the
+ <ulink url="http://lwn.net/Articles/283798/"><citetitle>GEM - the Graphics
+ Execution Manager</citetitle></ulink> article on LWN. While slightly
+ outdated, the document provides a good overview of the GEM API principles.
+ Buffer allocation and read and write operations, described as part of the
+ common GEM API, are currently implemented using driver-specific ioctls.
+ </para>
+ <para>
+ GEM is data-agnostic. It manages abstract buffer objects without knowing
+ what individual buffers contain. APIs that require knowledge of buffer
+ contents or purpose, such as buffer allocation or synchronization
+ primitives, are thus outside of the scope of GEM and must be implemented
+ using driver-specific ioctls.
+ </para>
+ <para>
+ On a fundamental level, GEM involves several operations:
+ <itemizedlist>
+ <listitem>Memory allocation and freeing</listitem>
+ <listitem>Command execution</listitem>
+ <listitem>Aperture management at command execution time</listitem>
+ </itemizedlist>
+ Buffer object allocation is relatively straightforward and largely
+ provided by Linux's shmem layer, which provides memory to back each
+ object.
+ </para>
+ <para>
+ Device-specific operations, such as command execution, pinning, buffer
+ read & write, mapping, and domain ownership transfers are left to
+ driver-specific ioctls.
+ </para>
+ <sect3>
+ <title>GEM Initialization</title>
+ <para>
+ Drivers that use GEM must set the DRIVER_GEM bit in the struct
+ <structname>drm_driver</structname>
+ <structfield>driver_features</structfield> field. The DRM core will
+ then automatically initialize the GEM core before calling the
+ <methodname>load</methodname> operation. Behind the scene, this will
+ create a DRM Memory Manager object which provides an address space
+ pool for object allocation.
+ </para>
+ <para>
+ In a KMS configuration, drivers need to allocate and initialize a
+ command ring buffer following core GEM initialization if required by
+ the hardware. UMA devices usually have what is called a "stolen"
+ memory region, which provides space for the initial framebuffer and
+ large, contiguous memory regions required by the device. This space is
+ typically not managed by GEM, and must be initialized separately into
+ its own DRM MM object.
+ </para>
+ </sect3>
+ <sect3>
+ <title>GEM Objects Creation</title>
+ <para>
+ GEM splits creation of GEM objects and allocation of the memory that
+ backs them in two distinct operations.
+ </para>
+ <para>
+ GEM objects are represented by an instance of struct
+ <structname>drm_gem_object</structname>. Drivers usually need to extend
+ GEM objects with private information and thus create a driver-specific
+ GEM object structure type that embeds an instance of struct
+ <structname>drm_gem_object</structname>.
+ </para>
+ <para>
+ To create a GEM object, a driver allocates memory for an instance of its
+ specific GEM object type and initializes the embedded struct
+ <structname>drm_gem_object</structname> with a call to
+ <function>drm_gem_object_init</function>. The function takes a pointer to
+ the DRM device, a pointer to the GEM object and the buffer object size
+ in bytes.
+ </para>
+ <para>
+ GEM uses shmem to allocate anonymous pageable memory.
+ <function>drm_gem_object_init</function> will create an shmfs file of
+ the requested size and store it into the struct
+ <structname>drm_gem_object</structname> <structfield>filp</structfield>
+ field. The memory is used as either main storage for the object when the
+ graphics hardware uses system memory directly or as a backing store
+ otherwise.
+ </para>
+ <para>
+ Drivers are responsible for the actual physical pages allocation by
+ calling <function>shmem_read_mapping_page_gfp</function> for each page.
+ Note that they can decide to allocate pages when initializing the GEM
+ object, or to delay allocation until the memory is needed (for instance
+ when a page fault occurs as a result of a userspace memory access or
+ when the driver needs to start a DMA transfer involving the memory).
+ </para>
+ <para>
+ Anonymous pageable memory allocation is not always desired, for instance
+ when the hardware requires physically contiguous system memory as is
+ often the case in embedded devices. Drivers can create GEM objects with
+ no shmfs backing (called private GEM objects) by initializing them with
+ a call to <function>drm_gem_private_object_init</function> instead of
+ <function>drm_gem_object_init</function>. Storage for private GEM
+ objects must be managed by drivers.
+ </para>
+ <para>
+ Drivers that do not need to extend GEM objects with private information
+ can call the <function>drm_gem_object_alloc</function> function to
+ allocate and initialize a struct <structname>drm_gem_object</structname>
+ instance. The GEM core will call the optional driver
+ <methodname>gem_init_object</methodname> operation after initializing
+ the GEM object with <function>drm_gem_object_init</function>.
+ <synopsis>int (*gem_init_object) (struct drm_gem_object *obj);</synopsis>
+ </para>
+ <para>
+ No alloc-and-init function exists for private GEM objects.
+ </para>
+ </sect3>
+ <sect3>
+ <title>GEM Objects Lifetime</title>
+ <para>
+ All GEM objects are reference-counted by the GEM core. References can be
+ acquired and release by <function>calling drm_gem_object_reference</function>
+ and <function>drm_gem_object_unreference</function> respectively. The
+ caller must hold the <structname>drm_device</structname>
+ <structfield>struct_mutex</structfield> lock. As a convenience, GEM
+ provides the <function>drm_gem_object_reference_unlocked</function> and
+ <function>drm_gem_object_unreference_unlocked</function> functions that
+ can be called without holding the lock.
+ </para>
+ <para>
+ When the last reference to a GEM object is released the GEM core calls
+ the <structname>drm_driver</structname>
+ <methodname>gem_free_object</methodname> operation. That operation is
+ mandatory for GEM-enabled drivers and must free the GEM object and all
+ associated resources.
+ </para>
+ <para>
+ <synopsis>void (*gem_free_object) (struct drm_gem_object *obj);</synopsis>
+ Drivers are responsible for freeing all GEM object resources, including
+ the resources created by the GEM core. If an mmap offset has been
+ created for the object (in which case
+ <structname>drm_gem_object</structname>::<structfield>map_list</structfield>::<structfield>map</structfield>
+ is not NULL) it must be freed by a call to
+ <function>drm_gem_free_mmap_offset</function>. The shmfs backing store
+ must be released by calling <function>drm_gem_object_release</function>
+ (that function can safely be called if no shmfs backing store has been
+ created).
+ </para>
+ </sect3>
+ <sect3>
+ <title>GEM Objects Naming</title>
+ <para>
+ Communication between userspace and the kernel refers to GEM objects
+ using local handles, global names or, more recently, file descriptors.
+ All of those are 32-bit integer values; the usual Linux kernel limits
+ apply to the file descriptors.
+ </para>
+ <para>
+ GEM handles are local to a DRM file. Applications get a handle to a GEM
+ object through a driver-specific ioctl, and can use that handle to refer
+ to the GEM object in other standard or driver-specific ioctls. Closing a
+ DRM file handle frees all its GEM handles and dereferences the
+ associated GEM objects.
+ </para>
+ <para>
+ To create a handle for a GEM object drivers call
+ <function>drm_gem_handle_create</function>. The function takes a pointer
+ to the DRM file and the GEM object and returns a locally unique handle.
+ When the handle is no longer needed drivers delete it with a call to
+ <function>drm_gem_handle_delete</function>. Finally the GEM object
+ associated with a handle can be retrieved by a call to
+ <function>drm_gem_object_lookup</function>.
+ </para>
+ <para>
+ Handles don't take ownership of GEM objects, they only take a reference
+ to the object that will be dropped when the handle is destroyed. To
+ avoid leaking GEM objects, drivers must make sure they drop the
+ reference(s) they own (such as the initial reference taken at object
+ creation time) as appropriate, without any special consideration for the
+ handle. For example, in the particular case of combined GEM object and
+ handle creation in the implementation of the
+ <methodname>dumb_create</methodname> operation, drivers must drop the
+ initial reference to the GEM object before returning the handle.
+ </para>
+ <para>
+ GEM names are similar in purpose to handles but are not local to DRM
+ files. They can be passed between processes to reference a GEM object
+ globally. Names can't be used directly to refer to objects in the DRM
+ API, applications must convert handles to names and names to handles
+ using the DRM_IOCTL_GEM_FLINK and DRM_IOCTL_GEM_OPEN ioctls
+ respectively. The conversion is handled by the DRM core without any
+ driver-specific support.
+ </para>
+ <para>
+ GEM also supports buffer sharing with dma-buf file descriptors through
+ PRIME. GEM-based drivers must use the provided helpers functions to
+ implement the exporting and importing correctly. See <xref linkend="drm-prime-support" />.
+ Since sharing file descriptors is inherently more secure than the
+ easily guessable and global GEM names it is the preferred buffer
+ sharing mechanism. Sharing buffers through GEM names is only supported
+ for legacy userspace. Furthermore PRIME also allows cross-device
+ buffer sharing since it is based on dma-bufs.
+ </para>
+ </sect3>
+ <sect3 id="drm-gem-objects-mapping">
+ <title>GEM Objects Mapping</title>
+ <para>
+ Because mapping operations are fairly heavyweight GEM favours
+ read/write-like access to buffers, implemented through driver-specific
+ ioctls, over mapping buffers to userspace. However, when random access
+ to the buffer is needed (to perform software rendering for instance),
+ direct access to the object can be more efficient.
+ </para>
+ <para>
+ The mmap system call can't be used directly to map GEM objects, as they
+ don't have their own file handle. Two alternative methods currently
+ co-exist to map GEM objects to userspace. The first method uses a
+ driver-specific ioctl to perform the mapping operation, calling
+ <function>do_mmap</function> under the hood. This is often considered
+ dubious, seems to be discouraged for new GEM-enabled drivers, and will
+ thus not be described here.
+ </para>
+ <para>
+ The second method uses the mmap system call on the DRM file handle.
+ <synopsis>void *mmap(void *addr, size_t length, int prot, int flags, int fd,
+ off_t offset);</synopsis>
+ DRM identifies the GEM object to be mapped by a fake offset passed
+ through the mmap offset argument. Prior to being mapped, a GEM object
+ must thus be associated with a fake offset. To do so, drivers must call
+ <function>drm_gem_create_mmap_offset</function> on the object. The
+ function allocates a fake offset range from a pool and stores the
+ offset divided by PAGE_SIZE in
+ <literal>obj->map_list.hash.key</literal>. Care must be taken not to
+ call <function>drm_gem_create_mmap_offset</function> if a fake offset
+ has already been allocated for the object. This can be tested by
+ <literal>obj->map_list.map</literal> being non-NULL.
+ </para>
+ <para>
+ Once allocated, the fake offset value
+ (<literal>obj->map_list.hash.key << PAGE_SHIFT</literal>)
+ must be passed to the application in a driver-specific way and can then
+ be used as the mmap offset argument.
+ </para>
+ <para>
+ The GEM core provides a helper method <function>drm_gem_mmap</function>
+ to handle object mapping. The method can be set directly as the mmap
+ file operation handler. It will look up the GEM object based on the
+ offset value and set the VMA operations to the
+ <structname>drm_driver</structname> <structfield>gem_vm_ops</structfield>
+ field. Note that <function>drm_gem_mmap</function> doesn't map memory to
+ userspace, but relies on the driver-provided fault handler to map pages
+ individually.
+ </para>
+ <para>
+ To use <function>drm_gem_mmap</function>, drivers must fill the struct
+ <structname>drm_driver</structname> <structfield>gem_vm_ops</structfield>
+ field with a pointer to VM operations.
+ </para>
+ <para>
+ <synopsis>struct vm_operations_struct *gem_vm_ops
+
+ struct vm_operations_struct {
+ void (*open)(struct vm_area_struct * area);
+ void (*close)(struct vm_area_struct * area);
+ int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
+ };</synopsis>
+ </para>
+ <para>
+ The <methodname>open</methodname> and <methodname>close</methodname>
+ operations must update the GEM object reference count. Drivers can use
+ the <function>drm_gem_vm_open</function> and
+ <function>drm_gem_vm_close</function> helper functions directly as open
+ and close handlers.
+ </para>
+ <para>
+ The fault operation handler is responsible for mapping individual pages
+ to userspace when a page fault occurs. Depending on the memory
+ allocation scheme, drivers can allocate pages at fault time, or can
+ decide to allocate memory for the GEM object at the time the object is
+ created.
+ </para>
+ <para>
+ Drivers that want to map the GEM object upfront instead of handling page
+ faults can implement their own mmap file operation handler.
+ </para>
+ </sect3>
+ <sect3>
+ <title>Memory Coherency</title>
+ <para>
+ When mapped to the device or used in a command buffer, backing pages
+ for an object are flushed to memory and marked write combined so as to
+ be coherent with the GPU. Likewise, if the CPU accesses an object
+ after the GPU has finished rendering to the object, then the object
+ must be made coherent with the CPU's view of memory, usually involving
+ GPU cache flushing of various kinds. This core CPU<->GPU
+ coherency management is provided by a device-specific ioctl, which
+ evaluates an object's current domain and performs any necessary
+ flushing or synchronization to put the object into the desired
+ coherency domain (note that the object may be busy, i.e. an active
+ render target; in that case, setting the domain blocks the client and
+ waits for rendering to complete before performing any necessary
+ flushing operations).
+ </para>
+ </sect3>
+ <sect3>
+ <title>Command Execution</title>
+ <para>
+ Perhaps the most important GEM function for GPU devices is providing a
+ command execution interface to clients. Client programs construct
+ command buffers containing references to previously allocated memory
+ objects, and then submit them to GEM. At that point, GEM takes care to
+ bind all the objects into the GTT, execute the buffer, and provide
+ necessary synchronization between clients accessing the same buffers.
+ This often involves evicting some objects from the GTT and re-binding
+ others (a fairly expensive operation), and providing relocation
+ support which hides fixed GTT offsets from clients. Clients must take
+ care not to submit command buffers that reference more objects than
+ can fit in the GTT; otherwise, GEM will reject them and no rendering
+ will occur. Similarly, if several objects in the buffer require fence
+ registers to be allocated for correct rendering (e.g. 2D blits on
+ pre-965 chips), care must be taken not to require more fence registers
+ than are available to the client. Such resource management should be
+ abstracted from the client in libdrm.
+ </para>
+ </sect3>
+ <sect3>
+ <title>GEM Function Reference</title>
+!Edrivers/gpu/drm/drm_gem.c
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>VMA Offset Manager</title>
+!Pdrivers/gpu/drm/drm_vma_manager.c vma offset manager
+!Edrivers/gpu/drm/drm_vma_manager.c
+!Iinclude/drm/drm_vma_manager.h
+ </sect2>
+ <sect2 id="drm-prime-support">
+ <title>PRIME Buffer Sharing</title>
+ <para>
+ PRIME is the cross device buffer sharing framework in drm, originally
+ created for the OPTIMUS range of multi-gpu platforms. To userspace
+ PRIME buffers are dma-buf based file descriptors.
+ </para>
+ <sect3>
+ <title>Overview and Driver Interface</title>
+ <para>
+ Similar to GEM global names, PRIME file descriptors are
+ also used to share buffer objects across processes. They offer
+ additional security: as file descriptors must be explicitly sent over
+ UNIX domain sockets to be shared between applications, they can't be
+ guessed like the globally unique GEM names.
+ </para>
+ <para>
+ Drivers that support the PRIME
+ API must set the DRIVER_PRIME bit in the struct
+ <structname>drm_driver</structname>
+ <structfield>driver_features</structfield> field, and implement the
+ <methodname>prime_handle_to_fd</methodname> and
+ <methodname>prime_fd_to_handle</methodname> operations.
+ </para>
+ <para>
+ <synopsis>int (*prime_handle_to_fd)(struct drm_device *dev,
+ struct drm_file *file_priv, uint32_t handle,
+ uint32_t flags, int *prime_fd);
+int (*prime_fd_to_handle)(struct drm_device *dev,
+ struct drm_file *file_priv, int prime_fd,
+ uint32_t *handle);</synopsis>
+ Those two operations convert a handle to a PRIME file descriptor and
+ vice versa. Drivers must use the kernel dma-buf buffer sharing framework
+ to manage the PRIME file descriptors. Similar to the mode setting
+ API PRIME is agnostic to the underlying buffer object manager, as
+ long as handles are 32bit unsigned integers.
+ </para>
+ <para>
+ While non-GEM drivers must implement the operations themselves, GEM
+ drivers must use the <function>drm_gem_prime_handle_to_fd</function>
+ and <function>drm_gem_prime_fd_to_handle</function> helper functions.
+ Those helpers rely on the driver
+ <methodname>gem_prime_export</methodname> and
+ <methodname>gem_prime_import</methodname> operations to create a dma-buf
+ instance from a GEM object (dma-buf exporter role) and to create a GEM
+ object from a dma-buf instance (dma-buf importer role).
+ </para>
+ <para>
+ <synopsis>struct dma_buf * (*gem_prime_export)(struct drm_device *dev,
+ struct drm_gem_object *obj,
+ int flags);
+struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev,
+ struct dma_buf *dma_buf);</synopsis>
+ These two operations are mandatory for GEM drivers that support
+ PRIME.
+ </para>
+ </sect3>
+ <sect3>
+ <title>PRIME Helper Functions</title>
+!Pdrivers/gpu/drm/drm_prime.c PRIME Helpers
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>PRIME Function References</title>
+!Edrivers/gpu/drm/drm_prime.c
+ </sect2>
+ <sect2>
+ <title>DRM MM Range Allocator</title>
+ <sect3>
+ <title>Overview</title>
+!Pdrivers/gpu/drm/drm_mm.c Overview
+ </sect3>
+ <sect3>
+ <title>LRU Scan/Eviction Support</title>
+!Pdrivers/gpu/drm/drm_mm.c lru scan roaster
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>DRM MM Range Allocator Function References</title>
+!Edrivers/gpu/drm/drm_mm.c
+!Iinclude/drm/drm_mm.h
+ </sect2>
+ <sect2>
+ <title>CMA Helper Functions Reference</title>
+!Pdrivers/gpu/drm/drm_gem_cma_helper.c cma helpers
+!Edrivers/gpu/drm/drm_gem_cma_helper.c
+!Iinclude/drm/drm_gem_cma_helper.h
+ </sect2>
+ </sect1>
+
+ <!-- Internals: mode setting -->
+
+ <sect1 id="drm-mode-setting">
+ <title>Mode Setting</title>
+ <para>
+ Drivers must initialize the mode setting core by calling
+ <function>drm_mode_config_init</function> on the DRM device. The function
+ initializes the <structname>drm_device</structname>
+ <structfield>mode_config</structfield> field and never fails. Once done,
+ mode configuration must be setup by initializing the following fields.
+ </para>
+ <itemizedlist>
+ <listitem>
+ <synopsis>int min_width, min_height;
+int max_width, max_height;</synopsis>
+ <para>
+ Minimum and maximum width and height of the frame buffers in pixel
+ units.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>struct drm_mode_config_funcs *funcs;</synopsis>
+ <para>Mode setting functions.</para>
+ </listitem>
+ </itemizedlist>
+ <sect2>
+ <title>Display Modes Function Reference</title>
+!Iinclude/drm/drm_modes.h
+!Edrivers/gpu/drm/drm_modes.c
+ </sect2>
+ <sect2>
+ <title>Atomic Mode Setting Function Reference</title>
+!Edrivers/gpu/drm/drm_atomic.c
+ </sect2>
+ <sect2>
+ <title>Frame Buffer Creation</title>
+ <synopsis>struct drm_framebuffer *(*fb_create)(struct drm_device *dev,
+ struct drm_file *file_priv,
+ struct drm_mode_fb_cmd2 *mode_cmd);</synopsis>
+ <para>
+ Frame buffers are abstract memory objects that provide a source of
+ pixels to scanout to a CRTC. Applications explicitly request the
+ creation of frame buffers through the DRM_IOCTL_MODE_ADDFB(2) ioctls and
+ receive an opaque handle that can be passed to the KMS CRTC control,
+ plane configuration and page flip functions.
+ </para>
+ <para>
+ Frame buffers rely on the underneath memory manager for low-level memory
+ operations. When creating a frame buffer applications pass a memory
+ handle (or a list of memory handles for multi-planar formats) through
+ the <parameter>drm_mode_fb_cmd2</parameter> argument. For drivers using
+ GEM as their userspace buffer management interface this would be a GEM
+ handle. Drivers are however free to use their own backing storage object
+ handles, e.g. vmwgfx directly exposes special TTM handles to userspace
+ and so expects TTM handles in the create ioctl and not GEM handles.
+ </para>
+ <para>
+ Drivers must first validate the requested frame buffer parameters passed
+ through the mode_cmd argument. In particular this is where invalid
+ sizes, pixel formats or pitches can be caught.
+ </para>
+ <para>
+ If the parameters are deemed valid, drivers then create, initialize and
+ return an instance of struct <structname>drm_framebuffer</structname>.
+ If desired the instance can be embedded in a larger driver-specific
+ structure. Drivers must fill its <structfield>width</structfield>,
+ <structfield>height</structfield>, <structfield>pitches</structfield>,
+ <structfield>offsets</structfield>, <structfield>depth</structfield>,
+ <structfield>bits_per_pixel</structfield> and
+ <structfield>pixel_format</structfield> fields from the values passed
+ through the <parameter>drm_mode_fb_cmd2</parameter> argument. They
+ should call the <function>drm_helper_mode_fill_fb_struct</function>
+ helper function to do so.
+ </para>
+
+ <para>
+ The initialization of the new framebuffer instance is finalized with a
+ call to <function>drm_framebuffer_init</function> which takes a pointer
+ to DRM frame buffer operations (struct
+ <structname>drm_framebuffer_funcs</structname>). Note that this function
+ publishes the framebuffer and so from this point on it can be accessed
+ concurrently from other threads. Hence it must be the last step in the
+ driver's framebuffer initialization sequence. Frame buffer operations
+ are
+ <itemizedlist>
+ <listitem>
+ <synopsis>int (*create_handle)(struct drm_framebuffer *fb,
+ struct drm_file *file_priv, unsigned int *handle);</synopsis>
+ <para>
+ Create a handle to the frame buffer underlying memory object. If
+ the frame buffer uses a multi-plane format, the handle will
+ reference the memory object associated with the first plane.
+ </para>
+ <para>
+ Drivers call <function>drm_gem_handle_create</function> to create
+ the handle.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*destroy)(struct drm_framebuffer *framebuffer);</synopsis>
+ <para>
+ Destroy the frame buffer object and frees all associated
+ resources. Drivers must call
+ <function>drm_framebuffer_cleanup</function> to free resources
+ allocated by the DRM core for the frame buffer object, and must
+ make sure to unreference all memory objects associated with the
+ frame buffer. Handles created by the
+ <methodname>create_handle</methodname> operation are released by
+ the DRM core.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*dirty)(struct drm_framebuffer *framebuffer,
+ struct drm_file *file_priv, unsigned flags, unsigned color,
+ struct drm_clip_rect *clips, unsigned num_clips);</synopsis>
+ <para>
+ This optional operation notifies the driver that a region of the
+ frame buffer has changed in response to a DRM_IOCTL_MODE_DIRTYFB
+ ioctl call.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ The lifetime of a drm framebuffer is controlled with a reference count,
+ drivers can grab additional references with
+ <function>drm_framebuffer_reference</function>and drop them
+ again with <function>drm_framebuffer_unreference</function>. For
+ driver-private framebuffers for which the last reference is never
+ dropped (e.g. for the fbdev framebuffer when the struct
+ <structname>drm_framebuffer</structname> is embedded into the fbdev
+ helper struct) drivers can manually clean up a framebuffer at module
+ unload time with
+ <function>drm_framebuffer_unregister_private</function>.
+ </para>
+ </sect2>
+ <sect2>
+ <title>Dumb Buffer Objects</title>
+ <para>
+ The KMS API doesn't standardize backing storage object creation and
+ leaves it to driver-specific ioctls. Furthermore actually creating a
+ buffer object even for GEM-based drivers is done through a
+ driver-specific ioctl - GEM only has a common userspace interface for
+ sharing and destroying objects. While not an issue for full-fledged
+ graphics stacks that include device-specific userspace components (in
+ libdrm for instance), this limit makes DRM-based early boot graphics
+ unnecessarily complex.
+ </para>
+ <para>
+ Dumb objects partly alleviate the problem by providing a standard
+ API to create dumb buffers suitable for scanout, which can then be used
+ to create KMS frame buffers.
+ </para>
+ <para>
+ To support dumb objects drivers must implement the
+ <methodname>dumb_create</methodname>,
+ <methodname>dumb_destroy</methodname> and
+ <methodname>dumb_map_offset</methodname> operations.
+ </para>
+ <itemizedlist>
+ <listitem>
+ <synopsis>int (*dumb_create)(struct drm_file *file_priv, struct drm_device *dev,
+ struct drm_mode_create_dumb *args);</synopsis>
+ <para>
+ The <methodname>dumb_create</methodname> operation creates a driver
+ object (GEM or TTM handle) suitable for scanout based on the
+ width, height and depth from the struct
+ <structname>drm_mode_create_dumb</structname> argument. It fills the
+ argument's <structfield>handle</structfield>,
+ <structfield>pitch</structfield> and <structfield>size</structfield>
+ fields with a handle for the newly created object and its line
+ pitch and size in bytes.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*dumb_destroy)(struct drm_file *file_priv, struct drm_device *dev,
+ uint32_t handle);</synopsis>
+ <para>
+ The <methodname>dumb_destroy</methodname> operation destroys a dumb
+ object created by <methodname>dumb_create</methodname>.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*dumb_map_offset)(struct drm_file *file_priv, struct drm_device *dev,
+ uint32_t handle, uint64_t *offset);</synopsis>
+ <para>
+ The <methodname>dumb_map_offset</methodname> operation associates an
+ mmap fake offset with the object given by the handle and returns
+ it. Drivers must use the
+ <function>drm_gem_create_mmap_offset</function> function to
+ associate the fake offset as described in
+ <xref linkend="drm-gem-objects-mapping"/>.
+ </para>
+ </listitem>
+ </itemizedlist>
+ <para>
+ Note that dumb objects may not be used for gpu acceleration, as has been
+ attempted on some ARM embedded platforms. Such drivers really must have
+ a hardware-specific ioctl to allocate suitable buffer objects.
+ </para>
+ </sect2>
+ <sect2>
+ <title>Output Polling</title>
+ <synopsis>void (*output_poll_changed)(struct drm_device *dev);</synopsis>
+ <para>
+ This operation notifies the driver that the status of one or more
+ connectors has changed. Drivers that use the fb helper can just call the
+ <function>drm_fb_helper_hotplug_event</function> function to handle this
+ operation.
+ </para>
+ </sect2>
+ <sect2>
+ <title>Locking</title>
+ <para>
+ Beside some lookup structures with their own locking (which is hidden
+ behind the interface functions) most of the modeset state is protected
+ by the <code>dev-<mode_config.lock</code> mutex and additionally
+ per-crtc locks to allow cursor updates, pageflips and similar operations
+ to occur concurrently with background tasks like output detection.
+ Operations which cross domains like a full modeset always grab all
+ locks. Drivers there need to protect resources shared between crtcs with
+ additional locking. They also need to be careful to always grab the
+ relevant crtc locks if a modset functions touches crtc state, e.g. for
+ load detection (which does only grab the <code>mode_config.lock</code>
+ to allow concurrent screen updates on live crtcs).
+ </para>
+ </sect2>
+ </sect1>
+
+ <!-- Internals: kms initialization and cleanup -->
+
+ <sect1 id="drm-kms-init">
+ <title>KMS Initialization and Cleanup</title>
+ <para>
+ A KMS device is abstracted and exposed as a set of planes, CRTCs, encoders
+ and connectors. KMS drivers must thus create and initialize all those
+ objects at load time after initializing mode setting.
+ </para>
+ <sect2>
+ <title>CRTCs (struct <structname>drm_crtc</structname>)</title>
+ <para>
+ A CRTC is an abstraction representing a part of the chip that contains a
+ pointer to a scanout buffer. Therefore, the number of CRTCs available
+ determines how many independent scanout buffers can be active at any
+ given time. The CRTC structure contains several fields to support this:
+ a pointer to some video memory (abstracted as a frame buffer object), a
+ display mode, and an (x, y) offset into the video memory to support
+ panning or configurations where one piece of video memory spans multiple
+ CRTCs.
+ </para>
+ <sect3>
+ <title>CRTC Initialization</title>
+ <para>
+ A KMS device must create and register at least one struct
+ <structname>drm_crtc</structname> instance. The instance is allocated
+ and zeroed by the driver, possibly as part of a larger structure, and
+ registered with a call to <function>drm_crtc_init</function> with a
+ pointer to CRTC functions.
+ </para>
+ </sect3>
+ <sect3 id="drm-kms-crtcops">
+ <title>CRTC Operations</title>
+ <sect4>
+ <title>Set Configuration</title>
+ <synopsis>int (*set_config)(struct drm_mode_set *set);</synopsis>
+ <para>
+ Apply a new CRTC configuration to the device. The configuration
+ specifies a CRTC, a frame buffer to scan out from, a (x,y) position in
+ the frame buffer, a display mode and an array of connectors to drive
+ with the CRTC if possible.
+ </para>
+ <para>
+ If the frame buffer specified in the configuration is NULL, the driver
+ must detach all encoders connected to the CRTC and all connectors
+ attached to those encoders and disable them.
+ </para>
+ <para>
+ This operation is called with the mode config lock held.
+ </para>
+ <note><para>
+ Note that the drm core has no notion of restoring the mode setting
+ state after resume, since all resume handling is in the full
+ responsibility of the driver. The common mode setting helper library
+ though provides a helper which can be used for this:
+ <function>drm_helper_resume_force_mode</function>.
+ </para></note>
+ </sect4>
+ <sect4>
+ <title>Page Flipping</title>
+ <synopsis>int (*page_flip)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+ struct drm_pending_vblank_event *event);</synopsis>
+ <para>
+ Schedule a page flip to the given frame buffer for the CRTC. This
+ operation is called with the mode config mutex held.
+ </para>
+ <para>
+ Page flipping is a synchronization mechanism that replaces the frame
+ buffer being scanned out by the CRTC with a new frame buffer during
+ vertical blanking, avoiding tearing. When an application requests a page
+ flip the DRM core verifies that the new frame buffer is large enough to
+ be scanned out by the CRTC in the currently configured mode and then
+ calls the CRTC <methodname>page_flip</methodname> operation with a
+ pointer to the new frame buffer.
+ </para>
+ <para>
+ The <methodname>page_flip</methodname> operation schedules a page flip.
+ Once any pending rendering targeting the new frame buffer has
+ completed, the CRTC will be reprogrammed to display that frame buffer
+ after the next vertical refresh. The operation must return immediately
+ without waiting for rendering or page flip to complete and must block
+ any new rendering to the frame buffer until the page flip completes.
+ </para>
+ <para>
+ If a page flip can be successfully scheduled the driver must set the
+ <code>drm_crtc->fb</code> field to the new framebuffer pointed to
+ by <code>fb</code>. This is important so that the reference counting
+ on framebuffers stays balanced.
+ </para>
+ <para>
+ If a page flip is already pending, the
+ <methodname>page_flip</methodname> operation must return
+ -<errorname>EBUSY</errorname>.
+ </para>
+ <para>
+ To synchronize page flip to vertical blanking the driver will likely
+ need to enable vertical blanking interrupts. It should call
+ <function>drm_vblank_get</function> for that purpose, and call
+ <function>drm_vblank_put</function> after the page flip completes.
+ </para>
+ <para>
+ If the application has requested to be notified when page flip completes
+ the <methodname>page_flip</methodname> operation will be called with a
+ non-NULL <parameter>event</parameter> argument pointing to a
+ <structname>drm_pending_vblank_event</structname> instance. Upon page
+ flip completion the driver must call <methodname>drm_send_vblank_event</methodname>
+ to fill in the event and send to wake up any waiting processes.
+ This can be performed with
+ <programlisting><![CDATA[
+ spin_lock_irqsave(&dev->event_lock, flags);
+ ...
+ drm_send_vblank_event(dev, pipe, event);
+ spin_unlock_irqrestore(&dev->event_lock, flags);
+ ]]></programlisting>
+ </para>
+ <note><para>
+ FIXME: Could drivers that don't need to wait for rendering to complete
+ just add the event to <literal>dev->vblank_event_list</literal> and
+ let the DRM core handle everything, as for "normal" vertical blanking
+ events?
+ </para></note>
+ <para>
+ While waiting for the page flip to complete, the
+ <literal>event->base.link</literal> list head can be used freely by
+ the driver to store the pending event in a driver-specific list.
+ </para>
+ <para>
+ If the file handle is closed before the event is signaled, drivers must
+ take care to destroy the event in their
+ <methodname>preclose</methodname> operation (and, if needed, call
+ <function>drm_vblank_put</function>).
+ </para>
+ </sect4>
+ <sect4>
+ <title>Miscellaneous</title>
+ <itemizedlist>
+ <listitem>
+ <synopsis>void (*set_property)(struct drm_crtc *crtc,
+ struct drm_property *property, uint64_t value);</synopsis>
+ <para>
+ Set the value of the given CRTC property to
+ <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
+ for more information about properties.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*gamma_set)(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
+ uint32_t start, uint32_t size);</synopsis>
+ <para>
+ Apply a gamma table to the device. The operation is optional.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*destroy)(struct drm_crtc *crtc);</synopsis>
+ <para>
+ Destroy the CRTC when not needed anymore. See
+ <xref linkend="drm-kms-init"/>.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </sect4>
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>Planes (struct <structname>drm_plane</structname>)</title>
+ <para>
+ A plane represents an image source that can be blended with or overlayed
+ on top of a CRTC during the scanout process. Planes are associated with
+ a frame buffer to crop a portion of the image memory (source) and
+ optionally scale it to a destination size. The result is then blended
+ with or overlayed on top of a CRTC.
+ </para>
+ <para>
+ The DRM core recognizes three types of planes:
+ <itemizedlist>
+ <listitem>
+ DRM_PLANE_TYPE_PRIMARY represents a "main" plane for a CRTC. Primary
+ planes are the planes operated upon by CRTC modesetting and flipping
+ operations described in <xref linkend="drm-kms-crtcops"/>.
+ </listitem>
+ <listitem>
+ DRM_PLANE_TYPE_CURSOR represents a "cursor" plane for a CRTC. Cursor
+ planes are the planes operated upon by the DRM_IOCTL_MODE_CURSOR and
+ DRM_IOCTL_MODE_CURSOR2 ioctls.
+ </listitem>
+ <listitem>
+ DRM_PLANE_TYPE_OVERLAY represents all non-primary, non-cursor planes.
+ Some drivers refer to these types of planes as "sprites" internally.
+ </listitem>
+ </itemizedlist>
+ For compatibility with legacy userspace, only overlay planes are made
+ available to userspace by default. Userspace clients may set the
+ DRM_CLIENT_CAP_UNIVERSAL_PLANES client capability bit to indicate that
+ they wish to receive a universal plane list containing all plane types.
+ </para>
+ <sect3>
+ <title>Plane Initialization</title>
+ <para>
+ To create a plane, a KMS drivers allocates and
+ zeroes an instances of struct <structname>drm_plane</structname>
+ (possibly as part of a larger structure) and registers it with a call
+ to <function>drm_universal_plane_init</function>. The function takes a bitmask
+ of the CRTCs that can be associated with the plane, a pointer to the
+ plane functions, a list of format supported formats, and the type of
+ plane (primary, cursor, or overlay) being initialized.
+ </para>
+ <para>
+ Cursor and overlay planes are optional. All drivers should provide
+ one primary plane per CRTC (although this requirement may change in
+ the future); drivers that do not wish to provide special handling for
+ primary planes may make use of the helper functions described in
+ <xref linkend="drm-kms-planehelpers"/> to create and register a
+ primary plane with standard capabilities.
+ </para>
+ </sect3>
+ <sect3>
+ <title>Plane Operations</title>
+ <itemizedlist>
+ <listitem>
+ <synopsis>int (*update_plane)(struct drm_plane *plane, struct drm_crtc *crtc,
+ struct drm_framebuffer *fb, int crtc_x, int crtc_y,
+ unsigned int crtc_w, unsigned int crtc_h,
+ uint32_t src_x, uint32_t src_y,
+ uint32_t src_w, uint32_t src_h);</synopsis>
+ <para>
+ Enable and configure the plane to use the given CRTC and frame buffer.
+ </para>
+ <para>
+ The source rectangle in frame buffer memory coordinates is given by
+ the <parameter>src_x</parameter>, <parameter>src_y</parameter>,
+ <parameter>src_w</parameter> and <parameter>src_h</parameter>
+ parameters (as 16.16 fixed point values). Devices that don't support
+ subpixel plane coordinates can ignore the fractional part.
+ </para>
+ <para>
+ The destination rectangle in CRTC coordinates is given by the
+ <parameter>crtc_x</parameter>, <parameter>crtc_y</parameter>,
+ <parameter>crtc_w</parameter> and <parameter>crtc_h</parameter>
+ parameters (as integer values). Devices scale the source rectangle to
+ the destination rectangle. If scaling is not supported, and the source
+ rectangle size doesn't match the destination rectangle size, the
+ driver must return a -<errorname>EINVAL</errorname> error.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*disable_plane)(struct drm_plane *plane);</synopsis>
+ <para>
+ Disable the plane. The DRM core calls this method in response to a
+ DRM_IOCTL_MODE_SETPLANE ioctl call with the frame buffer ID set to 0.
+ Disabled planes must not be processed by the CRTC.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*destroy)(struct drm_plane *plane);</synopsis>
+ <para>
+ Destroy the plane when not needed anymore. See
+ <xref linkend="drm-kms-init"/>.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>Encoders (struct <structname>drm_encoder</structname>)</title>
+ <para>
+ An encoder takes pixel data from a CRTC and converts it to a format
+ suitable for any attached connectors. On some devices, it may be
+ possible to have a CRTC send data to more than one encoder. In that
+ case, both encoders would receive data from the same scanout buffer,
+ resulting in a "cloned" display configuration across the connectors
+ attached to each encoder.
+ </para>
+ <sect3>
+ <title>Encoder Initialization</title>
+ <para>
+ As for CRTCs, a KMS driver must create, initialize and register at
+ least one struct <structname>drm_encoder</structname> instance. The
+ instance is allocated and zeroed by the driver, possibly as part of a
+ larger structure.
+ </para>
+ <para>
+ Drivers must initialize the struct <structname>drm_encoder</structname>
+ <structfield>possible_crtcs</structfield> and
+ <structfield>possible_clones</structfield> fields before registering the
+ encoder. Both fields are bitmasks of respectively the CRTCs that the
+ encoder can be connected to, and sibling encoders candidate for cloning.
+ </para>
+ <para>
+ After being initialized, the encoder must be registered with a call to
+ <function>drm_encoder_init</function>. The function takes a pointer to
+ the encoder functions and an encoder type. Supported types are
+ <itemizedlist>
+ <listitem>
+ DRM_MODE_ENCODER_DAC for VGA and analog on DVI-I/DVI-A
+ </listitem>
+ <listitem>
+ DRM_MODE_ENCODER_TMDS for DVI, HDMI and (embedded) DisplayPort
+ </listitem>
+ <listitem>
+ DRM_MODE_ENCODER_LVDS for display panels
+ </listitem>
+ <listitem>
+ DRM_MODE_ENCODER_TVDAC for TV output (Composite, S-Video, Component,
+ SCART)
+ </listitem>
+ <listitem>
+ DRM_MODE_ENCODER_VIRTUAL for virtual machine displays
+ </listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ Encoders must be attached to a CRTC to be used. DRM drivers leave
+ encoders unattached at initialization time. Applications (or the fbdev
+ compatibility layer when implemented) are responsible for attaching the
+ encoders they want to use to a CRTC.
+ </para>
+ </sect3>
+ <sect3>
+ <title>Encoder Operations</title>
+ <itemizedlist>
+ <listitem>
+ <synopsis>void (*destroy)(struct drm_encoder *encoder);</synopsis>
+ <para>
+ Called to destroy the encoder when not needed anymore. See
+ <xref linkend="drm-kms-init"/>.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*set_property)(struct drm_plane *plane,
+ struct drm_property *property, uint64_t value);</synopsis>
+ <para>
+ Set the value of the given plane property to
+ <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
+ for more information about properties.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>Connectors (struct <structname>drm_connector</structname>)</title>
+ <para>
+ A connector is the final destination for pixel data on a device, and
+ usually connects directly to an external display device like a monitor
+ or laptop panel. A connector can only be attached to one encoder at a
+ time. The connector is also the structure where information about the
+ attached display is kept, so it contains fields for display data, EDID
+ data, DPMS & connection status, and information about modes
+ supported on the attached displays.
+ </para>
+ <sect3>
+ <title>Connector Initialization</title>
+ <para>
+ Finally a KMS driver must create, initialize, register and attach at
+ least one struct <structname>drm_connector</structname> instance. The
+ instance is created as other KMS objects and initialized by setting the
+ following fields.
+ </para>
+ <variablelist>
+ <varlistentry>
+ <term><structfield>interlace_allowed</structfield></term>
+ <listitem><para>
+ Whether the connector can handle interlaced modes.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term><structfield>doublescan_allowed</structfield></term>
+ <listitem><para>
+ Whether the connector can handle doublescan.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term><structfield>display_info
+ </structfield></term>
+ <listitem><para>
+ Display information is filled from EDID information when a display
+ is detected. For non hot-pluggable displays such as flat panels in
+ embedded systems, the driver should initialize the
+ <structfield>display_info</structfield>.<structfield>width_mm</structfield>
+ and
+ <structfield>display_info</structfield>.<structfield>height_mm</structfield>
+ fields with the physical size of the display.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term id="drm-kms-connector-polled"><structfield>polled</structfield></term>
+ <listitem><para>
+ Connector polling mode, a combination of
+ <variablelist>
+ <varlistentry>
+ <term>DRM_CONNECTOR_POLL_HPD</term>
+ <listitem><para>
+ The connector generates hotplug events and doesn't need to be
+ periodically polled. The CONNECT and DISCONNECT flags must not
+ be set together with the HPD flag.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_CONNECTOR_POLL_CONNECT</term>
+ <listitem><para>
+ Periodically poll the connector for connection.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_CONNECTOR_POLL_DISCONNECT</term>
+ <listitem><para>
+ Periodically poll the connector for disconnection.
+ </para></listitem>
+ </varlistentry>
+ </variablelist>
+ Set to 0 for connectors that don't support connection status
+ discovery.
+ </para></listitem>
+ </varlistentry>
+ </variablelist>
+ <para>
+ The connector is then registered with a call to
+ <function>drm_connector_init</function> with a pointer to the connector
+ functions and a connector type, and exposed through sysfs with a call to
+ <function>drm_connector_register</function>.
+ </para>
+ <para>
+ Supported connector types are
+ <itemizedlist>
+ <listitem>DRM_MODE_CONNECTOR_VGA</listitem>
+ <listitem>DRM_MODE_CONNECTOR_DVII</listitem>
+ <listitem>DRM_MODE_CONNECTOR_DVID</listitem>
+ <listitem>DRM_MODE_CONNECTOR_DVIA</listitem>
+ <listitem>DRM_MODE_CONNECTOR_Composite</listitem>
+ <listitem>DRM_MODE_CONNECTOR_SVIDEO</listitem>
+ <listitem>DRM_MODE_CONNECTOR_LVDS</listitem>
+ <listitem>DRM_MODE_CONNECTOR_Component</listitem>
+ <listitem>DRM_MODE_CONNECTOR_9PinDIN</listitem>
+ <listitem>DRM_MODE_CONNECTOR_DisplayPort</listitem>
+ <listitem>DRM_MODE_CONNECTOR_HDMIA</listitem>
+ <listitem>DRM_MODE_CONNECTOR_HDMIB</listitem>
+ <listitem>DRM_MODE_CONNECTOR_TV</listitem>
+ <listitem>DRM_MODE_CONNECTOR_eDP</listitem>
+ <listitem>DRM_MODE_CONNECTOR_VIRTUAL</listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ Connectors must be attached to an encoder to be used. For devices that
+ map connectors to encoders 1:1, the connector should be attached at
+ initialization time with a call to
+ <function>drm_mode_connector_attach_encoder</function>. The driver must
+ also set the <structname>drm_connector</structname>
+ <structfield>encoder</structfield> field to point to the attached
+ encoder.
+ </para>
+ <para>
+ Finally, drivers must initialize the connectors state change detection
+ with a call to <function>drm_kms_helper_poll_init</function>. If at
+ least one connector is pollable but can't generate hotplug interrupts
+ (indicated by the DRM_CONNECTOR_POLL_CONNECT and
+ DRM_CONNECTOR_POLL_DISCONNECT connector flags), a delayed work will
+ automatically be queued to periodically poll for changes. Connectors
+ that can generate hotplug interrupts must be marked with the
+ DRM_CONNECTOR_POLL_HPD flag instead, and their interrupt handler must
+ call <function>drm_helper_hpd_irq_event</function>. The function will
+ queue a delayed work to check the state of all connectors, but no
+ periodic polling will be done.
+ </para>
+ </sect3>
+ <sect3>
+ <title>Connector Operations</title>
+ <note><para>
+ Unless otherwise state, all operations are mandatory.
+ </para></note>
+ <sect4>
+ <title>DPMS</title>
+ <synopsis>void (*dpms)(struct drm_connector *connector, int mode);</synopsis>
+ <para>
+ The DPMS operation sets the power state of a connector. The mode
+ argument is one of
+ <itemizedlist>
+ <listitem><para>DRM_MODE_DPMS_ON</para></listitem>
+ <listitem><para>DRM_MODE_DPMS_STANDBY</para></listitem>
+ <listitem><para>DRM_MODE_DPMS_SUSPEND</para></listitem>
+ <listitem><para>DRM_MODE_DPMS_OFF</para></listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ In all but DPMS_ON mode the encoder to which the connector is attached
+ should put the display in low-power mode by driving its signals
+ appropriately. If more than one connector is attached to the encoder
+ care should be taken not to change the power state of other displays as
+ a side effect. Low-power mode should be propagated to the encoders and
+ CRTCs when all related connectors are put in low-power mode.
+ </para>
+ </sect4>
+ <sect4>
+ <title>Modes</title>
+ <synopsis>int (*fill_modes)(struct drm_connector *connector, uint32_t max_width,
+ uint32_t max_height);</synopsis>
+ <para>
+ Fill the mode list with all supported modes for the connector. If the
+ <parameter>max_width</parameter> and <parameter>max_height</parameter>
+ arguments are non-zero, the implementation must ignore all modes wider
+ than <parameter>max_width</parameter> or higher than
+ <parameter>max_height</parameter>.
+ </para>
+ <para>
+ The connector must also fill in this operation its
+ <structfield>display_info</structfield>
+ <structfield>width_mm</structfield> and
+ <structfield>height_mm</structfield> fields with the connected display
+ physical size in millimeters. The fields should be set to 0 if the value
+ isn't known or is not applicable (for instance for projector devices).
+ </para>
+ </sect4>
+ <sect4>
+ <title>Connection Status</title>
+ <para>
+ The connection status is updated through polling or hotplug events when
+ supported (see <xref linkend="drm-kms-connector-polled"/>). The status
+ value is reported to userspace through ioctls and must not be used
+ inside the driver, as it only gets initialized by a call to
+ <function>drm_mode_getconnector</function> from userspace.
+ </para>
+ <synopsis>enum drm_connector_status (*detect)(struct drm_connector *connector,
+ bool force);</synopsis>
+ <para>
+ Check to see if anything is attached to the connector. The
+ <parameter>force</parameter> parameter is set to false whilst polling or
+ to true when checking the connector due to user request.
+ <parameter>force</parameter> can be used by the driver to avoid
+ expensive, destructive operations during automated probing.
+ </para>
+ <para>
+ Return connector_status_connected if something is connected to the
+ connector, connector_status_disconnected if nothing is connected and
+ connector_status_unknown if the connection state isn't known.
+ </para>
+ <para>
+ Drivers should only return connector_status_connected if the connection
+ status has really been probed as connected. Connectors that can't detect
+ the connection status, or failed connection status probes, should return
+ connector_status_unknown.
+ </para>
+ </sect4>
+ <sect4>
+ <title>Miscellaneous</title>
+ <itemizedlist>
+ <listitem>
+ <synopsis>void (*set_property)(struct drm_connector *connector,
+ struct drm_property *property, uint64_t value);</synopsis>
+ <para>
+ Set the value of the given connector property to
+ <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
+ for more information about properties.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*destroy)(struct drm_connector *connector);</synopsis>
+ <para>
+ Destroy the connector when not needed anymore. See
+ <xref linkend="drm-kms-init"/>.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </sect4>
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>Cleanup</title>
+ <para>
+ The DRM core manages its objects' lifetime. When an object is not needed
+ anymore the core calls its destroy function, which must clean up and
+ free every resource allocated for the object. Every
+ <function>drm_*_init</function> call must be matched with a
+ corresponding <function>drm_*_cleanup</function> call to cleanup CRTCs
+ (<function>drm_crtc_cleanup</function>), planes
+ (<function>drm_plane_cleanup</function>), encoders
+ (<function>drm_encoder_cleanup</function>) and connectors
+ (<function>drm_connector_cleanup</function>). Furthermore, connectors
+ that have been added to sysfs must be removed by a call to
+ <function>drm_connector_unregister</function> before calling
+ <function>drm_connector_cleanup</function>.
+ </para>
+ <para>
+ Connectors state change detection must be cleanup up with a call to
+ <function>drm_kms_helper_poll_fini</function>.
+ </para>
+ </sect2>
+ <sect2>
+ <title>Output discovery and initialization example</title>
+ <programlisting><![CDATA[
+void intel_crt_init(struct drm_device *dev)
+{
+ struct drm_connector *connector;
+ struct intel_output *intel_output;
+
+ intel_output = kzalloc(sizeof(struct intel_output), GFP_KERNEL);
+ if (!intel_output)
+ return;
+
+ connector = &intel_output->base;
+ drm_connector_init(dev, &intel_output->base,
+ &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA);
+
+ drm_encoder_init(dev, &intel_output->enc, &intel_crt_enc_funcs,
+ DRM_MODE_ENCODER_DAC);
+
+ drm_mode_connector_attach_encoder(&intel_output->base,
+ &intel_output->enc);
+
+ /* Set up the DDC bus. */
+ intel_output->ddc_bus = intel_i2c_create(dev, GPIOA, "CRTDDC_A");
+ if (!intel_output->ddc_bus) {
+ dev_printk(KERN_ERR, &dev->pdev->dev, "DDC bus registration "
+ "failed.\n");
+ return;
+ }
+
+ intel_output->type = INTEL_OUTPUT_ANALOG;
+ connector->interlace_allowed = 0;
+ connector->doublescan_allowed = 0;
+
+ drm_encoder_helper_add(&intel_output->enc, &intel_crt_helper_funcs);
+ drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
+
+ drm_connector_register(connector);
+}]]></programlisting>
+ <para>
+ In the example above (taken from the i915 driver), a CRTC, connector and
+ encoder combination is created. A device-specific i2c bus is also
+ created for fetching EDID data and performing monitor detection. Once
+ the process is complete, the new connector is registered with sysfs to
+ make its properties available to applications.
+ </para>
+ </sect2>
+ <sect2>
+ <title>KMS API Functions</title>
+!Edrivers/gpu/drm/drm_crtc.c
+ </sect2>
+ <sect2>
+ <title>KMS Data Structures</title>
+!Iinclude/drm/drm_crtc.h
+ </sect2>
+ <sect2>
+ <title>KMS Locking</title>
+!Pdrivers/gpu/drm/drm_modeset_lock.c kms locking
+!Iinclude/drm/drm_modeset_lock.h
+!Edrivers/gpu/drm/drm_modeset_lock.c
+ </sect2>
+ </sect1>
+
+ <!-- Internals: kms helper functions -->
+
+ <sect1>
+ <title>Mode Setting Helper Functions</title>
+ <para>
+ The plane, CRTC, encoder and connector functions provided by the drivers
+ implement the DRM API. They're called by the DRM core and ioctl handlers
+ to handle device state changes and configuration request. As implementing
+ those functions often requires logic not specific to drivers, mid-layer
+ helper functions are available to avoid duplicating boilerplate code.
+ </para>
+ <para>
+ The DRM core contains one mid-layer implementation. The mid-layer provides
+ implementations of several plane, CRTC, encoder and connector functions
+ (called from the top of the mid-layer) that pre-process requests and call
+ lower-level functions provided by the driver (at the bottom of the
+ mid-layer). For instance, the
+ <function>drm_crtc_helper_set_config</function> function can be used to
+ fill the struct <structname>drm_crtc_funcs</structname>
+ <structfield>set_config</structfield> field. When called, it will split
+ the <methodname>set_config</methodname> operation in smaller, simpler
+ operations and call the driver to handle them.
+ </para>
+ <para>
+ To use the mid-layer, drivers call <function>drm_crtc_helper_add</function>,
+ <function>drm_encoder_helper_add</function> and
+ <function>drm_connector_helper_add</function> functions to install their
+ mid-layer bottom operations handlers, and fill the
+ <structname>drm_crtc_funcs</structname>,
+ <structname>drm_encoder_funcs</structname> and
+ <structname>drm_connector_funcs</structname> structures with pointers to
+ the mid-layer top API functions. Installing the mid-layer bottom operation
+ handlers is best done right after registering the corresponding KMS object.
+ </para>
+ <para>
+ The mid-layer is not split between CRTC, encoder and connector operations.
+ To use it, a driver must provide bottom functions for all of the three KMS
+ entities.
+ </para>
+ <sect2>
+ <title>Helper Functions</title>
+ <itemizedlist>
+ <listitem>
+ <synopsis>int drm_crtc_helper_set_config(struct drm_mode_set *set);</synopsis>
+ <para>
+ The <function>drm_crtc_helper_set_config</function> helper function
+ is a CRTC <methodname>set_config</methodname> implementation. It
+ first tries to locate the best encoder for each connector by calling
+ the connector <methodname>best_encoder</methodname> helper
+ operation.
+ </para>
+ <para>
+ After locating the appropriate encoders, the helper function will
+ call the <methodname>mode_fixup</methodname> encoder and CRTC helper
+ operations to adjust the requested mode, or reject it completely in
+ which case an error will be returned to the application. If the new
+ configuration after mode adjustment is identical to the current
+ configuration the helper function will return without performing any
+ other operation.
+ </para>
+ <para>
+ If the adjusted mode is identical to the current mode but changes to
+ the frame buffer need to be applied, the
+ <function>drm_crtc_helper_set_config</function> function will call
+ the CRTC <methodname>mode_set_base</methodname> helper operation. If
+ the adjusted mode differs from the current mode, or if the
+ <methodname>mode_set_base</methodname> helper operation is not
+ provided, the helper function performs a full mode set sequence by
+ calling the <methodname>prepare</methodname>,
+ <methodname>mode_set</methodname> and
+ <methodname>commit</methodname> CRTC and encoder helper operations,
+ in that order.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void drm_helper_connector_dpms(struct drm_connector *connector, int mode);</synopsis>
+ <para>
+ The <function>drm_helper_connector_dpms</function> helper function
+ is a connector <methodname>dpms</methodname> implementation that
+ tracks power state of connectors. To use the function, drivers must
+ provide <methodname>dpms</methodname> helper operations for CRTCs
+ and encoders to apply the DPMS state to the device.
+ </para>
+ <para>
+ The mid-layer doesn't track the power state of CRTCs and encoders.
+ The <methodname>dpms</methodname> helper operations can thus be
+ called with a mode identical to the currently active mode.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
+ uint32_t maxX, uint32_t maxY);</synopsis>
+ <para>
+ The <function>drm_helper_probe_single_connector_modes</function> helper
+ function is a connector <methodname>fill_modes</methodname>
+ implementation that updates the connection status for the connector
+ and then retrieves a list of modes by calling the connector
+ <methodname>get_modes</methodname> helper operation.
+ </para>
+ <para>
+ If the helper operation returns no mode, and if the connector status
+ is connector_status_connected, standard VESA DMT modes up to
+ 1024x768 are automatically added to the modes list by a call to
+ <function>drm_add_modes_noedid</function>.
+ </para>
+ <para>
+ The function then filters out modes larger than
+ <parameter>max_width</parameter> and <parameter>max_height</parameter>
+ if specified. It finally calls the optional connector
+ <methodname>mode_valid</methodname> helper operation for each mode in
+ the probed list to check whether the mode is valid for the connector.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </sect2>
+ <sect2>
+ <title>CRTC Helper Operations</title>
+ <itemizedlist>
+ <listitem id="drm-helper-crtc-mode-fixup">
+ <synopsis>bool (*mode_fixup)(struct drm_crtc *crtc,
+ const struct drm_display_mode *mode,
+ struct drm_display_mode *adjusted_mode);</synopsis>
+ <para>
+ Let CRTCs adjust the requested mode or reject it completely. This
+ operation returns true if the mode is accepted (possibly after being
+ adjusted) or false if it is rejected.
+ </para>
+ <para>
+ The <methodname>mode_fixup</methodname> operation should reject the
+ mode if it can't reasonably use it. The definition of "reasonable"
+ is currently fuzzy in this context. One possible behaviour would be
+ to set the adjusted mode to the panel timings when a fixed-mode
+ panel is used with hardware capable of scaling. Another behaviour
+ would be to accept any input mode and adjust it to the closest mode
+ supported by the hardware (FIXME: This needs to be clarified).
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*mode_set_base)(struct drm_crtc *crtc, int x, int y,
+ struct drm_framebuffer *old_fb)</synopsis>
+ <para>
+ Move the CRTC on the current frame buffer (stored in
+ <literal>crtc->fb</literal>) to position (x,y). Any of the frame
+ buffer, x position or y position may have been modified.
+ </para>
+ <para>
+ This helper operation is optional. If not provided, the
+ <function>drm_crtc_helper_set_config</function> function will fall
+ back to the <methodname>mode_set</methodname> helper operation.
+ </para>
+ <note><para>
+ FIXME: Why are x and y passed as arguments, as they can be accessed
+ through <literal>crtc->x</literal> and
+ <literal>crtc->y</literal>?
+ </para></note>
+ </listitem>
+ <listitem>
+ <synopsis>void (*prepare)(struct drm_crtc *crtc);</synopsis>
+ <para>
+ Prepare the CRTC for mode setting. This operation is called after
+ validating the requested mode. Drivers use it to perform
+ device-specific operations required before setting the new mode.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*mode_set)(struct drm_crtc *crtc, struct drm_display_mode *mode,
+ struct drm_display_mode *adjusted_mode, int x, int y,
+ struct drm_framebuffer *old_fb);</synopsis>
+ <para>
+ Set a new mode, position and frame buffer. Depending on the device
+ requirements, the mode can be stored internally by the driver and
+ applied in the <methodname>commit</methodname> operation, or
+ programmed to the hardware immediately.
+ </para>
+ <para>
+ The <methodname>mode_set</methodname> operation returns 0 on success
+ or a negative error code if an error occurs.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*commit)(struct drm_crtc *crtc);</synopsis>
+ <para>
+ Commit a mode. This operation is called after setting the new mode.
+ Upon return the device must use the new mode and be fully
+ operational.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </sect2>
+ <sect2>
+ <title>Encoder Helper Operations</title>
+ <itemizedlist>
+ <listitem>
+ <synopsis>bool (*mode_fixup)(struct drm_encoder *encoder,
+ const struct drm_display_mode *mode,
+ struct drm_display_mode *adjusted_mode);</synopsis>
+ <para>
+ Let encoders adjust the requested mode or reject it completely. This
+ operation returns true if the mode is accepted (possibly after being
+ adjusted) or false if it is rejected. See the
+ <link linkend="drm-helper-crtc-mode-fixup">mode_fixup CRTC helper
+ operation</link> for an explanation of the allowed adjustments.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*prepare)(struct drm_encoder *encoder);</synopsis>
+ <para>
+ Prepare the encoder for mode setting. This operation is called after
+ validating the requested mode. Drivers use it to perform
+ device-specific operations required before setting the new mode.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*mode_set)(struct drm_encoder *encoder,
+ struct drm_display_mode *mode,
+ struct drm_display_mode *adjusted_mode);</synopsis>
+ <para>
+ Set a new mode. Depending on the device requirements, the mode can
+ be stored internally by the driver and applied in the
+ <methodname>commit</methodname> operation, or programmed to the
+ hardware immediately.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>void (*commit)(struct drm_encoder *encoder);</synopsis>
+ <para>
+ Commit a mode. This operation is called after setting the new mode.
+ Upon return the device must use the new mode and be fully
+ operational.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </sect2>
+ <sect2>
+ <title>Connector Helper Operations</title>
+ <itemizedlist>
+ <listitem>
+ <synopsis>struct drm_encoder *(*best_encoder)(struct drm_connector *connector);</synopsis>
+ <para>
+ Return a pointer to the best encoder for the connecter. Device that
+ map connectors to encoders 1:1 simply return the pointer to the
+ associated encoder. This operation is mandatory.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*get_modes)(struct drm_connector *connector);</synopsis>
+ <para>
+ Fill the connector's <structfield>probed_modes</structfield> list
+ by parsing EDID data with <function>drm_add_edid_modes</function>,
+ adding standard VESA DMT modes with <function>drm_add_modes_noedid</function>,
+ or calling <function>drm_mode_probed_add</function> directly for every
+ supported mode and return the number of modes it has detected. This
+ operation is mandatory.
+ </para>
+ <para>
+ Note that the caller function will automatically add standard VESA
+ DMT modes up to 1024x768 if the <methodname>get_modes</methodname>
+ helper operation returns no mode and if the connector status is
+ connector_status_connected. There is no need to call
+ <function>drm_add_edid_modes</function> manually in that case.
+ </para>
+ <para>
+ When adding modes manually the driver creates each mode with a call to
+ <function>drm_mode_create</function> and must fill the following fields.
+ <itemizedlist>
+ <listitem>
+ <synopsis>__u32 type;</synopsis>
+ <para>
+ Mode type bitmask, a combination of
+ <variablelist>
+ <varlistentry>
+ <term>DRM_MODE_TYPE_BUILTIN</term>
+ <listitem><para>not used?</para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_TYPE_CLOCK_C</term>
+ <listitem><para>not used?</para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_TYPE_CRTC_C</term>
+ <listitem><para>not used?</para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>
+ DRM_MODE_TYPE_PREFERRED - The preferred mode for the connector
+ </term>
+ <listitem>
+ <para>not used?</para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_TYPE_DEFAULT</term>
+ <listitem><para>not used?</para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_TYPE_USERDEF</term>
+ <listitem><para>not used?</para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_TYPE_DRIVER</term>
+ <listitem>
+ <para>
+ The mode has been created by the driver (as opposed to
+ to user-created modes).
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ Drivers must set the DRM_MODE_TYPE_DRIVER bit for all modes they
+ create, and set the DRM_MODE_TYPE_PREFERRED bit for the preferred
+ mode.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>__u32 clock;</synopsis>
+ <para>Pixel clock frequency in kHz unit</para>
+ </listitem>
+ <listitem>
+ <synopsis>__u16 hdisplay, hsync_start, hsync_end, htotal;
+ __u16 vdisplay, vsync_start, vsync_end, vtotal;</synopsis>
+ <para>Horizontal and vertical timing information</para>
+ <screen><![CDATA[
+ Active Front Sync Back
+ Region Porch Porch
+ <-----------------------><----------------><-------------><-------------->
+
+ //////////////////////|
+ ////////////////////// |
+ ////////////////////// |.................. ................
+ _______________
+
+ <----- [hv]display ----->
+ <------------- [hv]sync_start ------------>
+ <--------------------- [hv]sync_end --------------------->
+ <-------------------------------- [hv]total ----------------------------->
+]]></screen>
+ </listitem>
+ <listitem>
+ <synopsis>__u16 hskew;
+ __u16 vscan;</synopsis>
+ <para>Unknown</para>
+ </listitem>
+ <listitem>
+ <synopsis>__u32 flags;</synopsis>
+ <para>
+ Mode flags, a combination of
+ <variablelist>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_PHSYNC</term>
+ <listitem><para>
+ Horizontal sync is active high
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_NHSYNC</term>
+ <listitem><para>
+ Horizontal sync is active low
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_PVSYNC</term>
+ <listitem><para>
+ Vertical sync is active high
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_NVSYNC</term>
+ <listitem><para>
+ Vertical sync is active low
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_INTERLACE</term>
+ <listitem><para>
+ Mode is interlaced
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_DBLSCAN</term>
+ <listitem><para>
+ Mode uses doublescan
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_CSYNC</term>
+ <listitem><para>
+ Mode uses composite sync
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_PCSYNC</term>
+ <listitem><para>
+ Composite sync is active high
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_NCSYNC</term>
+ <listitem><para>
+ Composite sync is active low
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_HSKEW</term>
+ <listitem><para>
+ hskew provided (not used?)
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_BCAST</term>
+ <listitem><para>
+ not used?
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_PIXMUX</term>
+ <listitem><para>
+ not used?
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_DBLCLK</term>
+ <listitem><para>
+ not used?
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_FLAG_CLKDIV2</term>
+ <listitem><para>
+ ?
+ </para></listitem>
+ </varlistentry>
+ </variablelist>
+ </para>
+ <para>
+ Note that modes marked with the INTERLACE or DBLSCAN flags will be
+ filtered out by
+ <function>drm_helper_probe_single_connector_modes</function> if
+ the connector's <structfield>interlace_allowed</structfield> or
+ <structfield>doublescan_allowed</structfield> field is set to 0.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>char name[DRM_DISPLAY_MODE_LEN];</synopsis>
+ <para>
+ Mode name. The driver must call
+ <function>drm_mode_set_name</function> to fill the mode name from
+ <structfield>hdisplay</structfield>,
+ <structfield>vdisplay</structfield> and interlace flag after
+ filling the corresponding fields.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ The <structfield>vrefresh</structfield> value is computed by
+ <function>drm_helper_probe_single_connector_modes</function>.
+ </para>
+ <para>
+ When parsing EDID data, <function>drm_add_edid_modes</function> fills the
+ connector <structfield>display_info</structfield>
+ <structfield>width_mm</structfield> and
+ <structfield>height_mm</structfield> fields. When creating modes
+ manually the <methodname>get_modes</methodname> helper operation must
+ set the <structfield>display_info</structfield>
+ <structfield>width_mm</structfield> and
+ <structfield>height_mm</structfield> fields if they haven't been set
+ already (for instance at initialization time when a fixed-size panel is
+ attached to the connector). The mode <structfield>width_mm</structfield>
+ and <structfield>height_mm</structfield> fields are only used internally
+ during EDID parsing and should not be set when creating modes manually.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>int (*mode_valid)(struct drm_connector *connector,
+ struct drm_display_mode *mode);</synopsis>
+ <para>
+ Verify whether a mode is valid for the connector. Return MODE_OK for
+ supported modes and one of the enum drm_mode_status values (MODE_*)
+ for unsupported modes. This operation is optional.
+ </para>
+ <para>
+ As the mode rejection reason is currently not used beside for
+ immediately removing the unsupported mode, an implementation can
+ return MODE_BAD regardless of the exact reason why the mode is not
+ valid.
+ </para>
+ <note><para>
+ Note that the <methodname>mode_valid</methodname> helper operation is
+ only called for modes detected by the device, and
+ <emphasis>not</emphasis> for modes set by the user through the CRTC
+ <methodname>set_config</methodname> operation.
+ </para></note>
+ </listitem>
+ </itemizedlist>
+ </sect2>
+ <sect2>
+ <title>Atomic Modeset Helper Functions Reference</title>
+ <sect3>
+ <title>Overview</title>
+!Pdrivers/gpu/drm/drm_atomic_helper.c overview
+ </sect3>
+ <sect3>
+ <title>Implementing Asynchronous Atomic Commit</title>
+!Pdrivers/gpu/drm/drm_atomic_helper.c implementing async commit
+ </sect3>
+ <sect3>
+ <title>Atomic State Reset and Initialization</title>
+!Pdrivers/gpu/drm/drm_atomic_helper.c atomic state reset and initialization
+ </sect3>
+!Iinclude/drm/drm_atomic_helper.h
+!Edrivers/gpu/drm/drm_atomic_helper.c
+ </sect2>
+ <sect2>
+ <title>Modeset Helper Functions Reference</title>
+!Iinclude/drm/drm_crtc_helper.h
+!Edrivers/gpu/drm/drm_crtc_helper.c
+!Pdrivers/gpu/drm/drm_crtc_helper.c overview
+ </sect2>
+ <sect2>
+ <title>Output Probing Helper Functions Reference</title>
+!Pdrivers/gpu/drm/drm_probe_helper.c output probing helper overview
+!Edrivers/gpu/drm/drm_probe_helper.c
+ </sect2>
+ <sect2>
+ <title>fbdev Helper Functions Reference</title>
+!Pdrivers/gpu/drm/drm_fb_helper.c fbdev helpers
+!Edrivers/gpu/drm/drm_fb_helper.c
+!Iinclude/drm/drm_fb_helper.h
+ </sect2>
+ <sect2>
+ <title>Display Port Helper Functions Reference</title>
+!Pdrivers/gpu/drm/drm_dp_helper.c dp helpers
+!Iinclude/drm/drm_dp_helper.h
+!Edrivers/gpu/drm/drm_dp_helper.c
+ </sect2>
+ <sect2>
+ <title>Display Port MST Helper Functions Reference</title>
+!Pdrivers/gpu/drm/drm_dp_mst_topology.c dp mst helper
+!Iinclude/drm/drm_dp_mst_helper.h
+!Edrivers/gpu/drm/drm_dp_mst_topology.c
+ </sect2>
+ <sect2>
+ <title>MIPI DSI Helper Functions Reference</title>
+!Pdrivers/gpu/drm/drm_mipi_dsi.c dsi helpers
+!Iinclude/drm/drm_mipi_dsi.h
+!Edrivers/gpu/drm/drm_mipi_dsi.c
+ </sect2>
+ <sect2>
+ <title>EDID Helper Functions Reference</title>
+!Edrivers/gpu/drm/drm_edid.c
+ </sect2>
+ <sect2>
+ <title>Rectangle Utilities Reference</title>
+!Pinclude/drm/drm_rect.h rect utils
+!Iinclude/drm/drm_rect.h
+!Edrivers/gpu/drm/drm_rect.c
+ </sect2>
+ <sect2>
+ <title>Flip-work Helper Reference</title>
+!Pinclude/drm/drm_flip_work.h flip utils
+!Iinclude/drm/drm_flip_work.h
+!Edrivers/gpu/drm/drm_flip_work.c
+ </sect2>
+ <sect2>
+ <title>HDMI Infoframes Helper Reference</title>
+ <para>
+ Strictly speaking this is not a DRM helper library but generally useable
+ by any driver interfacing with HDMI outputs like v4l or alsa drivers.
+ But it nicely fits into the overall topic of mode setting helper
+ libraries and hence is also included here.
+ </para>
+!Iinclude/linux/hdmi.h
+!Edrivers/video/hdmi.c
+ </sect2>
+ <sect2>
+ <title id="drm-kms-planehelpers">Plane Helper Reference</title>
+!Edrivers/gpu/drm/drm_plane_helper.c
+!Pdrivers/gpu/drm/drm_plane_helper.c overview
+ </sect2>
+ <sect2>
+ <title>Tile group</title>
+!Pdrivers/gpu/drm/drm_crtc.c Tile group
+ </sect2>
+ <sect2>
+ <title>Bridges</title>
+ <sect3>
+ <title>Overview</title>
+!Pdrivers/gpu/drm/drm_bridge.c overview
+ </sect3>
+ <sect3>
+ <title>Default bridge callback sequence</title>
+!Pdrivers/gpu/drm/drm_bridge.c bridge callbacks
+ </sect3>
+!Edrivers/gpu/drm/drm_bridge.c
+ </sect2>
+ </sect1>
+
+ <!-- Internals: kms properties -->
+
+ <sect1 id="drm-kms-properties">
+ <title>KMS Properties</title>
+ <para>
+ Drivers may need to expose additional parameters to applications than
+ those described in the previous sections. KMS supports attaching
+ properties to CRTCs, connectors and planes and offers a userspace API to
+ list, get and set the property values.
+ </para>
+ <para>
+ Properties are identified by a name that uniquely defines the property
+ purpose, and store an associated value. For all property types except blob
+ properties the value is a 64-bit unsigned integer.
+ </para>
+ <para>
+ KMS differentiates between properties and property instances. Drivers
+ first create properties and then create and associate individual instances
+ of those properties to objects. A property can be instantiated multiple
+ times and associated with different objects. Values are stored in property
+ instances, and all other property information are stored in the property
+ and shared between all instances of the property.
+ </para>
+ <para>
+ Every property is created with a type that influences how the KMS core
+ handles the property. Supported property types are
+ <variablelist>
+ <varlistentry>
+ <term>DRM_MODE_PROP_RANGE</term>
+ <listitem><para>Range properties report their minimum and maximum
+ admissible values. The KMS core verifies that values set by
+ application fit in that range.</para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_PROP_ENUM</term>
+ <listitem><para>Enumerated properties take a numerical value that
+ ranges from 0 to the number of enumerated values defined by the
+ property minus one, and associate a free-formed string name to each
+ value. Applications can retrieve the list of defined value-name pairs
+ and use the numerical value to get and set property instance values.
+ </para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_PROP_BITMASK</term>
+ <listitem><para>Bitmask properties are enumeration properties that
+ additionally restrict all enumerated values to the 0..63 range.
+ Bitmask property instance values combine one or more of the
+ enumerated bits defined by the property.</para></listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_MODE_PROP_BLOB</term>
+ <listitem><para>Blob properties store a binary blob without any format
+ restriction. The binary blobs are created as KMS standalone objects,
+ and blob property instance values store the ID of their associated
+ blob object.</para>
+ <para>Blob properties are only used for the connector EDID property
+ and cannot be created by drivers.</para></listitem>
+ </varlistentry>
+ </variablelist>
+ </para>
+ <para>
+ To create a property drivers call one of the following functions depending
+ on the property type. All property creation functions take property flags
+ and name, as well as type-specific arguments.
+ <itemizedlist>
+ <listitem>
+ <synopsis>struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
+ const char *name,
+ uint64_t min, uint64_t max);</synopsis>
+ <para>Create a range property with the given minimum and maximum
+ values.</para>
+ </listitem>
+ <listitem>
+ <synopsis>struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
+ const char *name,
+ const struct drm_prop_enum_list *props,
+ int num_values);</synopsis>
+ <para>Create an enumerated property. The <parameter>props</parameter>
+ argument points to an array of <parameter>num_values</parameter>
+ value-name pairs.</para>
+ </listitem>
+ <listitem>
+ <synopsis>struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
+ int flags, const char *name,
+ const struct drm_prop_enum_list *props,
+ int num_values);</synopsis>
+ <para>Create a bitmask property. The <parameter>props</parameter>
+ argument points to an array of <parameter>num_values</parameter>
+ value-name pairs.</para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ Properties can additionally be created as immutable, in which case they
+ will be read-only for applications but can be modified by the driver. To
+ create an immutable property drivers must set the DRM_MODE_PROP_IMMUTABLE
+ flag at property creation time.
+ </para>
+ <para>
+ When no array of value-name pairs is readily available at property
+ creation time for enumerated or range properties, drivers can create
+ the property using the <function>drm_property_create</function> function
+ and manually add enumeration value-name pairs by calling the
+ <function>drm_property_add_enum</function> function. Care must be taken to
+ properly specify the property type through the <parameter>flags</parameter>
+ argument.
+ </para>
+ <para>
+ After creating properties drivers can attach property instances to CRTC,
+ connector and plane objects by calling the
+ <function>drm_object_attach_property</function>. The function takes a
+ pointer to the target object, a pointer to the previously created property
+ and an initial instance value.
+ </para>
+ <sect2>
+ <title>Existing KMS Properties</title>
+ <para>
+ The following table gives description of drm properties exposed by various
+ modules/drivers.
+ </para>
+ <table border="1" cellpadding="0" cellspacing="0">
+ <tbody>
+ <tr style="font-weight: bold;">
+ <td valign="top" >Owner Module/Drivers</td>
+ <td valign="top" >Group</td>
+ <td valign="top" >Property Name</td>
+ <td valign="top" >Type</td>
+ <td valign="top" >Property Values</td>
+ <td valign="top" >Object attached</td>
+ <td valign="top" >Description/Restrictions</td>
+ </tr>
+ <tr>
+ <td rowspan="37" valign="top" >DRM</td>
+ <td valign="top" >Generic</td>
+ <td valign="top" >“rotation”</td>
+ <td valign="top" >BITMASK</td>
+ <td valign="top" >{ 0, "rotate-0" },
+ { 1, "rotate-90" },
+ { 2, "rotate-180" },
+ { 3, "rotate-270" },
+ { 4, "reflect-x" },
+ { 5, "reflect-y" }</td>
+ <td valign="top" >CRTC, Plane</td>
+ <td valign="top" >rotate-(degrees) rotates the image by the specified amount in degrees
+ in counter clockwise direction. reflect-x and reflect-y reflects the
+ image along the specified axis prior to rotation</td>
+ </tr>
+ <tr>
+ <td rowspan="5" valign="top" >Connector</td>
+ <td valign="top" >“EDID”</td>
+ <td valign="top" >BLOB | IMMUTABLE</td>
+ <td valign="top" >0</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >Contains id of edid blob ptr object.</td>
+ </tr>
+ <tr>
+ <td valign="top" >“DPMS”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ “On”, “Standby”, “Suspend”, “Off” }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >Contains DPMS operation mode value.</td>
+ </tr>
+ <tr>
+ <td valign="top" >“PATH”</td>
+ <td valign="top" >BLOB | IMMUTABLE</td>
+ <td valign="top" >0</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >Contains topology path to a connector.</td>
+ </tr>
+ <tr>
+ <td valign="top" >“TILE”</td>
+ <td valign="top" >BLOB | IMMUTABLE</td>
+ <td valign="top" >0</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >Contains tiling information for a connector.</td>
+ </tr>
+ <tr>
+ <td valign="top" >“CRTC_ID”</td>
+ <td valign="top" >OBJECT</td>
+ <td valign="top" >DRM_MODE_OBJECT_CRTC</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >CRTC that connector is attached to (atomic)</td>
+ </tr>
+ <tr>
+ <td rowspan="11" valign="top" >Plane</td>
+ <td valign="top" >“type”</td>
+ <td valign="top" >ENUM | IMMUTABLE</td>
+ <td valign="top" >{ "Overlay", "Primary", "Cursor" }</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Plane type</td>
+ </tr>
+ <tr>
+ <td valign="top" >“SRC_X”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=UINT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout source x coordinate in 16.16 fixed point (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“SRC_Y”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=UINT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout source y coordinate in 16.16 fixed point (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“SRC_W”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=UINT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout source width in 16.16 fixed point (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“SRC_H”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=UINT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout source height in 16.16 fixed point (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“CRTC_X”</td>
+ <td valign="top" >SIGNED_RANGE</td>
+ <td valign="top" >Min=INT_MIN, Max=INT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout CRTC (destination) x coordinate (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“CRTC_Y”</td>
+ <td valign="top" >SIGNED_RANGE</td>
+ <td valign="top" >Min=INT_MIN, Max=INT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout CRTC (destination) y coordinate (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“CRTC_W”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=UINT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout CRTC (destination) width (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“CRTC_H”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=UINT_MAX</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout CRTC (destination) height (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“FB_ID”</td>
+ <td valign="top" >OBJECT</td>
+ <td valign="top" >DRM_MODE_OBJECT_FB</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >Scanout framebuffer (atomic)</td>
+ </tr>
+ <tr>
+ <td valign="top" >“CRTC_ID”</td>
+ <td valign="top" >OBJECT</td>
+ <td valign="top" >DRM_MODE_OBJECT_CRTC</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >CRTC that plane is attached to (atomic)</td>
+ </tr>
+ <tr>
+ <td rowspan="2" valign="top" >DVI-I</td>
+ <td valign="top" >“subconnector”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ “Unknown”, “DVI-D”, “DVI-A” }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“select subconnector”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ “Automatic”, “DVI-D”, “DVI-A” }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="13" valign="top" >TV</td>
+ <td valign="top" >“subconnector”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "Unknown", "Composite", "SVIDEO", "Component", "SCART" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“select subconnector”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "Automatic", "Composite", "SVIDEO", "Component", "SCART" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“mode”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "NTSC_M", "NTSC_J", "NTSC_443", "PAL_B" } etc.</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“left margin”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“right margin”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“top margin”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“bottom margin”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“brightness”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“contrast”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“flicker reduction”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“overscan”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“saturation”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“hue”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="2" valign="top" >Virtual GPU</td>
+ <td valign="top" >“suggested X”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0xffffffff</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >property to suggest an X offset for a connector</td>
+ </tr>
+ <tr>
+ <td valign="top" >“suggested Y”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0xffffffff</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >property to suggest an Y offset for a connector</td>
+ </tr>
+ <tr>
+ <td rowspan="3" valign="top" >Optional</td>
+ <td valign="top" >“scaling mode”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "None", "Full", "Center", "Full aspect" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"aspect ratio"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "None", "4:3", "16:9" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >DRM property to set aspect ratio from user space app.
+ This enum is made generic to allow addition of custom aspect
+ ratios.</td>
+ </tr>
+ <tr>
+ <td valign="top" >“dirty”</td>
+ <td valign="top" >ENUM | IMMUTABLE</td>
+ <td valign="top" >{ "Off", "On", "Annotate" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="20" valign="top" >i915</td>
+ <td rowspan="2" valign="top" >Generic</td>
+ <td valign="top" >"Broadcast RGB"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "Automatic", "Full", "Limited 16:235" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“audio”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "force-dvi", "off", "auto", "on" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="17" valign="top" >SDVO-TV</td>
+ <td valign="top" >“mode”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "NTSC_M", "NTSC_J", "NTSC_443", "PAL_B" } etc.</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"left_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"right_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"top_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"bottom_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“hpos”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“vpos”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“contrast”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“saturation”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“hue”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“sharpness”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“flicker_filter”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“flicker_filter_adaptive”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“flicker_filter_2d”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“tv_chroma_filter”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“tv_luma_filter”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“dot_crawl”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >SDVO-TV/LVDS</td>
+ <td valign="top" >“brightness”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="2" valign="top" >CDV gma-500</td>
+ <td rowspan="2" valign="top" >Generic</td>
+ <td valign="top" >"Broadcast RGB"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ “Full”, “Limited 16:235” }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"Broadcast RGB"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ “off”, “auto”, “on” }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="19" valign="top" >Poulsbo</td>
+ <td rowspan="1" valign="top" >Generic</td>
+ <td valign="top" >“backlight”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=100</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="17" valign="top" >SDVO-TV</td>
+ <td valign="top" >“mode”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "NTSC_M", "NTSC_J", "NTSC_443", "PAL_B" } etc.</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"left_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"right_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"top_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"bottom_margin"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“hpos”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“vpos”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“contrast”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“saturation”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“hue”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“sharpness”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“flicker_filter”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“flicker_filter_adaptive”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“flicker_filter_2d”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“tv_chroma_filter”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“tv_luma_filter”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“dot_crawl”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >SDVO-TV/LVDS</td>
+ <td valign="top" >“brightness”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max= SDVO dependent</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="11" valign="top" >armada</td>
+ <td rowspan="2" valign="top" >CRTC</td>
+ <td valign="top" >"CSC_YUV"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "Auto" , "CCIR601", "CCIR709" }</td>
+ <td valign="top" >CRTC</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"CSC_RGB"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "Auto", "Computer system", "Studio" }</td>
+ <td valign="top" >CRTC</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="9" valign="top" >Overlay</td>
+ <td valign="top" >"colorkey"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0xffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"colorkey_min"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0xffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"colorkey_max"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0xffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"colorkey_val"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0xffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"colorkey_alpha"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0xffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"colorkey_mode"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "disabled", "Y component", "U component"
+ , "V component", "RGB", “R component", "G component", "B component" }</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"brightness"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=256 + 255</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"contrast"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0x7fff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"saturation"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0x7fff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="2" valign="top" >exynos</td>
+ <td valign="top" >CRTC</td>
+ <td valign="top" >“mode”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "normal", "blank" }</td>
+ <td valign="top" >CRTC</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >Overlay</td>
+ <td valign="top" >“zpos”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=MAX_PLANE-1</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="2" valign="top" >i2c/ch7006_drv</td>
+ <td valign="top" >Generic</td>
+ <td valign="top" >“scale”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=2</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="1" valign="top" >TV</td>
+ <td valign="top" >“mode”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "PAL", "PAL-M","PAL-N"}, ”PAL-Nc"
+ , "PAL-60", "NTSC-M", "NTSC-J" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="15" valign="top" >nouveau</td>
+ <td rowspan="6" valign="top" >NV10 Overlay</td>
+ <td valign="top" >"colorkey"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0x01ffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“contrast”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=8192-1</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“brightness”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1024</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“hue”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=359</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“saturation”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=8192-1</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“iturbt_709”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="2" valign="top" >Nv04 Overlay</td>
+ <td valign="top" >“colorkey”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0x01ffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“brightness”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1024</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="7" valign="top" >Display</td>
+ <td valign="top" >“dithering mode”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "auto", "off", "on" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“dithering depth”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "auto", "off", "on", "static 2x2", "dynamic 2x2", "temporal" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“underscan”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "auto", "6 bpc", "8 bpc" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“underscan hborder”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=128</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“underscan vborder”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=128</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“vibrant hue”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=180</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >“color vibrance”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=200</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >omap</td>
+ <td valign="top" >Generic</td>
+ <td valign="top" >“zorder”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=3</td>
+ <td valign="top" >CRTC, Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >qxl</td>
+ <td valign="top" >Generic</td>
+ <td valign="top" >“hotplug_mode_update"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="9" valign="top" >radeon</td>
+ <td valign="top" >DVI-I</td>
+ <td valign="top" >“coherent”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >DAC enable load detect</td>
+ <td valign="top" >“load detection”</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=1</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >TV Standard</td>
+ <td valign="top" >"tv standard"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "ntsc", "pal", "pal-m", "pal-60", "ntsc-j"
+ , "scart-pal", "pal-cn", "secam" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >legacy TMDS PLL detect</td>
+ <td valign="top" >"tmds_pll"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "driver", "bios" }</td>
+ <td valign="top" >-</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="3" valign="top" >Underscan</td>
+ <td valign="top" >"underscan"</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "off", "on", "auto" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"underscan hborder"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=128</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"underscan vborder"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=128</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >Audio</td>
+ <td valign="top" >“audio”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "off", "on", "auto" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >FMT Dithering</td>
+ <td valign="top" >“dither”</td>
+ <td valign="top" >ENUM</td>
+ <td valign="top" >{ "off", "on" }</td>
+ <td valign="top" >Connector</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td rowspan="3" valign="top" >rcar-du</td>
+ <td rowspan="3" valign="top" >Generic</td>
+ <td valign="top" >"alpha"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=255</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"colorkey"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=0, Max=0x01ffffff</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ <tr>
+ <td valign="top" >"zpos"</td>
+ <td valign="top" >RANGE</td>
+ <td valign="top" >Min=1, Max=7</td>
+ <td valign="top" >Plane</td>
+ <td valign="top" >TBD</td>
+ </tr>
+ </tbody>
+ </table>
+ </sect2>
+ </sect1>
+
+ <!-- Internals: vertical blanking -->
+
+ <sect1 id="drm-vertical-blank">
+ <title>Vertical Blanking</title>
+ <para>
+ Vertical blanking plays a major role in graphics rendering. To achieve
+ tear-free display, users must synchronize page flips and/or rendering to
+ vertical blanking. The DRM API offers ioctls to perform page flips
+ synchronized to vertical blanking and wait for vertical blanking.
+ </para>
+ <para>
+ The DRM core handles most of the vertical blanking management logic, which
+ involves filtering out spurious interrupts, keeping race-free blanking
+ counters, coping with counter wrap-around and resets and keeping use
+ counts. It relies on the driver to generate vertical blanking interrupts
+ and optionally provide a hardware vertical blanking counter. Drivers must
+ implement the following operations.
+ </para>
+ <itemizedlist>
+ <listitem>
+ <synopsis>int (*enable_vblank) (struct drm_device *dev, int crtc);
+void (*disable_vblank) (struct drm_device *dev, int crtc);</synopsis>
+ <para>
+ Enable or disable vertical blanking interrupts for the given CRTC.
+ </para>
+ </listitem>
+ <listitem>
+ <synopsis>u32 (*get_vblank_counter) (struct drm_device *dev, int crtc);</synopsis>
+ <para>
+ Retrieve the value of the vertical blanking counter for the given
+ CRTC. If the hardware maintains a vertical blanking counter its value
+ should be returned. Otherwise drivers can use the
+ <function>drm_vblank_count</function> helper function to handle this
+ operation.
+ </para>
+ </listitem>
+ </itemizedlist>
+ <para>
+ Drivers must initialize the vertical blanking handling core with a call to
+ <function>drm_vblank_init</function> in their
+ <methodname>load</methodname> operation. The function will set the struct
+ <structname>drm_device</structname>
+ <structfield>vblank_disable_allowed</structfield> field to 0. This will
+ keep vertical blanking interrupts enabled permanently until the first mode
+ set operation, where <structfield>vblank_disable_allowed</structfield> is
+ set to 1. The reason behind this is not clear. Drivers can set the field
+ to 1 after <function>calling drm_vblank_init</function> to make vertical
+ blanking interrupts dynamically managed from the beginning.
+ </para>
+ <para>
+ Vertical blanking interrupts can be enabled by the DRM core or by drivers
+ themselves (for instance to handle page flipping operations). The DRM core
+ maintains a vertical blanking use count to ensure that the interrupts are
+ not disabled while a user still needs them. To increment the use count,
+ drivers call <function>drm_vblank_get</function>. Upon return vertical
+ blanking interrupts are guaranteed to be enabled.
+ </para>
+ <para>
+ To decrement the use count drivers call
+ <function>drm_vblank_put</function>. Only when the use count drops to zero
+ will the DRM core disable the vertical blanking interrupts after a delay
+ by scheduling a timer. The delay is accessible through the vblankoffdelay
+ module parameter or the <varname>drm_vblank_offdelay</varname> global
+ variable and expressed in milliseconds. Its default value is 5000 ms.
+ Zero means never disable, and a negative value means disable immediately.
+ Drivers may override the behaviour by setting the
+ <structname>drm_device</structname>
+ <structfield>vblank_disable_immediate</structfield> flag, which when set
+ causes vblank interrupts to be disabled immediately regardless of the
+ drm_vblank_offdelay value. The flag should only be set if there's a
+ properly working hardware vblank counter present.
+ </para>
+ <para>
+ When a vertical blanking interrupt occurs drivers only need to call the
+ <function>drm_handle_vblank</function> function to account for the
+ interrupt.
+ </para>
+ <para>
+ Resources allocated by <function>drm_vblank_init</function> must be freed
+ with a call to <function>drm_vblank_cleanup</function> in the driver
+ <methodname>unload</methodname> operation handler.
+ </para>
+ <sect2>
+ <title>Vertical Blanking and Interrupt Handling Functions Reference</title>
+!Edrivers/gpu/drm/drm_irq.c
+!Finclude/drm/drmP.h drm_crtc_vblank_waitqueue
+ </sect2>
+ </sect1>
+
+ <!-- Internals: open/close, file operations and ioctls -->
+
+ <sect1>
+ <title>Open/Close, File Operations and IOCTLs</title>
+ <sect2>
+ <title>Open and Close</title>
+ <synopsis>int (*firstopen) (struct drm_device *);
+void (*lastclose) (struct drm_device *);
+int (*open) (struct drm_device *, struct drm_file *);
+void (*preclose) (struct drm_device *, struct drm_file *);
+void (*postclose) (struct drm_device *, struct drm_file *);</synopsis>
+ <abstract>Open and close handlers. None of those methods are mandatory.
+ </abstract>
+ <para>
+ The <methodname>firstopen</methodname> method is called by the DRM core
+ for legacy UMS (User Mode Setting) drivers only when an application
+ opens a device that has no other opened file handle. UMS drivers can
+ implement it to acquire device resources. KMS drivers can't use the
+ method and must acquire resources in the <methodname>load</methodname>
+ method instead.
+ </para>
+ <para>
+ Similarly the <methodname>lastclose</methodname> method is called when
+ the last application holding a file handle opened on the device closes
+ it, for both UMS and KMS drivers. Additionally, the method is also
+ called at module unload time or, for hot-pluggable devices, when the
+ device is unplugged. The <methodname>firstopen</methodname> and
+ <methodname>lastclose</methodname> calls can thus be unbalanced.
+ </para>
+ <para>
+ The <methodname>open</methodname> method is called every time the device
+ is opened by an application. Drivers can allocate per-file private data
+ in this method and store them in the struct
+ <structname>drm_file</structname> <structfield>driver_priv</structfield>
+ field. Note that the <methodname>open</methodname> method is called
+ before <methodname>firstopen</methodname>.
+ </para>
+ <para>
+ The close operation is split into <methodname>preclose</methodname> and
+ <methodname>postclose</methodname> methods. Drivers must stop and
+ cleanup all per-file operations in the <methodname>preclose</methodname>
+ method. For instance pending vertical blanking and page flip events must
+ be cancelled. No per-file operation is allowed on the file handle after
+ returning from the <methodname>preclose</methodname> method.
+ </para>
+ <para>
+ Finally the <methodname>postclose</methodname> method is called as the
+ last step of the close operation, right before calling the
+ <methodname>lastclose</methodname> method if no other open file handle
+ exists for the device. Drivers that have allocated per-file private data
+ in the <methodname>open</methodname> method should free it here.
+ </para>
+ <para>
+ The <methodname>lastclose</methodname> method should restore CRTC and
+ plane properties to default value, so that a subsequent open of the
+ device will not inherit state from the previous user. It can also be
+ used to execute delayed power switching state changes, e.g. in
+ conjunction with the vga_switcheroo infrastructure (see
+ <xref linkend="vga_switcheroo"/>). Beyond that KMS drivers should not
+ do any further cleanup. Only legacy UMS drivers might need to clean up
+ device state so that the vga console or an independent fbdev driver
+ could take over.
+ </para>
+ </sect2>
+ <sect2>
+ <title>File Operations</title>
+ <synopsis>const struct file_operations *fops</synopsis>
+ <abstract>File operations for the DRM device node.</abstract>
+ <para>
+ Drivers must define the file operations structure that forms the DRM
+ userspace API entry point, even though most of those operations are
+ implemented in the DRM core. The <methodname>open</methodname>,
+ <methodname>release</methodname> and <methodname>ioctl</methodname>
+ operations are handled by
+ <programlisting>
+ .owner = THIS_MODULE,
+ .open = drm_open,
+ .release = drm_release,
+ .unlocked_ioctl = drm_ioctl,
+ #ifdef CONFIG_COMPAT
+ .compat_ioctl = drm_compat_ioctl,
+ #endif
+ </programlisting>
+ </para>
+ <para>
+ Drivers that implement private ioctls that requires 32/64bit
+ compatibility support must provide their own
+ <methodname>compat_ioctl</methodname> handler that processes private
+ ioctls and calls <function>drm_compat_ioctl</function> for core ioctls.
+ </para>
+ <para>
+ The <methodname>read</methodname> and <methodname>poll</methodname>
+ operations provide support for reading DRM events and polling them. They
+ are implemented by
+ <programlisting>
+ .poll = drm_poll,
+ .read = drm_read,
+ .llseek = no_llseek,
+ </programlisting>
+ </para>
+ <para>
+ The memory mapping implementation varies depending on how the driver
+ manages memory. Pre-GEM drivers will use <function>drm_mmap</function>,
+ while GEM-aware drivers will use <function>drm_gem_mmap</function>. See
+ <xref linkend="drm-gem"/>.
+ <programlisting>
+ .mmap = drm_gem_mmap,
+ </programlisting>
+ </para>
+ <para>
+ No other file operation is supported by the DRM API.
+ </para>
+ </sect2>
+ <sect2>
+ <title>IOCTLs</title>
+ <synopsis>struct drm_ioctl_desc *ioctls;
+int num_ioctls;</synopsis>
+ <abstract>Driver-specific ioctls descriptors table.</abstract>
+ <para>
+ Driver-specific ioctls numbers start at DRM_COMMAND_BASE. The ioctls
+ descriptors table is indexed by the ioctl number offset from the base
+ value. Drivers can use the DRM_IOCTL_DEF_DRV() macro to initialize the
+ table entries.
+ </para>
+ <para>
+ <programlisting>DRM_IOCTL_DEF_DRV(ioctl, func, flags)</programlisting>
+ <para>
+ <parameter>ioctl</parameter> is the ioctl name. Drivers must define
+ the DRM_##ioctl and DRM_IOCTL_##ioctl macros to the ioctl number
+ offset from DRM_COMMAND_BASE and the ioctl number respectively. The
+ first macro is private to the device while the second must be exposed
+ to userspace in a public header.
+ </para>
+ <para>
+ <parameter>func</parameter> is a pointer to the ioctl handler function
+ compatible with the <type>drm_ioctl_t</type> type.
+ <programlisting>typedef int drm_ioctl_t(struct drm_device *dev, void *data,
+ struct drm_file *file_priv);</programlisting>
+ </para>
+ <para>
+ <parameter>flags</parameter> is a bitmask combination of the following
+ values. It restricts how the ioctl is allowed to be called.
+ <itemizedlist>
+ <listitem><para>
+ DRM_AUTH - Only authenticated callers allowed
+ </para></listitem>
+ <listitem><para>
+ DRM_MASTER - The ioctl can only be called on the master file
+ handle
+ </para></listitem>
+ <listitem><para>
+ DRM_ROOT_ONLY - Only callers with the SYSADMIN capability allowed
+ </para></listitem>
+ <listitem><para>
+ DRM_CONTROL_ALLOW - The ioctl can only be called on a control
+ device
+ </para></listitem>
+ <listitem><para>
+ DRM_UNLOCKED - The ioctl handler will be called without locking
+ the DRM global mutex. This is the enforced default for kms drivers
+ (i.e. using the DRIVER_MODESET flag) and hence shouldn't be used
+ any more for new drivers.
+ </para></listitem>
+ </itemizedlist>
+ </para>
+ </para>
+!Edrivers/gpu/drm/drm_ioctl.c
+ </sect2>
+ </sect1>
+ <sect1>
+ <title>Legacy Support Code</title>
+ <para>
+ The section very briefly covers some of the old legacy support code which
+ is only used by old DRM drivers which have done a so-called shadow-attach
+ to the underlying device instead of registering as a real driver. This
+ also includes some of the old generic buffer management and command
+ submission code. Do not use any of this in new and modern drivers.
+ </para>
+
+ <sect2>
+ <title>Legacy Suspend/Resume</title>
+ <para>
+ The DRM core provides some suspend/resume code, but drivers wanting full
+ suspend/resume support should provide save() and restore() functions.
+ These are called at suspend, hibernate, or resume time, and should perform
+ any state save or restore required by your device across suspend or
+ hibernate states.
+ </para>
+ <synopsis>int (*suspend) (struct drm_device *, pm_message_t state);
+ int (*resume) (struct drm_device *);</synopsis>
+ <para>
+ Those are legacy suspend and resume methods which
+ <emphasis>only</emphasis> work with the legacy shadow-attach driver
+ registration functions. New driver should use the power management
+ interface provided by their bus type (usually through
+ the struct <structname>device_driver</structname> dev_pm_ops) and set
+ these methods to NULL.
+ </para>
+ </sect2>
+
+ <sect2>
+ <title>Legacy DMA Services</title>
+ <para>
+ This should cover how DMA mapping etc. is supported by the core.
+ These functions are deprecated and should not be used.
+ </para>
+ </sect2>
+ </sect1>
+ </chapter>
+
+<!-- TODO
+
+- Add a glossary
+- Document the struct_mutex catch-all lock
+- Document connector properties
+
+- Why is the load method optional?
+- What are drivers supposed to set the initial display state to, and how?
+ Connector's DPMS states are not initialized and are thus equal to
+ DRM_MODE_DPMS_ON. The fbcon compatibility layer calls
+ drm_helper_disable_unused_functions(), which disables unused encoders and
+ CRTCs, but doesn't touch the connectors' DPMS state, and
+ drm_helper_connector_dpms() in reaction to fbdev blanking events. Do drivers
+ that don't implement (or just don't use) fbcon compatibility need to call
+ those functions themselves?
+- KMS drivers must call drm_vblank_pre_modeset() and drm_vblank_post_modeset()
+ around mode setting. Should this be done in the DRM core?
+- vblank_disable_allowed is set to 1 in the first drm_vblank_post_modeset()
+ call and never set back to 0. It seems to be safe to permanently set it to 1
+ in drm_vblank_init() for KMS driver, and it might be safe for UMS drivers as
+ well. This should be investigated.
+- crtc and connector .save and .restore operations are only used internally in
+ drivers, should they be removed from the core?
+- encoder mid-layer .save and .restore operations are only used internally in
+ drivers, should they be removed from the core?
+- encoder mid-layer .detect operation is only used internally in drivers,
+ should it be removed from the core?
+-->
+
+ <!-- External interfaces -->
+
+ <chapter id="drmExternals">
+ <title>Userland interfaces</title>
+ <para>
+ The DRM core exports several interfaces to applications,
+ generally intended to be used through corresponding libdrm
+ wrapper functions. In addition, drivers export device-specific
+ interfaces for use by userspace drivers & device-aware
+ applications through ioctls and sysfs files.
+ </para>
+ <para>
+ External interfaces include: memory mapping, context management,
+ DMA operations, AGP management, vblank control, fence
+ management, memory management, and output management.
+ </para>
+ <para>
+ Cover generic ioctls and sysfs layout here. We only need high-level
+ info, since man pages should cover the rest.
+ </para>
+
+ <!-- External: render nodes -->
+
+ <sect1>
+ <title>Render nodes</title>
+ <para>
+ DRM core provides multiple character-devices for user-space to use.
+ Depending on which device is opened, user-space can perform a different
+ set of operations (mainly ioctls). The primary node is always created
+ and called card<num>. Additionally, a currently
+ unused control node, called controlD<num> is also
+ created. The primary node provides all legacy operations and
+ historically was the only interface used by userspace. With KMS, the
+ control node was introduced. However, the planned KMS control interface
+ has never been written and so the control node stays unused to date.
+ </para>
+ <para>
+ With the increased use of offscreen renderers and GPGPU applications,
+ clients no longer require running compositors or graphics servers to
+ make use of a GPU. But the DRM API required unprivileged clients to
+ authenticate to a DRM-Master prior to getting GPU access. To avoid this
+ step and to grant clients GPU access without authenticating, render
+ nodes were introduced. Render nodes solely serve render clients, that
+ is, no modesetting or privileged ioctls can be issued on render nodes.
+ Only non-global rendering commands are allowed. If a driver supports
+ render nodes, it must advertise it via the DRIVER_RENDER
+ DRM driver capability. If not supported, the primary node must be used
+ for render clients together with the legacy drmAuth authentication
+ procedure.
+ </para>
+ <para>
+ If a driver advertises render node support, DRM core will create a
+ separate render node called renderD<num>. There will
+ be one render node per device. No ioctls except PRIME-related ioctls
+ will be allowed on this node. Especially GEM_OPEN will be
+ explicitly prohibited. Render nodes are designed to avoid the
+ buffer-leaks, which occur if clients guess the flink names or mmap
+ offsets on the legacy interface. Additionally to this basic interface,
+ drivers must mark their driver-dependent render-only ioctls as
+ DRM_RENDER_ALLOW so render clients can use them. Driver
+ authors must be careful not to allow any privileged ioctls on render
+ nodes.
+ </para>
+ <para>
+ With render nodes, user-space can now control access to the render node
+ via basic file-system access-modes. A running graphics server which
+ authenticates clients on the privileged primary/legacy node is no longer
+ required. Instead, a client can open the render node and is immediately
+ granted GPU access. Communication between clients (or servers) is done
+ via PRIME. FLINK from render node to legacy node is not supported. New
+ clients must not use the insecure FLINK interface.
+ </para>
+ <para>
+ Besides dropping all modeset/global ioctls, render nodes also drop the
+ DRM-Master concept. There is no reason to associate render clients with
+ a DRM-Master as they are independent of any graphics server. Besides,
+ they must work without any running master, anyway.
+ Drivers must be able to run without a master object if they support
+ render nodes. If, on the other hand, a driver requires shared state
+ between clients which is visible to user-space and accessible beyond
+ open-file boundaries, they cannot support render nodes.
+ </para>
+ </sect1>
+
+ <!-- External: vblank handling -->
+
+ <sect1>
+ <title>VBlank event handling</title>
+ <para>
+ The DRM core exposes two vertical blank related ioctls:
+ <variablelist>
+ <varlistentry>
+ <term>DRM_IOCTL_WAIT_VBLANK</term>
+ <listitem>
+ <para>
+ This takes a struct drm_wait_vblank structure as its argument,
+ and it is used to block or request a signal when a specified
+ vblank event occurs.
+ </para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>DRM_IOCTL_MODESET_CTL</term>
+ <listitem>
+ <para>
+ This was only used for user-mode-settind drivers around
+ modesetting changes to allow the kernel to update the vblank
+ interrupt after mode setting, since on many devices the vertical
+ blank counter is reset to 0 at some point during modeset. Modern
+ drivers should not call this any more since with kernel mode
+ setting it is a no-op.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </para>
+ </sect1>
+
+ </chapter>
+</part>
+<part id="drmDrivers">
+ <title>DRM Drivers</title>
+
+ <partintro>
+ <para>
+ This second part of the GPU Driver Developer's Guide documents driver
+ code, implementation details and also all the driver-specific userspace
+ interfaces. Especially since all hardware-acceleration interfaces to
+ userspace are driver specific for efficiency and other reasons these
+ interfaces can be rather substantial. Hence every driver has its own
+ chapter.
+ </para>
+ </partintro>
+
+ <chapter id="drmI915">
+ <title>drm/i915 Intel GFX Driver</title>
+ <para>
+ The drm/i915 driver supports all (with the exception of some very early
+ models) integrated GFX chipsets with both Intel display and rendering
+ blocks. This excludes a set of SoC platforms with an SGX rendering unit,
+ those have basic support through the gma500 drm driver.
+ </para>
+ <sect1>
+ <title>Core Driver Infrastructure</title>
+ <para>
+ This section covers core driver infrastructure used by both the display
+ and the GEM parts of the driver.
+ </para>
+ <sect2>
+ <title>Runtime Power Management</title>
+!Pdrivers/gpu/drm/i915/intel_runtime_pm.c runtime pm
+!Idrivers/gpu/drm/i915/intel_runtime_pm.c
+!Idrivers/gpu/drm/i915/intel_uncore.c
+ </sect2>
+ <sect2>
+ <title>Interrupt Handling</title>
+!Pdrivers/gpu/drm/i915/i915_irq.c interrupt handling
+!Fdrivers/gpu/drm/i915/i915_irq.c intel_irq_init intel_irq_init_hw intel_hpd_init
+!Fdrivers/gpu/drm/i915/i915_irq.c intel_runtime_pm_disable_interrupts
+!Fdrivers/gpu/drm/i915/i915_irq.c intel_runtime_pm_enable_interrupts
+ </sect2>
+ <sect2>
+ <title>Intel GVT-g Guest Support(vGPU)</title>
+!Pdrivers/gpu/drm/i915/i915_vgpu.c Intel GVT-g guest support
+!Idrivers/gpu/drm/i915/i915_vgpu.c
+ </sect2>
+ </sect1>
+ <sect1>
+ <title>Display Hardware Handling</title>
+ <para>
+ This section covers everything related to the display hardware including
+ the mode setting infrastructure, plane, sprite and cursor handling and
+ display, output probing and related topics.
+ </para>
+ <sect2>
+ <title>Mode Setting Infrastructure</title>
+ <para>
+ The i915 driver is thus far the only DRM driver which doesn't use the
+ common DRM helper code to implement mode setting sequences. Thus it
+ has its own tailor-made infrastructure for executing a display
+ configuration change.
+ </para>
+ </sect2>
+ <sect2>
+ <title>Frontbuffer Tracking</title>
+!Pdrivers/gpu/drm/i915/intel_frontbuffer.c frontbuffer tracking
+!Idrivers/gpu/drm/i915/intel_frontbuffer.c
+!Fdrivers/gpu/drm/i915/i915_gem.c i915_gem_track_fb
+ </sect2>
+ <sect2>
+ <title>Display FIFO Underrun Reporting</title>
+!Pdrivers/gpu/drm/i915/intel_fifo_underrun.c fifo underrun handling
+!Idrivers/gpu/drm/i915/intel_fifo_underrun.c
+ </sect2>
+ <sect2>
+ <title>Plane Configuration</title>
+ <para>
+ This section covers plane configuration and composition with the
+ primary plane, sprites, cursors and overlays. This includes the
+ infrastructure to do atomic vsync'ed updates of all this state and
+ also tightly coupled topics like watermark setup and computation,
+ framebuffer compression and panel self refresh.
+ </para>
+ </sect2>
+ <sect2>
+ <title>Atomic Plane Helpers</title>
+!Pdrivers/gpu/drm/i915/intel_atomic_plane.c atomic plane helpers
+!Idrivers/gpu/drm/i915/intel_atomic_plane.c
+ </sect2>
+ <sect2>
+ <title>Output Probing</title>
+ <para>
+ This section covers output probing and related infrastructure like the
+ hotplug interrupt storm detection and mitigation code. Note that the
+ i915 driver still uses most of the common DRM helper code for output
+ probing, so those sections fully apply.
+ </para>
+ </sect2>
+ <sect2>
+ <title>Hotplug</title>
+!Pdrivers/gpu/drm/i915/intel_hotplug.c Hotplug
+!Idrivers/gpu/drm/i915/intel_hotplug.c
+ </sect2>
+ <sect2>
+ <title>High Definition Audio</title>
+!Pdrivers/gpu/drm/i915/intel_audio.c High Definition Audio over HDMI and Display Port
+!Idrivers/gpu/drm/i915/intel_audio.c
+!Iinclude/drm/i915_component.h
+ </sect2>
+ <sect2>
+ <title>Panel Self Refresh PSR (PSR/SRD)</title>
+!Pdrivers/gpu/drm/i915/intel_psr.c Panel Self Refresh (PSR/SRD)
+!Idrivers/gpu/drm/i915/intel_psr.c
+ </sect2>
+ <sect2>
+ <title>Frame Buffer Compression (FBC)</title>
+!Pdrivers/gpu/drm/i915/intel_fbc.c Frame Buffer Compression (FBC)
+!Idrivers/gpu/drm/i915/intel_fbc.c
+ </sect2>
+ <sect2>
+ <title>Display Refresh Rate Switching (DRRS)</title>
+!Pdrivers/gpu/drm/i915/intel_dp.c Display Refresh Rate Switching (DRRS)
+!Fdrivers/gpu/drm/i915/intel_dp.c intel_dp_set_drrs_state
+!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_enable
+!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_disable
+!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_invalidate
+!Fdrivers/gpu/drm/i915/intel_dp.c intel_edp_drrs_flush
+!Fdrivers/gpu/drm/i915/intel_dp.c intel_dp_drrs_init
+
+ </sect2>
+ <sect2>
+ <title>DPIO</title>
+!Pdrivers/gpu/drm/i915/i915_reg.h DPIO
+ <table id="dpiox2">
+ <title>Dual channel PHY (VLV/CHV/BXT)</title>
+ <tgroup cols="8">
+ <colspec colname="c0" />
+ <colspec colname="c1" />
+ <colspec colname="c2" />
+ <colspec colname="c3" />
+ <colspec colname="c4" />
+ <colspec colname="c5" />
+ <colspec colname="c6" />
+ <colspec colname="c7" />
+ <spanspec spanname="ch0" namest="c0" nameend="c3" />
+ <spanspec spanname="ch1" namest="c4" nameend="c7" />
+ <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
+ <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
+ <spanspec spanname="ch1pcs01" namest="c4" nameend="c5" />
+ <spanspec spanname="ch1pcs23" namest="c6" nameend="c7" />
+ <thead>
+ <row>
+ <entry spanname="ch0">CH0</entry>
+ <entry spanname="ch1">CH1</entry>
+ </row>
+ </thead>
+ <tbody valign="top" align="center">
+ <row>
+ <entry spanname="ch0">CMN/PLL/REF</entry>
+ <entry spanname="ch1">CMN/PLL/REF</entry>
+ </row>
+ <row>
+ <entry spanname="ch0pcs01">PCS01</entry>
+ <entry spanname="ch0pcs23">PCS23</entry>
+ <entry spanname="ch1pcs01">PCS01</entry>
+ <entry spanname="ch1pcs23">PCS23</entry>
+ </row>
+ <row>
+ <entry>TX0</entry>
+ <entry>TX1</entry>
+ <entry>TX2</entry>
+ <entry>TX3</entry>
+ <entry>TX0</entry>
+ <entry>TX1</entry>
+ <entry>TX2</entry>
+ <entry>TX3</entry>
+ </row>
+ <row>
+ <entry spanname="ch0">DDI0</entry>
+ <entry spanname="ch1">DDI1</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+ <table id="dpiox1">
+ <title>Single channel PHY (CHV/BXT)</title>
+ <tgroup cols="4">
+ <colspec colname="c0" />
+ <colspec colname="c1" />
+ <colspec colname="c2" />
+ <colspec colname="c3" />
+ <spanspec spanname="ch0" namest="c0" nameend="c3" />
+ <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
+ <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
+ <thead>
+ <row>
+ <entry spanname="ch0">CH0</entry>
+ </row>
+ </thead>
+ <tbody valign="top" align="center">
+ <row>
+ <entry spanname="ch0">CMN/PLL/REF</entry>
+ </row>
+ <row>
+ <entry spanname="ch0pcs01">PCS01</entry>
+ <entry spanname="ch0pcs23">PCS23</entry>
+ </row>
+ <row>
+ <entry>TX0</entry>
+ <entry>TX1</entry>
+ <entry>TX2</entry>
+ <entry>TX3</entry>
+ </row>
+ <row>
+ <entry spanname="ch0">DDI2</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+ </sect2>
+
+ <sect2>
+ <title>CSR firmware support for DMC</title>
+!Pdrivers/gpu/drm/i915/intel_csr.c csr support for dmc
+!Idrivers/gpu/drm/i915/intel_csr.c
+ </sect2>
+ </sect1>
+
+ <sect1>
+ <title>Memory Management and Command Submission</title>
+ <para>
+ This sections covers all things related to the GEM implementation in the
+ i915 driver.
+ </para>
+ <sect2>
+ <title>Batchbuffer Parsing</title>
+!Pdrivers/gpu/drm/i915/i915_cmd_parser.c batch buffer command parser
+!Idrivers/gpu/drm/i915/i915_cmd_parser.c
+ </sect2>
+ <sect2>
+ <title>Batchbuffer Pools</title>
+!Pdrivers/gpu/drm/i915/i915_gem_batch_pool.c batch pool
+!Idrivers/gpu/drm/i915/i915_gem_batch_pool.c
+ </sect2>
+ <sect2>
+ <title>Logical Rings, Logical Ring Contexts and Execlists</title>
+!Pdrivers/gpu/drm/i915/intel_lrc.c Logical Rings, Logical Ring Contexts and Execlists
+!Idrivers/gpu/drm/i915/intel_lrc.c
+ </sect2>
+ <sect2>
+ <title>Global GTT views</title>
+!Pdrivers/gpu/drm/i915/i915_gem_gtt.c Global GTT views
+!Idrivers/gpu/drm/i915/i915_gem_gtt.c
+ </sect2>
+ <sect2>
+ <title>GTT Fences and Swizzling</title>
+!Idrivers/gpu/drm/i915/i915_gem_fence.c
+ <sect3>
+ <title>Global GTT Fence Handling</title>
+!Pdrivers/gpu/drm/i915/i915_gem_fence.c fence register handling
+ </sect3>
+ <sect3>
+ <title>Hardware Tiling and Swizzling Details</title>
+!Pdrivers/gpu/drm/i915/i915_gem_fence.c tiling swizzling details
+ </sect3>
+ </sect2>
+ <sect2>
+ <title>Object Tiling IOCTLs</title>
+!Idrivers/gpu/drm/i915/i915_gem_tiling.c
+!Pdrivers/gpu/drm/i915/i915_gem_tiling.c buffer object tiling
+ </sect2>
+ <sect2>
+ <title>Buffer Object Eviction</title>
+ <para>
+ This section documents the interface functions for evicting buffer
+ objects to make space available in the virtual gpu address spaces.
+ Note that this is mostly orthogonal to shrinking buffer objects
+ caches, which has the goal to make main memory (shared with the gpu
+ through the unified memory architecture) available.
+ </para>
+!Idrivers/gpu/drm/i915/i915_gem_evict.c
+ </sect2>
+ <sect2>
+ <title>Buffer Object Memory Shrinking</title>
+ <para>
+ This section documents the interface function for shrinking memory
+ usage of buffer object caches. Shrinking is used to make main memory
+ available. Note that this is mostly orthogonal to evicting buffer
+ objects, which has the goal to make space in gpu virtual address
+ spaces.
+ </para>
+!Idrivers/gpu/drm/i915/i915_gem_shrinker.c
+ </sect2>
+ </sect1>
+ <sect1>
+ <title>GuC-based Command Submission</title>
+ <sect2>
+ <title>GuC</title>
+!Pdrivers/gpu/drm/i915/intel_guc_loader.c GuC-specific firmware loader
+!Idrivers/gpu/drm/i915/intel_guc_loader.c
+ </sect2>
+ <sect2>
+ <title>GuC Client</title>
+!Pdrivers/gpu/drm/i915/i915_guc_submission.c GuC-based command submissison
+!Idrivers/gpu/drm/i915/i915_guc_submission.c
+ </sect2>
+ </sect1>
+
+ <sect1>
+ <title> Tracing </title>
+ <para>
+ This sections covers all things related to the tracepoints implemented in
+ the i915 driver.
+ </para>
+ <sect2>
+ <title> i915_ppgtt_create and i915_ppgtt_release </title>
+!Pdrivers/gpu/drm/i915/i915_trace.h i915_ppgtt_create and i915_ppgtt_release tracepoints
+ </sect2>
+ <sect2>
+ <title> i915_context_create and i915_context_free </title>
+!Pdrivers/gpu/drm/i915/i915_trace.h i915_context_create and i915_context_free tracepoints
+ </sect2>
+ <sect2>
+ <title> switch_mm </title>
+!Pdrivers/gpu/drm/i915/i915_trace.h switch_mm tracepoint
+ </sect2>
+ </sect1>
+
+ </chapter>
+!Cdrivers/gpu/drm/i915/i915_irq.c
+</part>
+
+<part id="vga_switcheroo">
+ <title>vga_switcheroo</title>
+ <partintro>
+!Pdrivers/gpu/vga/vga_switcheroo.c Overview
+ </partintro>
+
+ <chapter id="modes_of_use">
+ <title>Modes of Use</title>
+ <sect1>
+ <title>Manual switching and manual power control</title>
+!Pdrivers/gpu/vga/vga_switcheroo.c Manual switching and manual power control
+ </sect1>
+ <sect1>
+ <title>Driver power control</title>
+!Pdrivers/gpu/vga/vga_switcheroo.c Driver power control
+ </sect1>
+ </chapter>
+
+ <chapter id="pubfunctions">
+ <title>Public functions</title>
+!Edrivers/gpu/vga/vga_switcheroo.c
+ </chapter>
+
+ <chapter id="pubstructures">
+ <title>Public structures</title>
+!Finclude/linux/vga_switcheroo.h vga_switcheroo_handler
+!Finclude/linux/vga_switcheroo.h vga_switcheroo_client_ops
+ </chapter>
+
+ <chapter id="pubconstants">
+ <title>Public constants</title>
+!Finclude/linux/vga_switcheroo.h vga_switcheroo_client_id
+!Finclude/linux/vga_switcheroo.h vga_switcheroo_state
+ </chapter>
+
+ <chapter id="privstructures">
+ <title>Private structures</title>
+!Fdrivers/gpu/vga/vga_switcheroo.c vgasr_priv
+!Fdrivers/gpu/vga/vga_switcheroo.c vga_switcheroo_client
+ </chapter>
+
+!Cdrivers/gpu/vga/vga_switcheroo.c
+!Cinclude/linux/vga_switcheroo.h
+</part>
+
+</book>
the amount of free space and expand the <COW device> before it fills up.
<persistent?> is P (Persistent) or N (Not persistent - will not survive
-after reboot).
-The difference is that for transient snapshots less metadata must be
-saved on disk - they can be kept in memory by the kernel.
+after reboot). O (Overflow) can be added as a persistent store option
+to allow userspace to advertise its support for seeing "Overflow" in the
+snapshot status. So supported store types are "P", "PO" and "N".
+
+The difference between persistent and transient is with transient
+snapshots less metadata must be saved on disk - they can be kept in
+memory by the kernel.
* snapshot-merge <origin> <COW device> <persistent> <chunksize>
These nodes must have the following properties:
- compatible : Should at least contain "arm,gic-v3-its".
- msi-controller : Boolean property. Identifies the node as an MSI controller
+- #msi-cells: Must be <1>. The single msi-cell is the DeviceID of the device
+ which will generate the MSI.
- reg: Specifies the base physical address and size of the ITS
registers.
gic-its@2c200000 {
compatible = "arm,gic-v3-its";
msi-controller;
+ #msi-cells = <1>;
reg = <0x0 0x2c200000 0 0x200000>;
};
};
gic-its@2c200000 {
compatible = "arm,gic-v3-its";
msi-controller;
+ #msi-cells = <1>;
reg = <0x0 0x2c200000 0 0x200000>;
};
gic-its@2c400000 {
compatible = "arm,gic-v3-its";
msi-controller;
+ #msi-cells = <1>;
reg = <0x0 0x2c400000 0 0x200000>;
};
};
};
idle-states {
- entry-method = "arm,psci";
+ entry-method = "psci";
CPU_RETENTION_0_0: cpu-retention-0-0 {
compatible = "arm,idle-state";
--- /dev/null
+Broadcom VC4 (VideoCore4) GPU
+
+The VC4 device present on the Raspberry Pi includes a display system
+with HDMI output and the HVS (Hardware Video Scaler) for compositing
+display planes.
+
+Required properties for VC4:
+- compatible: Should be "brcm,bcm2835-vc4"
+
+Required properties for Pixel Valve:
+- compatible: Should be one of "brcm,bcm2835-pixelvalve0",
+ "brcm,bcm2835-pixelvalve1", or "brcm,bcm2835-pixelvalve2"
+- reg: Physical base address and length of the PV's registers
+- interrupts: The interrupt number
+ See bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt
+
+Required properties for HVS:
+- compatible: Should be "brcm,bcm2835-hvs"
+- reg: Physical base address and length of the HVS's registers
+- interrupts: The interrupt number
+ See bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt
+
+Required properties for HDMI
+- compatible: Should be "brcm,bcm2835-hdmi"
+- reg: Physical base address and length of the two register ranges
+ ("HDMI" and "HD", in that order)
+- interrupts: The interrupt numbers
+ See bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt
+- ddc: phandle of the I2C controller used for DDC EDID probing
+- clocks: a) hdmi: The HDMI state machine clock
+ b) pixel: The pixel clock.
+
+Optional properties for HDMI:
+- hpd-gpios: The GPIO pin for HDMI hotplug detect (if it doesn't appear
+ as an interrupt/status bit in the HDMI controller
+ itself). See bindings/pinctrl/brcm,bcm2835-gpio.txt
+
+Example:
+pixelvalve@7e807000 {
+ compatible = "brcm,bcm2835-pixelvalve2";
+ reg = <0x7e807000 0x100>;
+ interrupts = <2 10>; /* pixelvalve */
+};
+
+hvs@7e400000 {
+ compatible = "brcm,bcm2835-hvs";
+ reg = <0x7e400000 0x6000>;
+ interrupts = <2 1>;
+};
+
+hdmi: hdmi@7e902000 {
+ compatible = "brcm,bcm2835-hdmi";
+ reg = <0x7e902000 0x600>,
+ <0x7e808000 0x100>;
+ interrupts = <2 8>, <2 9>;
+ ddc = <&i2c2>;
+ hpd-gpios = <&gpio 46 GPIO_ACTIVE_HIGH>;
+ clocks = <&clocks BCM2835_PLLH_PIX>,
+ <&clocks BCM2835_CLOCK_HSM>;
+ clock-names = "pixel", "hdmi";
+};
+
+vc4: gpu {
+ compatible = "brcm,bcm2835-vc4";
+};
Required properties:
- compatible: one of the following
+ * "qcom,hdmi-tx-8996"
* "qcom,hdmi-tx-8994"
* "qcom,hdmi-tx-8084"
* "qcom,hdmi-tx-8974"
Optional properties:
- qcom,hdmi-tx-mux-en-gpio: hdmi mux enable pin
- qcom,hdmi-tx-mux-sel-gpio: hdmi mux select pin
+- power-domains: reference to the power domain(s), if available.
- pinctrl-names: the pin control state names; should contain "default"
- pinctrl-0: the default pinctrl state (active)
- pinctrl-1: the "sleep" pinctrl state
reg-names = "core_physical";
reg = <0x04a00000 0x1000>;
interrupts = <GIC_SPI 79 0>;
+ power-domains = <&mmcc MDSS_GDSC>;
clock-names =
"core_clk",
"master_iface_clk",
- clock-names: the following clocks are required:
* "core_clk"
* "iface_clk"
- * "lut_clk"
* "src_clk"
* "hdmi_clk"
* "mpd_clk"
Optional properties:
- gpus: phandle for gpu device
+- clock-names: the following clocks are optional:
+ * "lut_clk"
Example:
GPIO properties should be named "[<name>-]gpios", with <name> being the purpose
of this GPIO for the device. While a non-existent <name> is considered valid
for compatibility reasons (resolving to the "gpios" property), it is not allowed
-for new bindings.
+for new bindings. Also, GPIO properties named "[<name>-]gpio" are valid and old
+bindings use it, but are only supported for compatibility reasons and should not
+be used for newer bindings since it has been deprecated.
GPIO properties can contain one or more GPIO phandles, but only in exceptional
cases should they contain more than one. If your device uses several GPIOs with
-* Bosch BMA180 triaxial acceleration sensor
+* Bosch BMA180 / BMA250 triaxial acceleration sensor
http://omapworld.com/BMA180_111_1002839.pdf
+http://ae-bst.resource.bosch.com/media/products/dokumente/bma250/bst-bma250-ds002-05.pdf
Required properties:
- - compatible : should be "bosch,bma180"
+ - compatible : should be "bosch,bma180" or "bosch,bma250"
- reg : the I2C address of the sensor
Optional properties:
- interrupts : interrupt mapping for GPIO IRQ, it should by configured with
flags IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING
+ For the bma250 the first interrupt listed must be the one
+ connected to the INT1 pin, the second (optional) interrupt
+ listed must be the one connected to the INT2 pin.
Example:
/* Cypress Gen3 touchpad */
touchpad@67 {
compatible = "cypress,cyapa";
- reg = <0x24>;
+ reg = <0x67>;
interrupt-parent = <&gpio>;
interrupts = <2 IRQ_TYPE_EDGE_FALLING>; /* GPIO 2 */
wakeup-source;
interrupt.
Required Properties:
-- compatible: has to be "qca,<soctype>-cpu-intc", "qca,ar7100-misc-intc"
- as fallback
+- compatible: has to be "qca,<soctype>-cpu-intc", "qca,ar7100-misc-intc" or
+ "qca,<soctype>-cpu-intc", "qca,ar7240-misc-intc"
- reg: Base address and size of the controllers memory area
- interrupt-parent: phandle of the parent interrupt controller.
- interrupts: Interrupt specifier for the controllers interrupt.
- #interrupt-cells : Specifies the number of cells needed to encode interrupt
source, should be 1
+Compatible fallback depends on the SoC. Use ar7100 for ar71xx and ar913x,
+use ar7240 for all other SoCs.
+
Please refer to interrupts.txt in this directory for details of the common
Interrupt Controllers bindings used by client devices.
interrupt-controller;
#interrupt-cells = <1>;
};
+
+Another example:
+
+ interrupt-controller@18060010 {
+ compatible = "qca,ar9331-misc-intc", qca,ar7240-misc-intc";
+ reg = <0x18060010 0x4>;
+
+ interrupt-parent = <&cpuintc>;
+ interrupts = <6>;
+
+ interrupt-controller;
+ #interrupt-cells = <1>;
+ };
Required properties:
- compatible: "renesas,pci-r8a7790" for the R8A7790 SoC;
- "renesas,pci-r8a7791" for the R8A7791 SoC.
+ "renesas,pci-r8a7791" for the R8A7791 SoC;
+ "renesas,pci-r8a7794" for the R8A7794 SoC.
- reg: A list of physical regions to access the device: the first is
the operational registers for the OHCI/EHCI controllers and the
second is for the bridge configuration and control registers.
Required properties:
- compatible:
- - "ti,pbias-omap" for OMAP2, OMAP3, OMAP4, OMAP5, DRA7.
+ - should be "ti,pbias-dra7" for DRA7
+ - should be "ti,pbias-omap2" for OMAP2
+ - should be "ti,pbias-omap3" for OMAP3
+ - should be "ti,pbias-omap4" for OMAP4
+ - should be "ti,pbias-omap5" for OMAP5
+ - "ti,pbias-omap" is deprecated
- reg: pbias register offset from syscon base and size of pbias register.
- syscon : phandle of the system control module
- regulator-name : should be
- renesas,tx-fifo-size : Overrides the default tx fifo size given in words
(default is 64)
- renesas,rx-fifo-size : Overrides the default rx fifo size given in words
- (default is 64, or 256 on R-Car Gen2)
+ (default is 64)
Pinctrl properties might be needed, too. See
Documentation/devicetree/bindings/pinctrl/renesas,*.
- interrupts: Should contain spi interrupt
- clocks: phandles to input clocks.
- The first should be <&topckgen CLK_TOP_SPI_SEL>.
- The second should be one of the following.
+ The first should be one of the following. It's PLL.
- <&clk26m>: specify parent clock 26MHZ.
- <&topckgen CLK_TOP_SYSPLL3_D2>: specify parent clock 109MHZ.
It's the default one.
- <&topckgen CLK_TOP_SYSPLL4_D2>: specify parent clock 78MHZ.
- <&topckgen CLK_TOP_UNIVPLL2_D4>: specify parent clock 104MHZ.
- <&topckgen CLK_TOP_UNIVPLL1_D8>: specify parent clock 78MHZ.
+ The second should be <&topckgen CLK_TOP_SPI_SEL>. It's clock mux.
+ The third is <&pericfg CLK_PERI_SPI0>. It's clock gate.
-- clock-names: shall be "spi-clk" for the controller clock, and
- "parent-clk" for the parent clock.
+- clock-names: shall be "parent-clk" for the parent clock, "sel-clk" for the
+ muxes clock, and "spi-clk" for the clock gate.
Optional properties:
- mediatek,pad-select: specify which pins group(ck/mi/mo/cs) spi
#size-cells = <0>;
reg = <0 0x1100a000 0 0x1000>;
interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_LOW>;
- clocks = <&topckgen CLK_TOP_SPI_SEL>, <&topckgen CLK_TOP_SYSPLL3_D2>;
- clock-names = "spi-clk", "parent-clk";
+ clocks = <&topckgen CLK_TOP_SYSPLL3_D2>,
+ <&topckgen CLK_TOP_SPI_SEL>,
+ <&pericfg CLK_PERI_SPI0>;
+ clock-names = "parent-clk", "sel-clk", "spi-clk";
+
mediatek,pad-select = <0>;
status = "disabled";
};
the different fan speeds possible. Cooling states are referred to by
single unsigned integers, where larger numbers mean greater heat
dissipation. The precise set of cooling states associated with a device
-(as referred to be the cooling-min-state and cooling-max-state
+(as referred to by the cooling-min-level and cooling-max-level
properties) should be defined in a particular device's binding.
For more examples of cooling devices, refer to the example sections below.
Required properties:
-- cooling-min-state: An integer indicating the smallest
- Type: unsigned cooling state accepted. Typically 0.
- Size: one cell
-
-- cooling-max-state: An integer indicating the largest
- Type: unsigned cooling state accepted.
- Size: one cell
-
- #cooling-cells: Used to provide cooling device specific information
Type: unsigned while referring to it. Must be at least 2, in order
Size: one cell to specify minimum and maximum cooling state used
See Cooling device maps section below for more details
on how consumers refer to cooling devices.
+Optional properties:
+- cooling-min-level: An integer indicating the smallest
+ Type: unsigned cooling state accepted. Typically 0.
+ Size: one cell
+
+- cooling-max-level: An integer indicating the largest
+ Type: unsigned cooling state accepted.
+ Size: one cell
+
* Trip points
The trip node is a node to describe a point in the temperature domain
396000 950000
198000 850000
>;
- cooling-min-state = <0>;
- cooling-max-state = <3>;
+ cooling-min-level = <0>;
+ cooling-max-level = <3>;
#cooling-cells = <2>; /* min followed by max */
};
...
*/
fan0: fan@0x48 {
...
- cooling-min-state = <0>;
- cooling-max-state = <9>;
+ cooling-min-level = <0>;
+ cooling-max-level = <9>;
#cooling-cells = <2>; /* min followed by max */
};
};
"lsi,zevio-usb"
"qcom,ci-hdrc"
"chipidea,usb2"
+ "xlnx,zynq-usb-2.20a"
- reg: base address and length of the registers
- interrupts: interrupt for the USB controller
- "renesas,usbhs-r8a7790"
- "renesas,usbhs-r8a7791"
- "renesas,usbhs-r8a7794"
+ - "renesas,usbhs-r8a7795"
- reg: Base address and length of the register for the USBHS
- interrupts: Interrupt specifier for the USBHS
- clocks: A list of phandle + clock specifier pairs
skyworks Skyworks Solutions, Inc.
smsc Standard Microsystems Corporation
snps Synopsys, Inc.
+socionext Socionext Inc.
solidrun SolidRun
solomon Solomon Systech Limited
sony Sony Corporation
- compatible: must be one of the following.
- "renesas,du-r8a7779" for R8A7779 (R-Car H1) compatible DU
- "renesas,du-r8a7790" for R8A7790 (R-Car H2) compatible DU
- - "renesas,du-r8a7791" for R8A7791 (R-Car M2) compatible DU
+ - "renesas,du-r8a7791" for R8A7791 (R-Car M2-W) compatible DU
+ - "renesas,du-r8a7793" for R8A7793 (R-Car M2-N) compatible DU
+ - "renesas,du-r8a7794" for R8A7794 (R-Car E2) compatible DU
- reg: A list of base address and length of each memory resource, one for
each entry in the reg-names property.
- clock-names: Name of the clocks. This property is model-dependent.
- R8A7779 uses a single functional clock. The clock doesn't need to be
named.
- - R8A7790 and R8A7791 use one functional clock per channel and one clock
- per LVDS encoder. The functional clocks must be named "du.x" with "x"
- being the channel numerical index. The LVDS clocks must be named
+ - R8A779[0134] use one functional clock per channel and one clock per LVDS
+ encoder (if available). The functional clocks must be named "du.x" with
+ "x" being the channel numerical index. The LVDS clocks must be named
"lvds.x" with "x" being the LVDS encoder numerical index.
- In addition to the functional and encoder clocks, all DU versions also
support externally supplied pixel clocks. Those clocks are optional.
-----------------------------------------------------------------------------
R8A7779 (H1) DPAD 0 DPAD 1 -
R8A7790 (H2) DPAD LVDS 0 LVDS 1
- R8A7791 (M2) DPAD LVDS 0 -
+ R8A7791 (M2-W) DPAD LVDS 0 -
+ R8A7793 (M2-N) DPAD LVDS 0 -
+ R8A7794 (E2) DPAD 0 DPAD 1 -
Example: R8A7790 (R-Car H2) DU
device tree bindings for your controller.
GPIOs mappings are defined in the consumer device's node, in a property named
-<function>-gpios, where <function> is the function the driver will request
-through gpiod_get(). For example:
+either <function>-gpios or <function>-gpio, where <function> is the function
+the driver will request through gpiod_get(). For example:
foo_device {
compatible = "acme,foo";
<&gpio 16 GPIO_ACTIVE_HIGH>, /* green */
<&gpio 17 GPIO_ACTIVE_HIGH>; /* blue */
- power-gpios = <&gpio 1 GPIO_ACTIVE_LOW>;
+ power-gpio = <&gpio 1 GPIO_ACTIVE_LOW>;
};
This property will make GPIOs 15, 16 and 17 available to the driver under the
struct gpio_desc *red, *green, *blue, *power;
- red = gpiod_get_index(dev, "led", 0);
- green = gpiod_get_index(dev, "led", 1);
- blue = gpiod_get_index(dev, "led", 2);
+ red = gpiod_get_index(dev, "led", 0, GPIOD_OUT_HIGH);
+ green = gpiod_get_index(dev, "led", 1, GPIOD_OUT_HIGH);
+ blue = gpiod_get_index(dev, "led", 2, GPIOD_OUT_HIGH);
- power = gpiod_get(dev, "power");
+ power = gpiod_get(dev, "power", GPIOD_OUT_HIGH);
The led GPIOs will be active-high, while the power GPIO will be active-low (i.e.
gpiod_is_active_low(power) will be true).
+The second parameter of the gpiod_get() functions, the con_id string, has to be
+the <function>-prefix of the GPIO suffixes ("gpios" or "gpio", automatically
+looked up by the gpiod functions internally) used in the device tree. With above
+"led-gpios" example, use the prefix without the "-" as con_id parameter: "led".
+
+Internally, the GPIO subsystem prefixes the GPIO suffix ("gpios" or "gpio")
+with the string passed in con_id to get the resulting string
+(snprintf(... "%s-%s", con_id, gpio_suffixes[]).
+
ACPI
----
ACPI also supports function names for GPIOs in a similar fashion to DT.
struct gpio_desc *red, *green, *blue, *power;
- red = gpiod_get_index(dev, "led", 0);
- green = gpiod_get_index(dev, "led", 1);
- blue = gpiod_get_index(dev, "led", 2);
+ red = gpiod_get_index(dev, "led", 0, GPIOD_OUT_HIGH);
+ green = gpiod_get_index(dev, "led", 1, GPIOD_OUT_HIGH);
+ blue = gpiod_get_index(dev, "led", 2, GPIOD_OUT_HIGH);
- power = gpiod_get(dev, "power");
- gpiod_direction_output(power, 1);
+ power = gpiod_get(dev, "power", GPIOD_OUT_HIGH);
-Since the "power" GPIO is mapped as active-low, its actual signal will be 0
-after this code. Contrary to the legacy integer GPIO interface, the active-low
-property is handled during mapping and is thus transparent to GPIO consumers.
+Since the "led" GPIOs are mapped as active-high, this example will switch their
+signals to 1, i.e. enabling the LEDs. And for the "power" GPIO, which is mapped
+as active-low, its actual signal will be 0 after this code. Contrary to the legacy
+integer GPIO interface, the active-low property is handled during mapping and is
+thus transparent to GPIO consumers.
const char *con_id, unsigned int idx,
enum gpiod_flags flags)
+For a more detailed description of the con_id parameter in the DeviceTree case
+see Documentation/gpio/board.txt
+
The flags parameter is used to optionally specify a direction and initial value
for the GPIO. Values can be:
Prefix: 'nct6792'
Addresses scanned: ISA address retrieved from Super I/O registers
Datasheet: Available from Nuvoton upon request
+ * Nuvoton NCT6793D
+ Prefix: 'nct6793'
+ Addresses scanned: ISA address retrieved from Super I/O registers
+ Datasheet: Available from Nuvoton upon request
Authors:
Guenter Roeck <linux@roeck-us.net>
ABS_MT_POSITION_X := T_X
ABS_MT_POSITION_Y := T_Y
ABS_MT_TOOL_X := C_X
- ABS_MT_TOOL_X := C_Y
+ ABS_MT_TOOL_Y := C_Y
Unfortunately, there is not enough information to specify both the touching
ellipse and the tool ellipse, so one has to resort to approximations. One
The filter can be disabled or changed to another
driver later using sysfs.
- drm_kms_helper.edid_firmware=[<connector>:]<file>
- Broken monitors, graphic adapters and KVMs may
- send no or incorrect EDID data sets. This parameter
- allows to specify an EDID data set in the
- /lib/firmware directory that is used instead.
+ drm_kms_helper.edid_firmware=[<connector>:]<file>[,[<connector>:]<file>]
+ Broken monitors, graphic adapters, KVMs and EDIDless
+ panels may send no or incorrect EDID data sets.
+ This parameter allows to specify an EDID data sets
+ in the /lib/firmware directory that are used instead.
Generic built-in EDID data sets are used, if one of
edid/1024x768.bin, edid/1280x1024.bin,
edid/1680x1050.bin, or edid/1920x1080.bin is given
available in Documentation/EDID/HOWTO.txt. An EDID
data set will only be used for a particular connector,
if its name and a colon are prepended to the EDID
- name.
+ name. Each connector may use a unique EDID data
+ set by separating the files with a comma. An EDID
+ data set with no connector name will be used for
+ any connectors not explicitly specified.
dscc4.setup= [NET]
--- /dev/null
+Virtual Routing and Forwarding (VRF)
+====================================
+The VRF device combined with ip rules provides the ability to create virtual
+routing and forwarding domains (aka VRFs, VRF-lite to be specific) in the
+Linux network stack. One use case is the multi-tenancy problem where each
+tenant has their own unique routing tables and in the very least need
+different default gateways.
+
+Processes can be "VRF aware" by binding a socket to the VRF device. Packets
+through the socket then use the routing table associated with the VRF
+device. An important feature of the VRF device implementation is that it
+impacts only Layer 3 and above so L2 tools (e.g., LLDP) are not affected
+(ie., they do not need to be run in each VRF). The design also allows
+the use of higher priority ip rules (Policy Based Routing, PBR) to take
+precedence over the VRF device rules directing specific traffic as desired.
+
+In addition, VRF devices allow VRFs to be nested within namespaces. For
+example network namespaces provide separation of network interfaces at L1
+(Layer 1 separation), VLANs on the interfaces within a namespace provide
+L2 separation and then VRF devices provide L3 separation.
+
+Design
+------
+A VRF device is created with an associated route table. Network interfaces
+are then enslaved to a VRF device:
+
+ +-----------------------------+
+ | vrf-blue | ===> route table 10
+ +-----------------------------+
+ | | |
+ +------+ +------+ +-------------+
+ | eth1 | | eth2 | ... | bond1 |
+ +------+ +------+ +-------------+
+ | |
+ +------+ +------+
+ | eth8 | | eth9 |
+ +------+ +------+
+
+Packets received on an enslaved device and are switched to the VRF device
+using an rx_handler which gives the impression that packets flow through
+the VRF device. Similarly on egress routing rules are used to send packets
+to the VRF device driver before getting sent out the actual interface. This
+allows tcpdump on a VRF device to capture all packets into and out of the
+VRF as a whole.[1] Similiarly, netfilter [2] and tc rules can be applied
+using the VRF device to specify rules that apply to the VRF domain as a whole.
+
+[1] Packets in the forwarded state do not flow through the device, so those
+ packets are not seen by tcpdump. Will revisit this limitation in a
+ future release.
+
+[2] Iptables on ingress is limited to NF_INET_PRE_ROUTING only with skb->dev
+ set to real ingress device and egress is limited to NF_INET_POST_ROUTING.
+ Will revisit this limitation in a future release.
+
+
+Setup
+-----
+1. VRF device is created with an association to a FIB table.
+ e.g, ip link add vrf-blue type vrf table 10
+ ip link set dev vrf-blue up
+
+2. Rules are added that send lookups to the associated FIB table when the
+ iif or oif is the VRF device. e.g.,
+ ip ru add oif vrf-blue table 10
+ ip ru add iif vrf-blue table 10
+
+ Set the default route for the table (and hence default route for the VRF).
+ e.g, ip route add table 10 prohibit default
+
+3. Enslave L3 interfaces to a VRF device.
+ e.g, ip link set dev eth1 master vrf-blue
+
+ Local and connected routes for enslaved devices are automatically moved to
+ the table associated with VRF device. Any additional routes depending on
+ the enslaved device will need to be reinserted following the enslavement.
+
+4. Additional VRF routes are added to associated table.
+ e.g., ip route add table 10 ...
+
+
+Applications
+------------
+Applications that are to work within a VRF need to bind their socket to the
+VRF device:
+
+ setsockopt(sd, SOL_SOCKET, SO_BINDTODEVICE, dev, strlen(dev)+1);
+
+or to specify the output device using cmsg and IP_PKTINFO.
+
+
+Limitations
+-----------
+VRF device currently only works for IPv4. Support for IPv6 is under development.
+
+Index of original ingress interface is not available via cmsg. Will address
+soon.
(alternatively, the runtime_suspend() callback will have to check if the
device should really be suspended and return -EAGAIN if that is not the case).
-The runtime PM of PCI devices is disabled by default. It is also blocked by
-pci_pm_init() that runs the pm_runtime_forbid() helper function. If a PCI
-driver implements the runtime PM callbacks and intends to use the runtime PM
-framework provided by the PM core and the PCI subsystem, it should enable this
-feature by executing the pm_runtime_enable() helper function. However, the
-driver should not call the pm_runtime_allow() helper function unblocking
-the runtime PM of the device. Instead, it should allow user space or some
-platform-specific code to do that (user space can do it via sysfs), although
-once it has called pm_runtime_enable(), it must be prepared to handle the
+The runtime PM of PCI devices is enabled by default by the PCI core. PCI
+device drivers do not need to enable it and should not attempt to do so.
+However, it is blocked by pci_pm_init() that runs the pm_runtime_forbid()
+helper function. In addition to that, the runtime PM usage counter of
+each PCI device is incremented by local_pci_probe() before executing the
+probe callback provided by the device's driver.
+
+If a PCI driver implements the runtime PM callbacks and intends to use the
+runtime PM framework provided by the PM core and the PCI subsystem, it needs
+to decrement the device's runtime PM usage counter in its probe callback
+function. If it doesn't do that, the counter will always be different from
+zero for the device and it will never be runtime-suspended. The simplest
+way to do that is by calling pm_runtime_put_noidle(), but if the driver
+wants to schedule an autosuspend right away, for example, it may call
+pm_runtime_put_autosuspend() instead for this purpose. Generally, it
+just needs to call a function that decrements the devices usage counter
+from its probe routine to make runtime PM work for the device.
+
+It is important to remember that the driver's runtime_suspend() callback
+may be executed right after the usage counter has been decremented, because
+user space may already have cuased the pm_runtime_allow() helper function
+unblocking the runtime PM of the device to run via sysfs, so the driver must
+be prepared to cope with that.
+
+The driver itself should not call pm_runtime_allow(), though. Instead, it
+should let user space or some platform-specific code do that (user space can
+do it via sysfs as stated above), but it must be prepared to handle the
runtime PM of the device correctly as soon as pm_runtime_allow() is called
-(which may happen at any time). [It also is possible that user space causes
-pm_runtime_allow() to be called via sysfs before the driver is loaded, so in
-fact the driver has to be prepared to handle the runtime PM of the device as
-soon as it calls pm_runtime_enable().]
+(which may happen at any time, even before the driver is loaded).
+
+When the driver's remove callback runs, it has to balance the decrementation
+of the device's runtime PM usage counter at the probe time. For this reason,
+if it has decremented the counter in its probe callback, it must run
+pm_runtime_get_noresume() in its remove callback. [Since the core carries
+out a runtime resume of the device and bumps up the device's usage counter
+before running the driver's remove callback, the runtime PM of the device
+is effectively disabled for the duration of the remove execution and all
+runtime PM helper functions incrementing the device's usage counter are
+then effectively equivalent to pm_runtime_get_noresume().]
The runtime PM framework works by processing requests to suspend or resume
devices, or to check if they are idle (in which cases it is reasonable to
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#define _GNU_SOURCE
+#define __SANE_USERSPACE_TYPES__ /* For PPC64, to get LL64 types */
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
DEFINE_STATIC_KEY_TRUE(key);
DEFINE_STATIC_KEY_FALSE(key);
-static_key_likely()
-statick_key_unlikely()
+static_branch_likely()
+static_branch_unlikely()
0) Abstract
--------------
The default queuing discipline to use for network devices. This allows
-overriding the default queue discipline of pfifo_fast with an
-alternative. Since the default queuing discipline is created with the
-no additional parameters so is best suited to queuing disciplines that
-work well without configuration like stochastic fair queue (sfq),
-CoDel (codel) or fair queue CoDel (fq_codel). Don't use queuing disciplines
-like Hierarchical Token Bucket or Deficit Round Robin which require setting
-up classes and bandwidths.
+overriding the default of pfifo_fast with an alternative. Since the default
+queuing discipline is created without additional parameters so is best suited
+to queuing disciplines that work well without configuration like stochastic
+fair queue (sfq), CoDel (codel) or fair queue CoDel (fq_codel). Don't use
+queuing disciplines like Hierarchical Token Bucket or Deficit Round Robin
+which require setting up classes and bandwidths. Note that physical multiqueue
+interfaces still use mq as root qdisc, which in turn uses this default for its
+leaves. Virtual devices (like e.g. lo or veth) ignore this setting and instead
+default to noqueue.
Default: pfifo_fast
busy_read
Trip points
-----------
-The governor requires the following two passive trip points:
+The governor works optimally with the following two passive trip points:
1. "switch on" trip point: temperature above which the governor
control loop starts operating. This is the first passive trip
F: drivers/hwmon/fam15h_power.c
AMD GEODE CS5536 USB DEVICE CONTROLLER DRIVER
-M: Thomas Dahlmann <dahlmann.thomas@arcor.de>
L: linux-geode@lists.infradead.org (moderated for non-subscribers)
-S: Supported
+S: Orphan
F: drivers/usb/gadget/udc/amd5536udc.*
AMD GEODE PROCESSOR/CHIPSET SUPPORT
F: drivers/video/fbdev/arcfb.c
F: drivers/video/fbdev/core/fb_defio.c
+ARCNET NETWORK LAYER
+M: Michael Grzeschik <m.grzeschik@pengutronix.de>
+L: netdev@vger.kernel.org
+S: Maintained
+F: drivers/net/arcnet/
+F: include/uapi/linux/if_arcnet.h
+
ARM MFM AND FLOPPY DRIVERS
M: Ian Molton <spyro@f2s.com>
S: Maintained
DIGI EPCA PCI PRODUCTS
M: Lidza Louina <lidza.louina@gmail.com>
-M: Mark Hounschell <markh@compro.net>
M: Daeseok Youn <daeseok.youn@gmail.com>
L: driverdev-devel@linuxdriverproject.org
S: Maintained
M: Jani Nikula <jani.nikula@linux.intel.com>
L: intel-gfx@lists.freedesktop.org
L: dri-devel@lists.freedesktop.org
+W: https://01.org/linuxgraphics/
Q: http://patchwork.freedesktop.org/project/intel-gfx/
T: git git://anongit.freedesktop.org/drm-intel
S: Supported
L: dri-devel@lists.freedesktop.org
S: Maintained
F: drivers/gpu/drm/imx/
+F: drivers/gpu/ipu-v3/
F: Documentation/devicetree/bindings/drm/imx/
DRM DRIVERS FOR NVIDIA TEGRA
F: sound/usb/misc/ua101.c
EXTENSIBLE FIRMWARE INTERFACE (EFI)
-M: Matt Fleming <matt.fleming@intel.com>
+M: Matt Fleming <matt@codeblueprint.co.uk>
L: linux-efi@vger.kernel.org
T: git git://git.kernel.org/pub/scm/linux/kernel/git/mfleming/efi.git
S: Maintained
EFI VARIABLE FILESYSTEM
M: Matthew Garrett <matthew.garrett@nebula.com>
M: Jeremy Kerr <jk@ozlabs.org>
-M: Matt Fleming <matt.fleming@intel.com>
+M: Matt Fleming <matt@codeblueprint.co.uk>
T: git git://git.kernel.org/pub/scm/linux/kernel/git/mfleming/efi.git
L: linux-efi@vger.kernel.org
S: Maintained
KERNEL VIRTUAL MACHINE (KVM) FOR AMD-V
M: Joerg Roedel <joro@8bytes.org>
L: kvm@vger.kernel.org
-W: http://kvm.qumranet.com
+W: http://www.linux-kvm.org/
S: Maintained
F: arch/x86/include/asm/svm.h
F: arch/x86/kvm/svm.c
KERNEL VIRTUAL MACHINE (KVM) FOR POWERPC
M: Alexander Graf <agraf@suse.com>
L: kvm-ppc@vger.kernel.org
-W: http://kvm.qumranet.com
+W: http://www.linux-kvm.org/
T: git git://github.com/agraf/linux-2.6.git
S: Supported
F: arch/powerpc/include/asm/kvm*
LTP (Linux Test Project)
M: Mike Frysinger <vapier@gentoo.org>
M: Cyril Hrubis <chrubis@suse.cz>
-M: Wanlong Gao <gaowanlong@cn.fujitsu.com>
+M: Wanlong Gao <wanlong.gao@gmail.com>
M: Jan Stancek <jstancek@redhat.com>
M: Stanislav Kholmanskikh <stanislav.kholmanskikh@oracle.com>
M: Alexey Kodanev <alexey.kodanev@oracle.com>
-L: ltp-list@lists.sourceforge.net (subscribers-only)
+L: ltp@lists.linux.it (subscribers-only)
W: http://linux-test-project.github.io/
T: git git://github.com/linux-test-project/ltp.git
S: Maintained
F: drivers/net/ethernet/qlogic/qla3xxx.*
QLOGIC QLCNIC (1/10)Gb ETHERNET DRIVER
-M: Shahed Shaikh <shahed.shaikh@qlogic.com>
M: Dept-GELinuxNICDev@qlogic.com
L: netdev@vger.kernel.org
S: Supported
STAGING - LUSTRE PARALLEL FILESYSTEM
M: Oleg Drokin <oleg.drokin@intel.com>
M: Andreas Dilger <andreas.dilger@intel.com>
-L: HPDD-discuss@lists.01.org (moderated for non-subscribers)
-W: http://lustre.opensfs.org/
+L: lustre-devel@lists.lustre.org (moderated for non-subscribers)
+W: http://wiki.lustre.org/
S: Maintained
F: drivers/staging/lustre
STAGING - NVIDIA COMPLIANT EMBEDDED CONTROLLER INTERFACE (nvec)
-M: Julian Andres Klode <jak@jak-linux.org>
M: Marc Dietrich <marvin24@gmx.de>
L: ac100@lists.launchpad.net (moderated for non-subscribers)
L: linux-tegra@vger.kernel.org
F: include/linux/cpu_cooling.h
F: Documentation/devicetree/bindings/thermal/
+THERMAL/CPU_COOLING
+M: Amit Daniel Kachhap <amit.kachhap@gmail.com>
+M: Viresh Kumar <viresh.kumar@linaro.org>
+M: Javi Merino <javi.merino@arm.com>
+L: linux-pm@vger.kernel.org
+S: Supported
+F: Documentation/thermal/cpu-cooling-api.txt
+F: drivers/thermal/cpu_cooling.c
+F: include/linux/cpu_cooling.h
+
THINGM BLINK(1) USB RGB LED DRIVER
M: Vivien Didelot <vivien.didelot@savoirfairelinux.com>
S: Maintained
F: include/linux/vlynq.h
VME SUBSYSTEM
-M: Martyn Welch <martyn.welch@ge.com>
+M: Martyn Welch <martyn@welchs.me.uk>
M: Manohar Vanga <manohar.vanga@gmail.com>
M: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
L: devel@driverdev.osuosl.org
M: Liam Girdwood <lgirdwood@gmail.com>
M: Mark Brown <broonie@kernel.org>
L: linux-kernel@vger.kernel.org
-W: http://opensource.wolfsonmicro.com/node/15
W: http://www.slimlogic.co.uk/?p=48
T: git git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git
S: Supported
S: Maintained
F: drivers/net/vrf.c
F: include/net/vrf.h
+F: Documentation/networking/vrf.txt
VT1211 HARDWARE MONITOR DRIVER
M: Juerg Haefliger <juergh@gmail.com>
S: Maintained
F: drivers/net/wireless/wl3501*
-WM97XX TOUCHSCREEN DRIVERS
-M: Mark Brown <broonie@kernel.org>
-M: Liam Girdwood <lrg@slimlogic.co.uk>
-L: linux-input@vger.kernel.org
-T: git git://opensource.wolfsonmicro.com/linux-2.6-touch
-W: http://opensource.wolfsonmicro.com/node/7
-S: Supported
-F: drivers/input/touchscreen/*wm97*
-F: include/linux/wm97xx.h
-
WOLFSON MICROELECTRONICS DRIVERS
L: patches@opensource.wolfsonmicro.com
-T: git git://opensource.wolfsonmicro.com/linux-2.6-asoc
-T: git git://opensource.wolfsonmicro.com/linux-2.6-audioplus
-W: http://opensource.wolfsonmicro.com/content/linux-drivers-wolfson-devices
+T: git https://github.com/CirrusLogic/linux-drivers.git
+W: https://github.com/CirrusLogic/linux-drivers/wiki
S: Supported
F: Documentation/hwmon/wm83??
F: arch/arm/mach-s3c64xx/mach-crag6410*
VERSION = 4
PATCHLEVEL = 3
SUBLEVEL = 0
-EXTRAVERSION = -rc1
-NAME = Hurr durr I'ma sheep
+EXTRAVERSION = -rc5
+NAME = Blurry Fish Butt
# *DOCUMENTATION*
# To see a list of typical targets execute "make help"
unsigned long size)
{
return ioremap(offset, size);
-}
+}
+
+#define ioremap_uc ioremap_nocache
static inline void iounmap(volatile void __iomem *addr)
{
#endif
}
+#define zero_bytemask(mask) ((2ul << (find_zero(mask) * 8)) - 1)
+
#endif /* _ASM_WORD_AT_A_TIME_H */
}
irq_enter();
- generic_handle_irq_desc(irq, desc);
+ generic_handle_irq_desc(desc);
irq_exit();
}
void pcibios_fixup_bus(struct pci_bus *bus)
{
- struct pci_dev *dev;
+ struct pci_dev *dev = bus->self;
+
+ if (pci_has_flag(PCI_PROBE_ONLY) && dev &&
+ (dev->class >> 8) == PCI_CLASS_BRIDGE_PCI) {
+ pci_read_bridge_bases(bus);
+ }
list_for_each_entry(dev, &bus->devices, bus_list) {
pdev_save_srm_config(dev);
" bgt %0,1b"
: "=&r" (tmp), "=r" (loops) : "1"(loops));
}
+EXPORT_SYMBOL(__delay);
#ifdef CONFIG_SMP
#define LPJ cpu_data[smp_processor_id()].loops_per_jiffy
generic-y += ucontext.h
generic-y += user.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
static int idu_first_irq;
-static void idu_cascade_isr(unsigned int __core_irq, struct irq_desc *desc)
+static void idu_cascade_isr(struct irq_desc *desc)
{
struct irq_domain *domain = irq_desc_get_handler_data(desc);
unsigned int core_irq = irq_desc_get_irq(desc);
LD += -EL
endif
+#
+# The Scalar Replacement of Aggregates (SRA) optimization pass in GCC 4.9 and
+# later may result in code being generated that handles signed short and signed
+# char struct members incorrectly. So disable it.
+# (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65932)
+#
+KBUILD_CFLAGS += $(call cc-option,-fno-ipa-sra)
+
# This selects which instruction set is used.
# Note that GCC does not numerically define an architecture version
# macro, but instead defines a whole series of macros which makes
sun4i-a10-hackberry.dtb \
sun4i-a10-hyundai-a7hd.dtb \
sun4i-a10-inet97fv2.dtb \
- sun4i-a10-itead-iteaduino-plus.dts \
+ sun4i-a10-itead-iteaduino-plus.dtb \
sun4i-a10-jesurun-q5.dtb \
sun4i-a10-marsboard.dtb \
sun4i-a10-mini-xplus.dtb \
};
vdd1_reg: regulator@2 {
- /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */
+ /* VDD_MPU voltage limits 0.95V - 1.325V with +/-4% tolerance */
regulator-name = "vdd_mpu";
regulator-min-microvolt = <912500>;
- regulator-max-microvolt = <1312500>;
+ regulator-max-microvolt = <1378000>;
regulator-boot-on;
regulator-always-on;
};
pinctrl-0 = <&extcon_usb1_pins>;
};
- extcon_usb2: extcon_usb2 {
- compatible = "linux,extcon-usb-gpio";
- id-gpio = <&gpio7 24 GPIO_ACTIVE_HIGH>;
- pinctrl-names = "default";
- pinctrl-0 = <&extcon_usb2_pins>;
- };
-
hdmi0: connector {
compatible = "hdmi-connector";
label = "hdmi";
>;
};
- extcon_usb2_pins: extcon_usb2_pins {
- pinctrl-single,pins = <
- 0x3e8 (PIN_INPUT_PULLUP | MUX_MODE14) /* uart1_ctsn.gpio7_24 */
- >;
- };
-
tpd12s015_pins: pinmux_tpd12s015_pins {
pinctrl-single,pins = <
0x3b0 (PIN_OUTPUT | MUX_MODE14) /* gpio7_10 CT_CP_HPD */
};
ldo3_reg: ldo3 {
- /* VDDA_1V8_PHY */
+ /* VDDA_1V8_PHYA */
regulator-name = "ldo3";
regulator-min-microvolt = <1800000>;
regulator-max-microvolt = <1800000>;
regulator-boot-on;
};
+ ldo4_reg: ldo4 {
+ /* VDDA_1V8_PHYB */
+ regulator-name = "ldo4";
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <1800000>;
+ regulator-always-on;
+ regulator-boot-on;
+ };
+
ldo9_reg: ldo9 {
/* VDD_RTC */
regulator-name = "ldo9";
gpio-controller;
#gpio-cells = <2>;
};
+
+ extcon_usb2: tps659038_usb {
+ compatible = "ti,palmas-usb-vid";
+ ti,enable-vbus-detection;
+ ti,enable-id-detection;
+ id-gpios = <&gpio7 24 GPIO_ACTIVE_HIGH>;
+ };
+
};
tmp102: tmp102@48 {
mcp_rtc: rtc@6f {
compatible = "microchip,mcp7941x";
reg = <0x6f>;
- interrupts = <GIC_SPI 2 IRQ_TYPE_EDGE_RISING>; /* IRQ_SYS_1N */
+ interrupts-extended = <&crossbar_mpu GIC_SPI 2 IRQ_TYPE_EDGE_RISING>,
+ <&dra7_pmx_core 0x424>;
pinctrl-names = "default";
pinctrl-0 = <&mcp79410_pins_default>;
pinctrl-0 = <&mmc1_pins_default>;
vmmc-supply = <&ldo1_reg>;
- vmmc_aux-supply = <&vdd_3v3>;
bus-width = <4>;
cd-gpios = <&gpio6 27 0>; /* gpio 219 */
};
};
&usb2 {
+ /*
+ * Stand alone usage is peripheral only.
+ * However, with some resistor modifications
+ * this port can be used via expansion connectors
+ * as "host" or "dual-role". If so, provide
+ * the necessary dr_mode override in the expansion
+ * board's DT.
+ */
dr_mode = "peripheral";
};
&hdmi {
status = "ok";
- vdda-supply = <&ldo3_reg>;
+ vdda-supply = <&ldo4_reg>;
pinctrl-names = "default";
pinctrl-0 = <&hdmi_pins>;
&cpsw_emac0 {
phy_id = <&davinci_mdio>, <0>;
- phy-mode = "mii";
+ phy-mode = "rgmii";
};
&cpsw_emac1 {
phy_id = <&davinci_mdio>, <1>;
- phy-mode = "mii";
+ phy-mode = "rgmii";
};
#include "dm814x.dtsi"
/ {
- model = "DM8148 EVM";
+ model = "HP t410 Smart Zero Client";
compatible = "hp,t410", "ti,dm8148";
memory {
&cpsw_emac0 {
phy_id = <&davinci_mdio>, <0>;
- phy-mode = "mii";
+ phy-mode = "rgmii";
};
&cpsw_emac1 {
phy_id = <&davinci_mdio>, <1>;
- phy-mode = "mii";
+ phy-mode = "rgmii";
};
ti,hwmods = "timer3";
};
- control: control@160000 {
+ control: control@140000 {
compatible = "ti,dm814-scm", "simple-bus";
- reg = <0x160000 0x16d000>;
+ reg = <0x140000 0x16d000>;
#address-cells = <1>;
#size-cells = <1>;
ranges = <0 0x160000 0x16d000>;
mac-address = [ 00 00 00 00 00 00 ];
};
- phy_sel: cpsw-phy-sel@0x48160650 {
+ phy_sel: cpsw-phy-sel@48140650 {
compatible = "ti,am3352-cpsw-phy-sel";
- reg= <0x48160650 0x4>;
+ reg= <0x48140650 0x4>;
reg-names = "gmii-sel";
};
};
reg = <0x0 0x1400>;
#address-cells = <1>;
#size-cells = <1>;
+ ranges = <0 0x0 0x1400>;
pbias_regulator: pbias_regulator {
- compatible = "ti,pbias-omap";
+ compatible = "ti,pbias-dra7", "ti,pbias-omap";
reg = <0xe00 0x4>;
syscon = <&scm_conf>;
pbias_mmc_reg: pbias_mmc_omap5 {
ti,irqs-safe-map = <0>;
};
- mac: ethernet@4a100000 {
+ mac: ethernet@48484000 {
compatible = "ti,dra7-cpsw","ti,cpsw";
ti,hwmods = "gmac";
clocks = <&dpll_gmac_ck>, <&gmac_gmii_ref_clk_div>;
opp-hz = /bits/ 64 <800000000>;
opp-microvolt = <1000000>;
clock-latency-ns = <200000>;
+ opp-suspend;
};
opp07 {
opp-hz = /bits/ 64 <900000000>;
regulator-name = "P1.8V_LDO_OUT10";
regulator-min-microvolt = <1800000>;
regulator-max-microvolt = <1800000>;
+ regulator-always-on;
};
ldo11_reg: LDO11 {
interrupt-parent = <&combiner>;
interrupts = <3 0>;
clock-names = "sysmmu", "master";
- clocks = <&clock CLK_SMMU_FIMD1M0>, <&clock CLK_FIMD1>;
+ clocks = <&clock CLK_SMMU_FIMD1M1>, <&clock CLK_FIMD1>;
power-domains = <&disp_pd>;
#iommu-cells = <0>;
};
*/
pinctrl-0 = <&pwm0_out &pwm1_out &pwm2_out &pwm3_out>;
pinctrl-names = "default";
- samsung,pwm-outputs = <0>;
status = "okay";
};
pinctrl-0 = <&pinctrl_pmic>;
reg = <0x08>;
interrupt-parent = <&gpio5>;
- interrupts = <23 0x8>;
+ interrupts = <23 IRQ_TYPE_LEVEL_HIGH>;
regulators {
sw1_reg: sw1a {
regulator-name = "SW1";
#include <dt-bindings/clock/imx5-clock.h>
#include <dt-bindings/gpio/gpio.h>
#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
/ {
aliases {
compatible = "regulator-fixed";
reg = <1>;
pinctrl-names = "default";
- pinctrl-0 = <&pinctrl_usbh1>;
regulator-name = "usbh1_vbus";
regulator-min-microvolt = <5000000>;
regulator-max-microvolt = <5000000>;
compatible = "regulator-fixed";
reg = <2>;
pinctrl-names = "default";
- pinctrl-0 = <&pinctrl_usbotg>;
regulator-name = "usb_otg_vbus";
regulator-min-microvolt = <5000000>;
regulator-max-microvolt = <5000000>;
reg = <0x270 0x240>;
#address-cells = <1>;
#size-cells = <1>;
+ ranges = <0 0x270 0x240>;
scm_clocks: clocks {
#address-cells = <1>;
};
pbias_regulator: pbias_regulator {
- compatible = "ti,pbias-omap";
+ compatible = "ti,pbias-omap2", "ti,pbias-omap";
reg = <0x230 0x4>;
syscon = <&scm_conf>;
pbias_mmc_reg: pbias_mmc_omap2430 {
tfp410_pins: pinmux_tfp410_pins {
pinctrl-single,pins = <
- 0x194 (PIN_OUTPUT | MUX_MODE4) /* hdq_sio.gpio_170 */
+ 0x196 (PIN_OUTPUT | MUX_MODE4) /* hdq_sio.gpio_170 */
>;
};
>;
};
- smsc9221_pins: pinmux_smsc9221_pins {
- pinctrl-single,pins = <
- 0x1a2 (PIN_INPUT | MUX_MODE4) /* mcspi1_cs2.gpio_176 */
- >;
- };
-
i2c1_pins: pinmux_i2c1_pins {
pinctrl-single,pins = <
0x18a (PIN_INPUT | MUX_MODE0) /* i2c1_scl.i2c1_scl */
OMAP3_CORE1_IOPAD(0x217a, PIN_INPUT | MUX_MODE0) /* uart2_rx.uart2_rx */
>;
};
+
+ smsc9221_pins: pinmux_smsc9221_pins {
+ pinctrl-single,pins = <
+ OMAP3_CORE1_IOPAD(0x21d2, PIN_INPUT | MUX_MODE4) /* mcspi1_cs2.gpio_176 */
+ >;
+ };
};
&omap3_pmx_core2 {
};
scm_conf: scm_conf@270 {
- compatible = "syscon";
+ compatible = "syscon", "simple-bus";
reg = <0x270 0x330>;
#address-cells = <1>;
#size-cells = <1>;
+ ranges = <0 0x270 0x330>;
+
+ pbias_regulator: pbias_regulator {
+ compatible = "ti,pbias-omap3", "ti,pbias-omap";
+ reg = <0x2b0 0x4>;
+ syscon = <&scm_conf>;
+ pbias_mmc_reg: pbias_mmc_omap2430 {
+ regulator-name = "pbias_mmc_omap2430";
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <3000000>;
+ };
+ };
scm_clocks: clocks {
#address-cells = <1>;
dma-requests = <96>;
};
- pbias_regulator: pbias_regulator {
- compatible = "ti,pbias-omap";
- reg = <0x2b0 0x4>;
- syscon = <&scm_conf>;
- pbias_mmc_reg: pbias_mmc_omap2430 {
- regulator-name = "pbias_mmc_omap2430";
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3000000>;
- };
- };
-
gpio1: gpio@48310000 {
compatible = "ti,omap3-gpio";
reg = <0x48310000 0x200>;
reg = <0x5a0 0x170>;
#address-cells = <1>;
#size-cells = <1>;
+ ranges = <0 0x5a0 0x170>;
pbias_regulator: pbias_regulator {
- compatible = "ti,pbias-omap";
+ compatible = "ti,pbias-omap4", "ti,pbias-omap";
reg = <0x60 0x4>;
syscon = <&omap4_padconf_global>;
pbias_mmc_reg: pbias_mmc_omap4 {
i2c5_pins: pinmux_i2c5_pins {
pinctrl-single,pins = <
- 0x184 (PIN_INPUT | MUX_MODE0) /* i2c5_scl */
- 0x186 (PIN_INPUT | MUX_MODE0) /* i2c5_sda */
+ 0x186 (PIN_INPUT | MUX_MODE0) /* i2c5_scl */
+ 0x188 (PIN_INPUT | MUX_MODE0) /* i2c5_sda */
>;
};
reg = <0x5a0 0xec>;
#address-cells = <1>;
#size-cells = <1>;
+ ranges = <0 0x5a0 0xec>;
pbias_regulator: pbias_regulator {
- compatible = "ti,pbias-omap";
+ compatible = "ti,pbias-omap5", "ti,pbias-omap";
reg = <0x60 0x4>;
syscon = <&omap5_padconf_global>;
pbias_mmc_reg: pbias_mmc_omap5 {
"mix.0", "mix.1",
"dvc.0", "dvc.1",
"clk_a", "clk_b", "clk_c", "clk_i";
+ power-domains = <&cpg_clocks>;
status = "disabled";
"mix.0", "mix.1",
"dvc.0", "dvc.1",
"clk_a", "clk_b", "clk_c", "clk_i";
+ power-domains = <&cpg_clocks>;
status = "disabled";
};
&hdmi {
+ ddc-i2c-bus = <&i2c5>;
status = "okay";
};
<&clk_s_d0_quadfs 0>,
<&clk_s_d2_quadfs 0>,
<&clk_s_d2_quadfs 0>;
- ranges;
-
- sti-hdmi@8d04000 {
- compatible = "st,stih407-hdmi";
- reg = <0x8d04000 0x1000>;
- reg-names = "hdmi-reg";
- interrupts = <GIC_SPI 106 IRQ_TYPE_NONE>;
- interrupt-names = "irq";
- clock-names = "pix",
- "tmds",
- "phy",
- "audio",
- "main_parent",
- "aux_parent";
-
- clocks = <&clk_s_d2_flexgen CLK_PIX_HDMI>,
- <&clk_s_d2_flexgen CLK_TMDS_HDMI>,
- <&clk_s_d2_flexgen CLK_REF_HDMIPHY>,
- <&clk_s_d0_flexgen CLK_PCM_0>,
- <&clk_s_d2_quadfs 0>,
- <&clk_s_d2_quadfs 1>;
-
- hdmi,hpd-gpio = <&pio5 3>;
- reset-names = "hdmi";
- resets = <&softreset STIH407_HDMI_TX_PHY_SOFTRESET>;
- ddc = <&hdmiddc>;
-
- };
-
- sti-hda@8d02000 {
- compatible = "st,stih407-hda";
- reg = <0x8d02000 0x400>, <0x92b0120 0x4>;
- reg-names = "hda-reg", "video-dacs-ctrl";
- clock-names = "pix",
- "hddac",
- "main_parent",
- "aux_parent";
- clocks = <&clk_s_d2_flexgen CLK_PIX_HDDAC>,
- <&clk_s_d2_flexgen CLK_HDDAC>,
- <&clk_s_d2_quadfs 0>,
- <&clk_s_d2_quadfs 1>;
- };
+ };
+
+ sti-hdmi@8d04000 {
+ compatible = "st,stih407-hdmi";
+ reg = <0x8d04000 0x1000>;
+ reg-names = "hdmi-reg";
+ interrupts = <GIC_SPI 106 IRQ_TYPE_NONE>;
+ interrupt-names = "irq";
+ clock-names = "pix",
+ "tmds",
+ "phy",
+ "audio",
+ "main_parent",
+ "aux_parent";
+
+ clocks = <&clk_s_d2_flexgen CLK_PIX_HDMI>,
+ <&clk_s_d2_flexgen CLK_TMDS_HDMI>,
+ <&clk_s_d2_flexgen CLK_REF_HDMIPHY>,
+ <&clk_s_d0_flexgen CLK_PCM_0>,
+ <&clk_s_d2_quadfs 0>,
+ <&clk_s_d2_quadfs 1>;
+
+ hdmi,hpd-gpio = <&pio5 3>;
+ reset-names = "hdmi";
+ resets = <&softreset STIH407_HDMI_TX_PHY_SOFTRESET>;
+ ddc = <&hdmiddc>;
+ };
+
+ sti-hda@8d02000 {
+ compatible = "st,stih407-hda";
+ reg = <0x8d02000 0x400>, <0x92b0120 0x4>;
+ reg-names = "hda-reg", "video-dacs-ctrl";
+ clock-names = "pix",
+ "hddac",
+ "main_parent",
+ "aux_parent";
+ clocks = <&clk_s_d2_flexgen CLK_PIX_HDDAC>,
+ <&clk_s_d2_flexgen CLK_HDDAC>,
+ <&clk_s_d2_quadfs 0>,
+ <&clk_s_d2_quadfs 1>;
};
};
};
<&clk_s_d0_quadfs 0>,
<&clk_s_d2_quadfs 0>,
<&clk_s_d2_quadfs 0>;
- ranges;
-
- sti-hdmi@8d04000 {
- compatible = "st,stih407-hdmi";
- reg = <0x8d04000 0x1000>;
- reg-names = "hdmi-reg";
- interrupts = <GIC_SPI 106 IRQ_TYPE_NONE>;
- interrupt-names = "irq";
- clock-names = "pix",
- "tmds",
- "phy",
- "audio",
- "main_parent",
- "aux_parent";
-
- clocks = <&clk_s_d2_flexgen CLK_PIX_HDMI>,
- <&clk_s_d2_flexgen CLK_TMDS_HDMI>,
- <&clk_s_d2_flexgen CLK_REF_HDMIPHY>,
- <&clk_s_d0_flexgen CLK_PCM_0>,
- <&clk_s_d2_quadfs 0>,
- <&clk_s_d2_quadfs 1>;
-
- hdmi,hpd-gpio = <&pio5 3>;
- reset-names = "hdmi";
- resets = <&softreset STIH407_HDMI_TX_PHY_SOFTRESET>;
- ddc = <&hdmiddc>;
-
- };
-
- sti-hda@8d02000 {
- compatible = "st,stih407-hda";
- reg = <0x8d02000 0x400>, <0x92b0120 0x4>;
- reg-names = "hda-reg", "video-dacs-ctrl";
- clock-names = "pix",
- "hddac",
- "main_parent",
- "aux_parent";
- clocks = <&clk_s_d2_flexgen CLK_PIX_HDDAC>,
- <&clk_s_d2_flexgen CLK_HDDAC>,
- <&clk_s_d2_quadfs 0>,
- <&clk_s_d2_quadfs 1>;
- };
+ };
+
+ sti-hdmi@8d04000 {
+ compatible = "st,stih407-hdmi";
+ reg = <0x8d04000 0x1000>;
+ reg-names = "hdmi-reg";
+ interrupts = <GIC_SPI 106 IRQ_TYPE_NONE>;
+ interrupt-names = "irq";
+ clock-names = "pix",
+ "tmds",
+ "phy",
+ "audio",
+ "main_parent",
+ "aux_parent";
+
+ clocks = <&clk_s_d2_flexgen CLK_PIX_HDMI>,
+ <&clk_s_d2_flexgen CLK_TMDS_HDMI>,
+ <&clk_s_d2_flexgen CLK_REF_HDMIPHY>,
+ <&clk_s_d0_flexgen CLK_PCM_0>,
+ <&clk_s_d2_quadfs 0>,
+ <&clk_s_d2_quadfs 1>;
+
+ hdmi,hpd-gpio = <&pio5 3>;
+ reset-names = "hdmi";
+ resets = <&softreset STIH407_HDMI_TX_PHY_SOFTRESET>;
+ ddc = <&hdmiddc>;
+ };
+
+ sti-hda@8d02000 {
+ compatible = "st,stih407-hda";
+ reg = <0x8d02000 0x400>, <0x92b0120 0x4>;
+ reg-names = "hda-reg", "video-dacs-ctrl";
+ clock-names = "pix",
+ "hddac",
+ "main_parent",
+ "aux_parent";
+ clocks = <&clk_s_d2_flexgen CLK_PIX_HDDAC>,
+ <&clk_s_d2_flexgen CLK_HDDAC>,
+ <&clk_s_d2_quadfs 0>,
+ <&clk_s_d2_quadfs 1>;
};
};
720000 1200000
528000 1100000
312000 1000000
- 144000 900000
+ 144000 1000000
>;
#cooling-cells = <2>;
cooling-min-level = <0>;
}
}
-void it8152_irq_demux(unsigned int irq, struct irq_desc *desc)
+void it8152_irq_demux(struct irq_desc *desc)
{
int bits_pd, bits_lp, bits_ld;
int i;
},
};
-static void locomo_handler(unsigned int __irq, struct irq_desc *desc)
+static void locomo_handler(struct irq_desc *desc)
{
struct locomo *lchip = irq_desc_get_chip_data(desc);
int req, i;
* active IRQs causes the interrupt output to pulse, the upper levels
* will call us again if there are more interrupts to process.
*/
-static void
-sa1111_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void sa1111_irq_handler(struct irq_desc *desc)
{
- unsigned int irq = irq_desc_get_irq(desc);
unsigned int stat0, stat1, i;
struct sa1111 *sachip = irq_desc_get_handler_data(desc);
void __iomem *mapbase = sachip->base + SA1111_INTC;
sa1111_writel(stat1, mapbase + SA1111_INTSTATCLR1);
if (stat0 == 0 && stat1 == 0) {
- do_bad_IRQ(irq, desc);
+ do_bad_IRQ(desc);
return;
}
CONFIG_DRM_EXYNOS=y
CONFIG_DRM_EXYNOS_FIMD=y
CONFIG_DRM_EXYNOS_DSI=y
+CONFIG_DRM_EXYNOS_MIXER=y
CONFIG_DRM_EXYNOS_HDMI=y
CONFIG_DRM_PANEL_SIMPLE=y
CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=y
CONFIG_PINCTRL_SINGLE=y
CONFIG_DEBUG_GPIO=y
CONFIG_GPIO_SYSFS=y
-CONFIG_GPIO_PCF857X=m
+CONFIG_GPIO_PCA953X=m
+CONFIG_GPIO_PCF857X=y
CONFIG_GPIO_TWL4030=y
CONFIG_GPIO_PALMAS=y
CONFIG_W1=m
CONFIG_USB_MUSB_OMAP2PLUS=m
CONFIG_USB_MUSB_AM35X=m
CONFIG_USB_MUSB_DSPS=m
+CONFIG_USB_INVENTRA_DMA=y
+CONFIG_USB_TI_CPPI41_DMA=y
CONFIG_USB_DWC3=m
CONFIG_USB_TEST=m
CONFIG_AM335X_PHY_USB=y
#endif
.endm
- .macro uaccess_save_and_disable, tmp
- uaccess_save \tmp
- uaccess_disable \tmp
- .endm
-
.irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo
.macro ret\c, reg
#if __LINUX_ARM_ARCH__ < 6
"2:\t.asciz " #__file "\n" \
".popsection\n" \
".pushsection __bug_table,\"a\"\n" \
+ ".align 2\n" \
"3:\t.word 1b, 2b\n" \
"\t.hword " #__line ", 0\n" \
".popsection"); \
#ifndef __ASSEMBLY__
#include <asm/barrier.h>
+#include <asm/thread_info.h>
#endif
/*
asm(
"mrc p15, 0, %0, c3, c0 @ get domain"
- : "=r" (domain));
+ : "=r" (domain)
+ : "m" (current_thread_info()->cpu_domain));
return domain;
}
{
asm volatile(
"mcr p15, 0, %0, c3, c0 @ set domain"
- : : "r" (val));
+ : : "r" (val) : "memory");
isb();
}
struct pci_dev;
struct pci_sys_data;
-extern void it8152_irq_demux(unsigned int irq, struct irq_desc *desc);
+extern void it8152_irq_demux(struct irq_desc *desc);
extern void it8152_init_irq(void);
extern int it8152_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin);
extern int it8152_pci_setup(int nr, struct pci_sys_data *sys);
pr_crit("unexpected IRQ trap at vector %02x\n", irq);
}
-void set_irq_flags(unsigned int irq, unsigned int flags);
-
-#define IRQF_VALID (1 << 0)
-#define IRQF_PROBE (1 << 1)
-#define IRQF_NOAUTOEN (1 << 2)
-
#define ARCH_IRQ_INIT_FLAGS (IRQ_NOREQUEST | IRQ_NOPROBE)
#endif
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
-#if defined(CONFIG_KVM_ARM_MAX_VCPUS)
-#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
-#else
-#define KVM_MAX_VCPUS 0
-#endif
-
#define KVM_USER_MEM_SLOTS 32
#define KVM_PRIVATE_MEM_SLOTS 4
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
#define KVM_HAVE_ONE_REG
+#define KVM_HALT_POLL_NS_DEFAULT 500000
#define KVM_VCPU_MAX_FEATURES 2
#include <kvm/arm_vgic.h>
+#define KVM_MAX_VCPUS VGIC_V2_MAX_CPUS
+
u32 *kvm_vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num, u32 mode);
int __attribute_const__ kvm_target_cpu(void);
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
struct kvm_vcpu_stat {
u32 halt_successful_poll;
+ u32 halt_attempted_poll;
u32 halt_wakeup;
};
/*
* This is for easy migration, but should be changed in the source
*/
-#define do_bad_IRQ(irq,desc) \
+#define do_bad_IRQ(desc) \
do { \
raw_spin_lock(&desc->lock); \
- handle_bad_irq(irq, desc); \
+ handle_bad_irq(desc); \
raw_spin_unlock(&desc->lock); \
} while(0)
struct task_struct;
#include <asm/types.h>
-#include <asm/domain.h>
typedef unsigned long mm_segment_t;
* This may need to be greater than __NR_last_syscall+1 in order to
* account for the padding in the syscall table
*/
-#define __NR_syscalls (388)
+#define __NR_syscalls (392)
/*
* *NOTE*: This is a ghost syscall private to the kernel. Only the
#define __NR_memfd_create (__NR_SYSCALL_BASE+385)
#define __NR_bpf (__NR_SYSCALL_BASE+386)
#define __NR_execveat (__NR_SYSCALL_BASE+387)
+#define __NR_userfaultfd (__NR_SYSCALL_BASE+388)
+#define __NR_membarrier (__NR_SYSCALL_BASE+389)
/*
* The following SWIs are ARM private.
/* 385 */ CALL(sys_memfd_create)
CALL(sys_bpf)
CALL(sys_execveat)
+ CALL(sys_userfaultfd)
+ CALL(sys_membarrier)
#ifndef syscalls_counted
.equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls
#define syscalls_counted
handle_IRQ(irq, regs);
}
-void set_irq_flags(unsigned int irq, unsigned int iflags)
-{
- unsigned long clr = 0, set = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN;
-
- if (irq >= nr_irqs) {
- pr_err("Trying to set irq flags for IRQ%d\n", irq);
- return;
- }
-
- if (iflags & IRQF_VALID)
- clr |= IRQ_NOREQUEST;
- if (iflags & IRQF_PROBE)
- clr |= IRQ_NOPROBE;
- if (!(iflags & IRQF_NOAUTOEN))
- clr |= IRQ_NOAUTOEN;
- /* Order is clear bits in "clr" then set bits in "set" */
- irq_modify_status(irq, clr, set & ~clr);
-}
-EXPORT_SYMBOL_GPL(set_irq_flags);
-
void __init init_IRQ(void)
{
int ret;
if (err)
return err;
- patch_text((void *)bpt->bpt_addr,
- *(unsigned int *)arch_kgdb_ops.gdb_bpt_instr);
+ /* Machine is already stopped, so we can use __patch_text() directly */
+ __patch_text((void *)bpt->bpt_addr,
+ *(unsigned int *)arch_kgdb_ops.gdb_bpt_instr);
return err;
}
int kgdb_arch_remove_breakpoint(struct kgdb_bkpt *bpt)
{
- patch_text((void *)bpt->bpt_addr, *(unsigned int *)bpt->saved_instr);
+ /* Machine is already stopped, so we can use __patch_text() directly */
+ __patch_text((void *)bpt->bpt_addr, *(unsigned int *)bpt->saved_instr);
return 0;
}
memset(&thread->cpu_context, 0, sizeof(struct cpu_context_save));
+#ifdef CONFIG_CPU_USE_DOMAINS
/*
* Copy the initial value of the domain access control register
* from the current thread: thread->addr_limit will have been
* kernel/fork.c
*/
thread->cpu_domain = get_domain();
+#endif
if (likely(!(p->flags & PF_KTHREAD))) {
*childregs = *current_pt_regs();
*/
thumb = handler & 1;
-#if __LINUX_ARM_ARCH__ >= 7
/*
- * Clear the If-Then Thumb-2 execution state
- * ARM spec requires this to be all 000s in ARM mode
- * Snapdragon S4/Krait misbehaves on a Thumb=>ARM
- * signal transition without this.
+ * Clear the If-Then Thumb-2 execution state. ARM spec
+ * requires this to be all 000s in ARM mode. Snapdragon
+ * S4/Krait misbehaves on a Thumb=>ARM signal transition
+ * without this.
+ *
+ * We must do this whenever we are running on a Thumb-2
+ * capable CPU, which includes ARMv6T2. However, we elect
+ * to always do this to simplify the code; this field is
+ * marked UNK/SBZP for older architectures.
*/
cpsr &= ~PSR_IT_MASK;
-#endif
if (thumb) {
cpsr |= PSR_T_BIT;
---help---
Provides host support for ARM processors.
-config KVM_ARM_MAX_VCPUS
- int "Number maximum supported virtual CPUs per VM"
- depends on KVM_ARM_HOST
- default 4
- help
- Static number of max supported virtual CPUs per VM.
-
- If you choose a high number, the vcpu structures will be quite
- large, so only choose a reasonable number that you expect to
- actually use.
-
endif # VIRTUALIZATION
* Map the VGIC hardware resources before running a vcpu the first
* time on this VM.
*/
- if (unlikely(!vgic_ready(kvm))) {
+ if (unlikely(irqchip_in_kernel(kvm) && !vgic_ready(kvm))) {
ret = kvm_vgic_map_resources(kvm);
if (ret)
return ret;
mrc p15, 0, r2, c14, c3, 1 @ CNTV_CTL
str r2, [vcpu, #VCPU_TIMER_CNTV_CTL]
- bic r2, #1 @ Clear ENABLE
- mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL
+
isb
mrrc p15, 3, rr_lo_hi(r2, r3), c14 @ CNTV_CVAL
mcrr p15, 4, r2, r2, c14 @ CNTVOFF
1:
+ mov r2, #0 @ Clear ENABLE
+ mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL
+
@ Allow physical timer/counter access for the host
mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL
orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN)
if (vma->vm_flags & VM_PFNMAP) {
gpa_t gpa = mem->guest_phys_addr +
(vm_start - mem->userspace_addr);
- phys_addr_t pa = (vma->vm_pgoff << PAGE_SHIFT) +
- vm_start - vma->vm_start;
+ phys_addr_t pa;
+
+ pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
+ pa += vm_start - vma->vm_start;
/* IO region dirty page logging not allowed */
if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES)
static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
{
- int i;
+ int i, matching_cpus = 0;
unsigned long mpidr;
unsigned long target_affinity;
unsigned long target_affinity_mask;
*/
kvm_for_each_vcpu(i, tmp, kvm) {
mpidr = kvm_vcpu_get_mpidr_aff(tmp);
- if (((mpidr & target_affinity_mask) == target_affinity) &&
- !tmp->arch.pause) {
- return PSCI_0_2_AFFINITY_LEVEL_ON;
+ if ((mpidr & target_affinity_mask) == target_affinity) {
+ matching_cpus++;
+ if (!tmp->arch.pause)
+ return PSCI_0_2_AFFINITY_LEVEL_ON;
}
}
+ if (!matching_cpus)
+ return PSCI_RET_INVALID_PARAMS;
+
return PSCI_0_2_AFFINITY_LEVEL_OFF;
}
.irq_ack = pmu_irq_ack,
};
-static void pmu_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void pmu_irq_handler(struct irq_desc *desc)
{
- unsigned int irq = irq_desc_get_irq(desc);
unsigned long cause = readl(PMU_INTERRUPT_CAUSE);
+ unsigned int irq;
cause &= readl(PMU_INTERRUPT_MASK);
if (cause == 0) {
- do_bad_IRQ(irq, desc);
+ do_bad_IRQ(desc);
return;
}
#include <asm/cputype.h>
#include <asm/cp15.h>
#include <asm/mcpm.h>
+#include <asm/smp_plat.h>
#include "regs-pmu.h"
#include "common.h"
cluster >= EXYNOS5420_NR_CLUSTERS)
return -EINVAL;
- exynos_cpu_power_up(cpunr);
+ if (!exynos_cpu_power_state(cpunr)) {
+ exynos_cpu_power_up(cpunr);
+
+ /*
+ * This assumes the cluster number of the big cores(Cortex A15)
+ * is 0 and the Little cores(Cortex A7) is 1.
+ * When the system was booted from the Little core,
+ * they should be reset during power up cpu.
+ */
+ if (cluster &&
+ cluster == MPIDR_AFFINITY_LEVEL(cpu_logical_map(0), 1)) {
+ /*
+ * Before we reset the Little cores, we should wait
+ * the SPARE2 register is set to 1 because the init
+ * codes of the iROM will set the register after
+ * initialization.
+ */
+ while (!pmu_raw_readl(S5P_PMU_SPARE2))
+ udelay(10);
+
+ pmu_raw_writel(EXYNOS5420_KFC_CORE_RESET(cpu),
+ EXYNOS_SWRESET);
+ }
+ }
+
return 0;
}
#define SPREAD_ENABLE 0xF
#define SPREAD_USE_STANDWFI 0xF
+#define EXYNOS5420_KFC_CORE_RESET0 BIT(8)
+#define EXYNOS5420_KFC_ETM_RESET0 BIT(20)
+
+#define EXYNOS5420_KFC_CORE_RESET(_nr) \
+ ((EXYNOS5420_KFC_CORE_RESET0 | EXYNOS5420_KFC_ETM_RESET0) << (_nr))
+
#define EXYNOS5420_BB_CON1 0x0784
#define EXYNOS5420_BB_SEL_EN BIT(31)
#define EXYNOS5420_BB_PMOS_EN BIT(7)
.irq_unmask = isa_unmask_pic_hi_irq,
};
-static void
-isa_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void isa_irq_handler(struct irq_desc *desc)
{
unsigned int isa_irq = *(unsigned char *)PCIIACK_BASE;
if (isa_irq < _ISA_IRQ(0) || isa_irq >= _ISA_IRQ(16)) {
- do_bad_IRQ(isa_irq, desc);
+ do_bad_IRQ(desc);
return;
}
return 0;
}
-static void gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void gpio_irq_handler(struct irq_desc *desc)
{
unsigned int port = (unsigned int)irq_desc_get_handler_data(desc);
unsigned int gpio_irq_no, irq_stat;
.resource = smsc911x_resources,
};
-static void mxc_expio_irq_handler(u32 irq, struct irq_desc *desc)
+static void mxc_expio_irq_handler(struct irq_desc *desc)
{
u32 imr_val;
u32 int_valid;
imx31_add_imx_uart0(&uart_pdata);
}
-static void mx31ads_expio_irq_handler(u32 irq, struct irq_desc *desc)
+static void mx31ads_expio_irq_handler(struct irq_desc *desc)
{
u32 imr_val;
u32 int_valid;
write_imipr_3,
};
-static void iop13xx_msi_handler(unsigned int irq, struct irq_desc *desc)
+static void iop13xx_msi_handler(struct irq_desc *desc)
{
int i, j;
unsigned long status;
.irq_set_wake = lpc32xx_irq_wake
};
-static void lpc32xx_sic1_handler(unsigned int irq, struct irq_desc *desc)
+static void lpc32xx_sic1_handler(struct irq_desc *desc)
{
unsigned long ints = __raw_readl(LPC32XX_INTC_STAT(LPC32XX_SIC1_BASE));
}
}
-static void lpc32xx_sic2_handler(unsigned int irq, struct irq_desc *desc)
+static void lpc32xx_sic2_handler(struct irq_desc *desc)
{
unsigned long ints = __raw_readl(LPC32XX_INTC_STAT(LPC32XX_SIC2_BASE));
#define DEBUG_IRQ(fmt...) while (0) {}
#endif
-static void
-netx_hif_demux_handler(unsigned int irq_unused, struct irq_desc *desc)
+static void netx_hif_demux_handler(struct irq_desc *desc)
{
unsigned int irq = NETX_IRQ_HIF_CHAINED(0);
unsigned int stat;
fpga_ack_irq(d);
}
-static void innovator_fpga_IRQ_demux(unsigned int irq, struct irq_desc *desc)
+static void innovator_fpga_IRQ_demux(struct irq_desc *desc)
{
u32 stat;
int fpga_irq;
select ARM_CPU_SUSPEND if PM
select ARM_GIC
select HAVE_ARM_SCU if SMP
- select HAVE_ARM_TWD if SMP
select HAVE_ARM_ARCH_TIMER
select ARM_ERRATA_798181 if SMP
+ select OMAP_INTERCONNECT
select OMAP_INTERCONNECT_BARRIER
+ select PM_OPP if PM
config SOC_AM33XX
bool "TI AM33XX"
select ARCH_OMAP2PLUS
select ARM_CPU_SUSPEND if PM
select ARM_GIC
+ select HAVE_ARM_SCU if SMP
select HAVE_ARM_ARCH_TIMER
select IRQ_CROSSBAR
select ARM_ERRATA_798181 if SMP
+ select OMAP_INTERCONNECT
select OMAP_INTERCONNECT_BARRIER
+ select PM_OPP if PM
config ARCH_OMAP2PLUS
bool
#include "common.h"
-#if !(defined(CONFIG_ARCH_OMAP2) || defined(CONFIG_ARCH_OMAP3))
-#define intc_of_init NULL
-#endif
-#ifndef CONFIG_ARCH_OMAP4
-#define gic_of_init NULL
-#endif
-
static const struct of_device_id omap_dt_match_table[] __initconst = {
{ .compatible = "simple-bus", },
{ .compatible = "ti,omap-infra", },
omap_revision = DRA752_REV_ES1_0;
break;
case 1:
- default:
omap_revision = DRA752_REV_ES1_1;
+ break;
+ case 2:
+ default:
+ omap_revision = DRA752_REV_ES2_0;
+ break;
}
break;
/* Unknown default to latest silicon rev as default*/
pr_warn("%s: unknown idcode=0x%08x (hawkeye=0x%08x,rev=0x%x)\n",
__func__, idcode, hawkeye, rev);
- omap_revision = DRA752_REV_ES1_1;
+ omap_revision = DRA752_REV_ES2_0;
}
sprintf(soc_name, "DRA%03x", omap_rev() >> 16);
void __init am43xx_init_late(void)
{
omap_common_late_init();
+ omap2_clk_enable_autoidle_all();
}
#endif
if (od->hwmods[i]->flags & HWMOD_INIT_NO_IDLE)
return 0;
- if (od->_driver_status != BUS_NOTIFY_BOUND_DRIVER) {
+ if (od->_driver_status != BUS_NOTIFY_BOUND_DRIVER &&
+ od->_driver_status != BUS_NOTIFY_BIND_DRIVER) {
if (od->_state == OMAP_DEVICE_STATE_ENABLED) {
dev_warn(dev, "%s: enabled but no driver. Idling\n",
__func__);
#define PM_OMAP4_ROM_SMP_BOOT_ERRATUM_GICD (1 << 0)
#define PM_OMAP4_CPU_OSWR_DISABLE (1 << 1)
-#if defined(CONFIG_PM) && defined(CONFIG_ARCH_OMAP4)
+#if defined(CONFIG_PM) && (defined(CONFIG_ARCH_OMAP4) ||\
+ defined(CONFIG_SOC_OMAP5) || defined(CONFIG_SOC_DRA7XX))
extern u16 pm44xx_errata;
#define IS_PM44XX_ERRATUM(id) (pm44xx_errata & (id))
#else
* dispatched accordingly. Clearing of the wakeup events should be
* done by the SoC specific individual handlers.
*/
-static void omap_prcm_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void omap_prcm_irq_handler(struct irq_desc *desc)
{
unsigned long pending[OMAP_PRCM_MAX_NR_PENDING_REG];
unsigned long priority_pending[OMAP_PRCM_MAX_NR_PENDING_REG];
#define DRA7XX_CLASS 0x07000000
#define DRA752_REV_ES1_0 (DRA7XX_CLASS | (0x52 << 16) | (0x10 << 8))
#define DRA752_REV_ES1_1 (DRA7XX_CLASS | (0x52 << 16) | (0x11 << 8))
+#define DRA752_REV_ES2_0 (DRA7XX_CLASS | (0x52 << 16) | (0x20 << 8))
+#define DRA722_REV_ES1_0 (DRA7XX_CLASS | (0x22 << 16) | (0x10 << 8))
#define DRA722_REV_ES1_0 (DRA7XX_CLASS | (0x22 << 16) | (0x10 << 8))
void omap2xxx_check_revision(void);
if (IS_ERR(src))
return PTR_ERR(src);
- r = clk_set_parent(timer->fclk, src);
- if (r < 0) {
- pr_warn("%s: %s cannot set source\n", __func__, oh->name);
- clk_put(src);
- return r;
- }
+ WARN(clk_set_parent(timer->fclk, src) < 0,
+ "Cannot set timer parent clock, no PLL clock driver?");
clk_put(src);
val = voltdm->read(OMAP3_PRM_POLCTRL_OFFSET);
if (!(val & OMAP3430_PRM_POLCTRL_CLKREQ_POL) ||
- (val & OMAP3430_PRM_POLCTRL_CLKREQ_POL)) {
+ (val & OMAP3430_PRM_POLCTRL_OFFMODE_POL)) {
val |= OMAP3430_PRM_POLCTRL_CLKREQ_POL;
val &= ~OMAP3430_PRM_POLCTRL_OFFMODE_POL;
pr_debug("PM: fixing sys_clkreq and sys_off_mode polarity to 0x%x\n",
.irq_unmask = balloon3_unmask_irq,
};
-static void balloon3_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void balloon3_irq_handler(struct irq_desc *desc)
{
unsigned long pending = __raw_readl(BALLOON3_INT_CONTROL_REG) &
balloon3_irq_enabled;
do {
struct irq_data *d = irq_desc_get_irq_data(desc);
- struct irq_chip *chip = irq_data_get_chip(d);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int irq;
/* clear useless edge notification */
void __iomem *it8152_base_address;
static int cmx2xx_it8152_irq_gpio;
-static void cmx2xx_it8152_irq_demux(unsigned int __irq, struct irq_desc *desc)
+static void cmx2xx_it8152_irq_demux(struct irq_desc *desc)
{
- unsigned int irq = irq_desc_get_irq(desc);
/* clear our parent irq */
desc->irq_data.chip->irq_ack(&desc->irq_data);
- it8152_irq_demux(irq, desc);
+ it8152_irq_demux(desc);
}
void __cmx2xx_pci_init_irq(int irq_gpio)
* 0xf6200000..0xf6201000
*/
+/*
+ * DFI Bus for NAND, PXA3xx only
+ */
+#define NAND_PHYS 0x43100000
+#define NAND_VIRT IOMEM(0xf6300000)
+#define NAND_SIZE 0x00100000
+
/*
* Internal Memory Controller (PXA27x and later)
*/
.irq_unmask = lpd270_unmask_irq,
};
-static void lpd270_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void lpd270_irq_handler(struct irq_desc *desc)
{
unsigned int irq;
unsigned long pending;
.irq_unmask = pcm990_unmask_irq,
};
-static void pcm990_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void pcm990_irq_handler(struct irq_desc *desc)
{
unsigned int irq;
unsigned long pending;
#define ISRAM_START 0x5c000000
#define ISRAM_SIZE SZ_256K
+/*
+ * NAND NFC: DFI bus arbitration subset
+ */
+#define NDCR (*(volatile u32 __iomem*)(NAND_VIRT + 0))
+#define NDCR_ND_ARB_EN (1 << 12)
+#define NDCR_ND_ARB_CNTL (1 << 19)
+
static void __iomem *sram;
static unsigned long wakeup_src;
.pfn = __phys_to_pfn(PXA3XX_SMEMC_BASE),
.length = SMEMC_SIZE,
.type = MT_DEVICE
- }
+ }, {
+ .virtual = (unsigned long)NAND_VIRT,
+ .pfn = __phys_to_pfn(NAND_PHYS),
+ .length = NAND_SIZE,
+ .type = MT_DEVICE
+ },
};
void __init pxa3xx_map_io(void)
*/
ASCR &= ~(ASCR_RDH | ASCR_D1S | ASCR_D2S | ASCR_D3S);
+ /*
+ * Disable DFI bus arbitration, to prevent a system bus lock if
+ * somebody disables the NAND clock (unused clock) while this
+ * bit remains set.
+ */
+ NDCR = (NDCR & ~NDCR_ND_ARB_EN) | NDCR_ND_ARB_CNTL;
+
if ((ret = pxa_init_dma(IRQ_DMA, 32)))
return ret;
viper_irq_enabled_mask;
}
-static void viper_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void viper_irq_handler(struct irq_desc *desc)
{
unsigned int irq;
unsigned long pending;
return __raw_readw(ZEUS_CPLD_ISA_IRQ) & zeus_irq_enabled_mask;
}
-static void zeus_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void zeus_irq_handler(struct irq_desc *desc)
{
unsigned int irq;
unsigned long pending;
}
}
-static void
-ecard_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ecard_irq_handler(struct irq_desc *desc)
{
ecard_t *ec;
int called = 0;
.irq_ack = bast_pc104_maskack
};
-static void
-bast_irq_pc104_demux(unsigned int irq,
- struct irq_desc *desc)
+static void bast_irq_pc104_demux(struct irq_desc *desc)
{
unsigned int stat;
unsigned int irqno;
}
}
-static void s3c_irq_demux_eint0_3(unsigned int irq, struct irq_desc *desc)
+static void s3c_irq_demux_eint0_3(struct irq_desc *desc)
{
s3c_irq_demux_eint(0, 3);
}
-static void s3c_irq_demux_eint4_11(unsigned int irq, struct irq_desc *desc)
+static void s3c_irq_demux_eint4_11(struct irq_desc *desc)
{
s3c_irq_demux_eint(4, 11);
}
-static void s3c_irq_demux_eint12_19(unsigned int irq, struct irq_desc *desc)
+static void s3c_irq_demux_eint12_19(struct irq_desc *desc)
{
s3c_irq_demux_eint(12, 19);
}
-static void s3c_irq_demux_eint20_27(unsigned int irq, struct irq_desc *desc)
+static void s3c_irq_demux_eint20_27(struct irq_desc *desc)
{
s3c_irq_demux_eint(20, 27);
}
* ensure that the IRQ signal is deasserted before returning. This
* is rather unfortunate.
*/
-static void neponset_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void neponset_irq_handler(struct irq_desc *desc)
{
struct neponset_drvdata *d = irq_desc_get_handler_data(desc);
unsigned int irr;
user:
if (LDST_L_BIT(instr)) {
unsigned long val;
+ unsigned int __ua_flags = uaccess_save_and_enable();
+
get16t_unaligned_check(val, addr);
+ uaccess_restore(__ua_flags);
/* signed half-word? */
if (instr & 0x40)
val = (signed long)((signed short) val);
regs->uregs[rd] = val;
- } else
+ } else {
+ unsigned int __ua_flags = uaccess_save_and_enable();
put16t_unaligned_check(regs->uregs[rd], addr);
+ uaccess_restore(__ua_flags);
+ }
return TYPE_LDST;
user:
if (load) {
- unsigned long val;
+ unsigned long val, val2;
+ unsigned int __ua_flags = uaccess_save_and_enable();
+
get32t_unaligned_check(val, addr);
+ get32t_unaligned_check(val2, addr + 4);
+
+ uaccess_restore(__ua_flags);
+
regs->uregs[rd] = val;
- get32t_unaligned_check(val, addr + 4);
- regs->uregs[rd2] = val;
+ regs->uregs[rd2] = val2;
} else {
+ unsigned int __ua_flags = uaccess_save_and_enable();
put32t_unaligned_check(regs->uregs[rd], addr);
put32t_unaligned_check(regs->uregs[rd2], addr + 4);
+ uaccess_restore(__ua_flags);
}
return TYPE_LDST;
trans:
if (LDST_L_BIT(instr)) {
unsigned int val;
+ unsigned int __ua_flags = uaccess_save_and_enable();
get32t_unaligned_check(val, addr);
+ uaccess_restore(__ua_flags);
regs->uregs[rd] = val;
- } else
+ } else {
+ unsigned int __ua_flags = uaccess_save_and_enable();
put32t_unaligned_check(regs->uregs[rd], addr);
+ uaccess_restore(__ua_flags);
+ }
return TYPE_LDST;
fault:
#endif
if (user_mode(regs)) {
+ unsigned int __ua_flags = uaccess_save_and_enable();
for (regbits = REGMASK_BITS(instr), rd = 0; regbits;
regbits >>= 1, rd += 1)
if (regbits & 1) {
put32t_unaligned_check(regs->uregs[rd], eaddr);
eaddr += 4;
}
+ uaccess_restore(__ua_flags);
} else {
for (regbits = REGMASK_BITS(instr), rd = 0; regbits;
regbits >>= 1, rd += 1)
struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
dma_addr_t dma_addr, iova;
- int i, ret = DMA_ERROR_CODE;
+ int i;
dma_addr = __alloc_iova(mapping, size);
if (dma_addr == DMA_ERROR_CODE)
iova = dma_addr;
for (i = 0; i < count; ) {
+ int ret;
+
unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
phys_addr_t phys = page_to_phys(pages[i]);
unsigned int len, j;
reteq r4 @ no, return failure
next:
+ uaccess_enable r3
.Lx1: ldrt r6, [r5], #4 @ get the next instruction and
@ increment PC
-
+ uaccess_disable r3
and r2, r6, #0x0F000000 @ test for FP insns
teq r2, #0x0C000000
teqne r2, #0x0D000000
return 0;
}
-static void gpio_irq_handler(unsigned __irq, struct irq_desc *desc)
+static void gpio_irq_handler(struct irq_desc *desc)
{
struct orion_gpio_chip *ochip = irq_desc_get_handler_data(desc);
u32 cause, type;
{ .compatible = "mvrl,pxa168-ssp", .data = (void *) PXA168_SSP },
{ .compatible = "mrvl,pxa910-ssp", .data = (void *) PXA910_SSP },
{ .compatible = "mrvl,ce4100-ssp", .data = (void *) CE4100_SSP },
- { .compatible = "mrvl,lpss-ssp", .data = (void *) LPSS_SSP },
{ },
};
MODULE_DEVICE_TABLE(of, pxa_ssp_of_ids);
mov r1, r2
mov r2, r3
ldr r3, [sp, #8]
+ /*
+ * Privcmd calls are issued by the userspace. We need to allow the
+ * kernel to access the userspace memory before issuing the hypercall.
+ */
+ uaccess_enable r4
+
+ /* r4 is loaded now as we use it as scratch register before */
ldr r4, [sp, #4]
__HVC(XEN_IMM)
+
+ /*
+ * Disable userspace access from kernel. This is fine to do it
+ * unconditionally as no set_fs(KERNEL_DS)/set_fs(get_ds()) is
+ * called before.
+ */
+ uaccess_disable r4
+
ldm sp!, {r4}
ret lr
ENDPROC(privcmd_call);
select GENERIC_CLOCKEVENTS_BROADCAST
select GENERIC_CPU_AUTOPROBE
select GENERIC_EARLY_IOREMAP
+ select GENERIC_IDLE_POLL_SETUP
select GENERIC_IRQ_PROBE
select GENERIC_IRQ_SHOW
select GENERIC_IRQ_SHOW_LEVEL
If unsure, say Y.
+config ARM64_ERRATUM_843419
+ bool "Cortex-A53: 843419: A load or store might access an incorrect address"
+ depends on MODULES
+ default y
+ help
+ This option builds kernel modules using the large memory model in
+ order to avoid the use of the ADRP instruction, which can cause
+ a subsequent memory access to use an incorrect address on Cortex-A53
+ parts up to r0p4.
+
+ Note that the kernel itself must be linked with a version of ld
+ which fixes potentially affected ADRP instructions through the
+ use of veneers.
+
+ If unsure, say Y.
+
endmenu
CHECKFLAGS += -D__aarch64__
+ifeq ($(CONFIG_ARM64_ERRATUM_843419), y)
+CFLAGS_MODULE += -mcmodel=large
+endif
+
# Default value
head-y := arch/arm64/kernel/head.o
};
idle-states {
- entry-method = "arm,psci";
+ entry-method = "psci";
CPU_SLEEP_0: cpu-sleep-0 {
compatible = "arm,idle-state";
};
idle-states {
- entry-method = "arm,psci";
+ entry-method = "psci";
cpu_sleep: cpu-sleep-0 {
compatible = "arm,idle-state";
irq_err_count++;
}
-/*
- * No arch-specific IRQ flags.
- */
-#define set_irq_flags(irq, flags)
-
#endif /* __ASM_HARDIRQ_H */
SCTLR_EL2_SA | SCTLR_EL2_I)
/* TCR_EL2 Registers bits */
+#define TCR_EL2_RES1 ((1 << 31) | (1 << 23))
#define TCR_EL2_TBI (1 << 20)
#define TCR_EL2_PS (7 << 16)
#define TCR_EL2_PS_40B (2 << 16)
#define TCR_EL2_MASK (TCR_EL2_TG0 | TCR_EL2_SH0 | \
TCR_EL2_ORGN0 | TCR_EL2_IRGN0 | TCR_EL2_T0SZ)
-#define TCR_EL2_FLAGS (TCR_EL2_PS_40B)
+#define TCR_EL2_FLAGS (TCR_EL2_RES1 | TCR_EL2_PS_40B)
/* VTCR_EL2 Registers bits */
+#define VTCR_EL2_RES1 (1 << 31)
#define VTCR_EL2_PS_MASK (7 << 16)
#define VTCR_EL2_TG0_MASK (1 << 14)
#define VTCR_EL2_TG0_4K (0 << 14)
*/
#define VTCR_EL2_FLAGS (VTCR_EL2_TG0_64K | VTCR_EL2_SH0_INNER | \
VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \
- VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B)
+ VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B | \
+ VTCR_EL2_RES1)
#define VTTBR_X (38 - VTCR_EL2_T0SZ_40B)
#else
/*
*/
#define VTCR_EL2_FLAGS (VTCR_EL2_TG0_4K | VTCR_EL2_SH0_INNER | \
VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \
- VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B)
+ VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B | \
+ VTCR_EL2_RES1)
#define VTTBR_X (37 - VTCR_EL2_T0SZ_40B)
#endif
#define VTTBR_VMID_MASK (UL(0xFF) << VTTBR_VMID_SHIFT)
/* Hyp System Trap Register */
-#define HSTR_EL2_TTEE (1 << 16)
#define HSTR_EL2_T(x) (1 << x)
/* Hyp Coproccessor Trap Register Shifts */
#define IFSR32_EL2 25 /* Instruction Fault Status Register */
#define FPEXC32_EL2 26 /* Floating-Point Exception Control Register */
#define DBGVCR32_EL2 27 /* Debug Vector Catch Register */
-#define TEECR32_EL1 28 /* ThumbEE Configuration Register */
-#define TEEHBR32_EL1 29 /* ThumbEE Handler Base Register */
-#define NR_SYS_REGS 30
+#define NR_SYS_REGS 28
/* 32bit mapping */
#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
-#if defined(CONFIG_KVM_ARM_MAX_VCPUS)
-#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
-#else
-#define KVM_MAX_VCPUS 0
-#endif
-
#define KVM_USER_MEM_SLOTS 32
#define KVM_PRIVATE_MEM_SLOTS 4
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
+#define KVM_HALT_POLL_NS_DEFAULT 500000
#include <kvm/arm_vgic.h>
#include <kvm/arm_arch_timer.h>
+#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
+
#define KVM_VCPU_MAX_FEATURES 3
int __attribute_const__ kvm_target_cpu(void);
struct kvm_vcpu_stat {
u32 halt_successful_poll;
+ u32 halt_attempted_poll;
u32 halt_wakeup;
};
* Software defined PTE bits definition.
*/
#define PTE_VALID (_AT(pteval_t, 1) << 0)
+#define PTE_WRITE (PTE_DBM) /* same as DBM (51) */
#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)
-#ifdef CONFIG_ARM64_HW_AFDBM
-#define PTE_WRITE (PTE_DBM) /* same as DBM */
-#else
-#define PTE_WRITE (_AT(pteval_t, 1) << 57)
-#endif
#define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */
/*
#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
-#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_TYPE_MASK) | PTE_PROT_NONE | PTE_PXN | PTE_UXN)
+#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN)
#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
#define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
#define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
#define pte_exec(pte) (!(pte_val(pte) & PTE_UXN))
#ifdef CONFIG_ARM64_HW_AFDBM
-#define pte_hw_dirty(pte) (!(pte_val(pte) & PTE_RDONLY))
+#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
#else
#define pte_hw_dirty(pte) (0)
#endif
* When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
* the page fault mechanism. Checking the dirty status of a pte becomes:
*
- * PTE_DIRTY || !PTE_RDONLY
+ * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
*/
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte)
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
- PTE_PROT_NONE | PTE_WRITE | PTE_TYPE_MASK;
+ PTE_PROT_NONE | PTE_VALID | PTE_WRITE;
/* preserve the hardware dirty information */
if (pte_hw_dirty(pte))
- newprot |= PTE_DIRTY;
+ pte = pte_mkdirty(pte);
pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
return pte;
}
unsigned long action, void *data)
{
int cpu = (unsigned long)data;
- if (action == CPU_ONLINE)
+ if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE)
smp_call_function_single(cpu, clear_os_lock, NULL, 1);
return NOTIFY_OK;
}
}
/*
- * Call registered single step handers
+ * Call registered single step handlers
* There is no Syndrome info to check for determining the handler.
* So we call all the registered handlers, until the right handler is
* found which returns zero.
* Use reader/writer locks instead of plain spinlock.
*/
static LIST_HEAD(break_hook);
-static DEFINE_RWLOCK(break_hook_lock);
+static DEFINE_SPINLOCK(break_hook_lock);
void register_break_hook(struct break_hook *hook)
{
- write_lock(&break_hook_lock);
- list_add(&hook->node, &break_hook);
- write_unlock(&break_hook_lock);
+ spin_lock(&break_hook_lock);
+ list_add_rcu(&hook->node, &break_hook);
+ spin_unlock(&break_hook_lock);
}
void unregister_break_hook(struct break_hook *hook)
{
- write_lock(&break_hook_lock);
- list_del(&hook->node);
- write_unlock(&break_hook_lock);
+ spin_lock(&break_hook_lock);
+ list_del_rcu(&hook->node);
+ spin_unlock(&break_hook_lock);
+ synchronize_rcu();
}
static int call_break_hook(struct pt_regs *regs, unsigned int esr)
struct break_hook *hook;
int (*fn)(struct pt_regs *regs, unsigned int esr) = NULL;
- read_lock(&break_hook_lock);
- list_for_each_entry(hook, &break_hook, node)
+ rcu_read_lock();
+ list_for_each_entry_rcu(hook, &break_hook, node)
if ((esr & hook->esr_mask) == hook->esr_val)
fn = hook->fn;
- read_unlock(&break_hook_lock);
+ rcu_read_unlock();
return fn ? fn(regs, esr) : DBG_HOOK_ERROR;
}
*/
if (!is_normal_ram(md))
prot = __pgprot(PROT_DEVICE_nGnRE);
- else if (md->type == EFI_RUNTIME_SERVICES_CODE)
+ else if (md->type == EFI_RUNTIME_SERVICES_CODE ||
+ !PAGE_ALIGNED(md->phys_addr))
prot = PAGE_KERNEL_EXEC;
else
prot = PAGE_KERNEL;
ENDPROC(ftrace_stub)
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ /* save return value regs*/
+ .macro save_return_regs
+ sub sp, sp, #64
+ stp x0, x1, [sp]
+ stp x2, x3, [sp, #16]
+ stp x4, x5, [sp, #32]
+ stp x6, x7, [sp, #48]
+ .endm
+
+ /* restore return value regs*/
+ .macro restore_return_regs
+ ldp x0, x1, [sp]
+ ldp x2, x3, [sp, #16]
+ ldp x4, x5, [sp, #32]
+ ldp x6, x7, [sp, #48]
+ add sp, sp, #64
+ .endm
+
/*
* void ftrace_graph_caller(void)
*
* only when CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST is enabled.
*/
ENTRY(return_to_handler)
- str x0, [sp, #-16]!
+ save_return_regs
mov x0, x29 // parent's fp
bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
mov x30, x0 // restore the original return address
- ldr x0, [sp], #16
+ restore_return_regs
ret
END(return_to_handler)
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
msr hstr_el2, xzr // Disable CP15 traps to EL2
#endif
+ /* EL2 debug */
+ mrs x0, pmcr_el0 // Disable debug access traps
+ ubfx x0, x0, #11, #5 // to EL2 and allow access to
+ msr mdcr_el2, x0 // all PMU counters from EL1
+
/* Stage-2 translation */
msr vttbr_el2, xzr
void *hcpu)
{
int cpu = (long)hcpu;
- if (action == CPU_ONLINE)
+ if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE)
smp_call_function_single(cpu, hw_breakpoint_reset, NULL, 1);
return NOTIFY_OK;
}
aarch64_insn_is_bcond(insn));
}
-static DEFINE_SPINLOCK(patch_lock);
+static DEFINE_RAW_SPINLOCK(patch_lock);
static void __kprobes *patch_map(void *addr, int fixmap)
{
unsigned long flags = 0;
int ret;
- spin_lock_irqsave(&patch_lock, flags);
+ raw_spin_lock_irqsave(&patch_lock, flags);
waddr = patch_map(addr, FIX_TEXT_POKE0);
ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE);
patch_unmap(FIX_TEXT_POKE0);
- spin_unlock_irqrestore(&patch_lock, flags);
+ raw_spin_unlock_irqrestore(&patch_lock, flags);
return ret;
}
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21,
AARCH64_INSN_IMM_ADR);
break;
+#ifndef CONFIG_ARM64_ERRATUM_843419
case R_AARCH64_ADR_PREL_PG_HI21_NC:
overflow_check = false;
case R_AARCH64_ADR_PREL_PG_HI21:
ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21,
AARCH64_INSN_IMM_ADR);
break;
+#endif
case R_AARCH64_ADD_ABS_LO12_NC:
case R_AARCH64_LDST8_ABS_LO12_NC:
overflow_check = false;
to_free = ram_end - orig_start;
size = orig_end - orig_start;
+ if (!size)
+ return;
/* initrd needs to be relocated completely inside linear mapping */
new_start = memblock_find_in_range(0, PFN_PHYS(max_pfn),
/*
* VFP save/restore code.
+ *
+ * We have to be careful with endianness, since the fpsimd context-switch
+ * code operates on 128-bit (Q) register values whereas the compat ABI
+ * uses an array of 64-bit (D) registers. Consequently, we need to swap
+ * the two halves of each Q register when running on a big-endian CPU.
*/
+union __fpsimd_vreg {
+ __uint128_t raw;
+ struct {
+#ifdef __AARCH64EB__
+ u64 hi;
+ u64 lo;
+#else
+ u64 lo;
+ u64 hi;
+#endif
+ };
+};
+
static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)
{
struct fpsimd_state *fpsimd = ¤t->thread.fpsimd_state;
compat_ulong_t magic = VFP_MAGIC;
compat_ulong_t size = VFP_STORAGE_SIZE;
compat_ulong_t fpscr, fpexc;
- int err = 0;
+ int i, err = 0;
/*
* Save the hardware registers to the fpsimd_state structure.
/*
* Now copy the FP registers. Since the registers are packed,
* we can copy the prefix we want (V0-V15) as it is.
- * FIXME: Won't work if big endian.
*/
- err |= __copy_to_user(&frame->ufp.fpregs, fpsimd->vregs,
- sizeof(frame->ufp.fpregs));
+ for (i = 0; i < ARRAY_SIZE(frame->ufp.fpregs); i += 2) {
+ union __fpsimd_vreg vreg = {
+ .raw = fpsimd->vregs[i >> 1],
+ };
+
+ __put_user_error(vreg.lo, &frame->ufp.fpregs[i], err);
+ __put_user_error(vreg.hi, &frame->ufp.fpregs[i + 1], err);
+ }
/* Create an AArch32 fpscr from the fpsr and the fpcr. */
fpscr = (fpsimd->fpsr & VFP_FPSCR_STAT_MASK) |
compat_ulong_t magic = VFP_MAGIC;
compat_ulong_t size = VFP_STORAGE_SIZE;
compat_ulong_t fpscr;
- int err = 0;
+ int i, err = 0;
__get_user_error(magic, &frame->magic, err);
__get_user_error(size, &frame->size, err);
if (magic != VFP_MAGIC || size != VFP_STORAGE_SIZE)
return -EINVAL;
- /*
- * Copy the FP registers into the start of the fpsimd_state.
- * FIXME: Won't work if big endian.
- */
- err |= __copy_from_user(fpsimd.vregs, frame->ufp.fpregs,
- sizeof(frame->ufp.fpregs));
+ /* Copy the FP registers into the start of the fpsimd_state. */
+ for (i = 0; i < ARRAY_SIZE(frame->ufp.fpregs); i += 2) {
+ union __fpsimd_vreg vreg;
+
+ __get_user_error(vreg.lo, &frame->ufp.fpregs[i], err);
+ __get_user_error(vreg.hi, &frame->ufp.fpregs[i + 1], err);
+ fpsimd.vregs[i >> 1] = vreg.raw;
+ }
/* Extract the fpsr and the fpcr from the fpscr */
__get_user_error(fpscr, &frame->ufp.fpscr, err);
---help---
Provides host support for ARM processors.
-config KVM_ARM_MAX_VCPUS
- int "Number maximum supported virtual CPUs per VM"
- depends on KVM_ARM_HOST
- default 4
- help
- Static number of max supported virtual CPUs per VM.
-
- If you choose a high number, the vcpu structures will be quite
- large, so only choose a reasonable number that you expect to
- actually use.
-
endif # VIRTUALIZATION
mrs x5, ifsr32_el2
stp x4, x5, [x3]
- skip_fpsimd_state x8, 3f
+ skip_fpsimd_state x8, 2f
mrs x6, fpexc32_el2
str x6, [x3, #16]
-3:
- skip_debug_state x8, 2f
+2:
+ skip_debug_state x8, 1f
mrs x7, dbgvcr32_el2
str x7, [x3, #24]
-2:
- skip_tee_state x8, 1f
-
- add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1)
- mrs x4, teecr32_el1
- mrs x5, teehbr32_el1
- stp x4, x5, [x3]
1:
.endm
msr dacr32_el2, x4
msr ifsr32_el2, x5
- skip_debug_state x8, 2f
+ skip_debug_state x8, 1f
ldr x7, [x3, #24]
msr dbgvcr32_el2, x7
-2:
- skip_tee_state x8, 1f
-
- add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1)
- ldp x4, x5, [x3]
- msr teecr32_el1, x4
- msr teehbr32_el1, x5
1:
.endm
mrs x3, cntv_ctl_el0
and x3, x3, #3
str w3, [x0, #VCPU_TIMER_CNTV_CTL]
- bic x3, x3, #1 // Clear Enable
- msr cntv_ctl_el0, x3
isb
str x3, [x0, #VCPU_TIMER_CNTV_CVAL]
1:
+ // Disable the virtual timer
+ msr cntv_ctl_el0, xzr
+
// Allow physical timer/counter access for the host
mrs x2, cnthctl_el2
orr x2, x2, #3
// Guest context
add x2, x0, #VCPU_CONTEXT
+ // We must restore the 32-bit state before the sysregs, thanks
+ // to Cortex-A57 erratum #852523.
+ restore_guest_32bit_state
bl __restore_sysregs
skip_debug_state x3, 1f
kern_hyp_va x3
bl __restore_debug
1:
- restore_guest_32bit_state
restore_guest_regs
// That's it, no more messing around.
{
__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
- if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
+ if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
return -EFAULT;
return 0;
}
{
__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
- if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
+ if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
return -EFAULT;
return 0;
{
__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
- if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
+ if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
return -EFAULT;
return 0;
}
{
__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
- if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
+ if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
return -EFAULT;
return 0;
}
{ Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110),
trap_dbgauthstatus_el1 },
- /* TEECR32_EL1 */
- { Op0(0b10), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
- NULL, reset_val, TEECR32_EL1, 0 },
- /* TEEHBR32_EL1 */
- { Op0(0b10), Op1(0b010), CRn(0b0001), CRm(0b0000), Op2(0b000),
- NULL, reset_val, TEEHBR32_EL1, 0 },
-
/* MDCCSR_EL1 */
{ Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000),
trap_raz_wi },
if (IS_ENABLED(CONFIG_ZONE_DMA) &&
dev->coherent_dma_mask <= DMA_BIT_MASK(32))
flags |= GFP_DMA;
- if (IS_ENABLED(CONFIG_DMA_CMA) && (flags & __GFP_WAIT)) {
+ if (dev_get_cma_area(dev) && (flags & __GFP_WAIT)) {
struct page *page;
void *addr;
* starvation.
*/
mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
+ mm_flags |= FAULT_FLAG_TRIED;
goto retry;
}
}
generic-y += topology.h
generic-y += trace_clock.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
.irq_set_type = eic_set_irq_type,
};
-static void demux_eic_irq(unsigned int irq, struct irq_desc *desc)
+static void demux_eic_irq(struct irq_desc *desc)
{
struct eic *eic = irq_desc_get_handler_data(desc);
unsigned long status, pending;
.irq_set_type = gpio_irq_type,
};
-static void gpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void gpio_irq_handler(struct irq_desc *desc)
{
struct pio_device *pio = irq_desc_get_chip_data(desc);
unsigned gpio_irq;
generic-y += ucontext.h
generic-y += unaligned.h
generic-y += user.h
+generic-y += word-at-a-time.h
generic-y += xor.h
extern void bfin_internal_unmask_irq(unsigned int irq);
struct irq_desc;
-extern void bfin_demux_mac_status_irq(unsigned int, struct irq_desc *);
-extern void bfin_demux_gpio_irq(unsigned int, struct irq_desc *);
+extern void bfin_demux_mac_status_irq(struct irq_desc *);
+extern void bfin_demux_gpio_irq(struct irq_desc *);
#endif
* than crashing, do something sensible.
*/
if (irq >= NR_IRQS)
- handle_bad_irq(irq, &bad_irq_desc);
+ handle_bad_irq(&bad_irq_desc);
else
generic_handle_irq(irq);
.irq_unmask = bf537_generic_error_unmask_irq,
};
-static void bf537_demux_error_irq(unsigned int int_err_irq,
- struct irq_desc *inta_desc)
+static void bf537_demux_error_irq(struct irq_desc *inta_desc)
{
int irq = 0;
.irq_unmask = bf537_mac_rx_unmask_irq,
};
-static void bf537_demux_mac_rx_irq(unsigned int __int_irq,
- struct irq_desc *desc)
+static void bf537_demux_mac_rx_irq(struct irq_desc *desc)
{
- unsigned int int_irq = irq_desc_get_irq(desc);
-
if (bfin_read_DMA1_IRQ_STATUS() & (DMA_DONE | DMA_ERR))
bfin_handle_irq(IRQ_MAC_RX);
else
- bfin_demux_gpio_irq(int_irq, desc);
+ bfin_demux_gpio_irq(desc);
}
#endif
.irq_set_wake = bfin_mac_status_set_wake,
};
-void bfin_demux_mac_status_irq(unsigned int int_err_irq,
- struct irq_desc *inta_desc)
+void bfin_demux_mac_status_irq(struct irq_desc *inta_desc)
{
int i, irq = 0;
u32 status = bfin_read_EMAC_SYSTAT();
}
}
-void bfin_demux_gpio_irq(unsigned int __inta_irq, struct irq_desc *desc)
+void bfin_demux_gpio_irq(struct irq_desc *desc)
{
unsigned int inta_irq = irq_desc_get_irq(desc);
unsigned int irq;
generic-y += ucontext.h
generic-y += user.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
.irq_unmask = unmask_megamod,
};
-static void megamod_irq_cascade(unsigned int __irq, struct irq_desc *desc)
+static void megamod_irq_cascade(struct irq_desc *desc)
{
struct megamod_cascade_data *cascade;
struct megamod_pic *pic;
generic-y += trace_clock.h
generic-y += types.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
generic-y += mm-arch-hooks.h
generic-y += preempt.h
generic-y += trace_clock.h
+generic-y += word-at-a-time.h
printk("### PCIBIOS_FIXUP_BUS(%d)\n",bus->number);
#endif
+ pci_read_bridge_bases(bus);
+
if (bus->number == 0) {
struct pci_dev *dev;
list_for_each_entry(dev, &bus->devices, bus_list) {
generic-y += ucontext.h
generic-y += unaligned.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
generic-y += ucontext.h
generic-y += unaligned.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
generic-y += preempt.h
generic-y += trace_clock.h
generic-y += vtime.h
+generic-y += word-at-a-time.h
-#define NR_syscalls 319 /* length of syscall table */
+#define NR_syscalls 321 /* length of syscall table */
/*
* The following defines stop scripts/checksyscalls.sh from complaining about
#define __NR_memfd_create 1340
#define __NR_bpf 1341
#define __NR_execveat 1342
+#define __NR_userfaultfd 1343
+#define __NR_membarrier 1344
#endif /* _UAPI_ASM_IA64_UNISTD_H */
data8 sys_memfd_create // 1340
data8 sys_bpf
data8 sys_execveat
+ data8 sys_userfaultfd
+ data8 sys_membarrier
.org sys_call_table + 8*NR_syscalls // guard against failures to increase NR_syscalls
{
struct pci_dev *dev;
- if (b->self)
+ if (b->self) {
+ pci_read_bridge_bases(b);
pcibios_fixup_bridge_resources(b->self);
-
+ }
list_for_each_entry(dev, &b->devices, bus_list)
pcibios_fixup_device_resources(dev);
platform_pci_fixup_bus(b);
generic-y += preempt.h
generic-y += sections.h
generic-y += trace_clock.h
+generic-y += word-at-a-time.h
* The builtin Amiga hardware interrupt handlers.
*/
-static void ami_int1(unsigned int irq, struct irq_desc *desc)
+static void ami_int1(struct irq_desc *desc)
{
unsigned short ints = amiga_custom.intreqr & amiga_custom.intenar;
}
}
-static void ami_int3(unsigned int irq, struct irq_desc *desc)
+static void ami_int3(struct irq_desc *desc)
{
unsigned short ints = amiga_custom.intreqr & amiga_custom.intenar;
}
}
-static void ami_int4(unsigned int irq, struct irq_desc *desc)
+static void ami_int4(struct irq_desc *desc)
{
unsigned short ints = amiga_custom.intreqr & amiga_custom.intenar;
}
}
-static void ami_int5(unsigned int irq, struct irq_desc *desc)
+static void ami_int5(struct irq_desc *desc)
{
unsigned short ints = amiga_custom.intreqr & amiga_custom.intenar;
* We need to be careful with the masking/acking due to the side effects
* of masking an interrupt.
*/
-static void intc_external_irq(unsigned int __irq, struct irq_desc *desc)
+static void intc_external_irq(struct irq_desc *desc)
{
- unsigned int irq = irq_desc_get_irq(desc);
-
irq_desc_get_chip(desc)->irq_ack(&desc->irq_data);
- handle_simple_irq(irq, desc);
+ handle_simple_irq(desc);
}
static struct irq_chip intc_irq_chip = {
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_SMSC is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SAMSUNG is not set
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SEEQ is not set
CONFIG_SMC91X=y
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SAMSUNG is not set
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SAMSUNG is not set
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_SMSC is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SEEQ is not set
CONFIG_SMC91X=y
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PLIP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SAMSUNG is not set
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SAMSUNG is not set
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_SMSC is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PLIP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_STMICRO is not set
# CONFIG_NET_VENDOR_SUN is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
CONFIG_CRYPTO_MANAGER=y
# CONFIG_PID_NS is not set
# CONFIG_NET_NS is not set
CONFIG_BLK_DEV_INITRD=y
+CONFIG_USERFAULTFD=y
CONFIG_SLAB=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET_IPGRE=m
CONFIG_NET_IPVTI=m
CONFIG_NET_FOU_IP_TUNNELS=y
-CONFIG_GENEVE_CORE=m
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
# CONFIG_INET_LRO is not set
CONFIG_INET_DIAG=m
CONFIG_INET_UDP_DIAG=m
+CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_INET6_AH=m
CONFIG_INET6_ESP=m
CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_ILA=m
CONFIG_IPV6_VTI=m
CONFIG_IPV6_GRE=m
CONFIG_NETFILTER=y
CONFIG_IP_SET_LIST_SET=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
CONFIG_NF_TABLES_ARP=m
CONFIG_NF_LOG_ARP=m
CONFIG_NFT_CHAIN_NAT_IPV4=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
CONFIG_NFT_CHAIN_NAT_IPV6=m
CONFIG_NFT_MASQ_IPV6=m
CONFIG_NFT_REDIR_IPV6=m
CONFIG_MPLS=y
CONFIG_NET_MPLS_GSO=m
CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
# CONFIG_NET_VENDOR_SAMSUNG is not set
# CONFIG_NET_VENDOR_SEEQ is not set
# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
# CONFIG_NET_VENDOR_VIA is not set
# CONFIG_NET_VENDOR_WIZNET is not set
CONFIG_PPP=m
CONFIG_TEST_BPF=m
CONFIG_TEST_FIRMWARE=m
CONFIG_TEST_UDELAY=m
+CONFIG_TEST_STATIC_KEYS=m
CONFIG_EARLY_PRINTK=y
CONFIG_ENCRYPTED_KEYS=m
CONFIG_CRYPTO_RSA=m
struct pt_regs *));
extern void m68k_setup_user_interrupt(unsigned int vec, unsigned int cnt);
extern void m68k_setup_irq_controller(struct irq_chip *,
- void (*handle)(unsigned int irq,
- struct irq_desc *desc),
+ void (*handle)(struct irq_desc *desc),
unsigned int irq, unsigned int cnt);
extern unsigned int irq_canonicalize(unsigned int irq);
#define __ALIGN .align 4
#define __ALIGN_STR ".align 4"
+/*
+ * Make sure the compiler doesn't do anything stupid with the
+ * arguments on the stack - they are owned by the *caller*, not
+ * the callee. This just fools gcc into not spilling into them,
+ * and keeps it from doing tailcall recursion and/or using the
+ * stack slots for temporaries, since they are live and "used"
+ * all the way to the end of the function.
+ */
+#define asmlinkage_protect(n, ret, args...) \
+ __asmlinkage_protect##n(ret, ##args)
+#define __asmlinkage_protect_n(ret, args...) \
+ __asm__ __volatile__ ("" : "=r" (ret) : "0" (ret), ##args)
+#define __asmlinkage_protect0(ret) \
+ __asmlinkage_protect_n(ret)
+#define __asmlinkage_protect1(ret, arg1) \
+ __asmlinkage_protect_n(ret, "m" (arg1))
+#define __asmlinkage_protect2(ret, arg1, arg2) \
+ __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2))
+#define __asmlinkage_protect3(ret, arg1, arg2, arg3) \
+ __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3))
+#define __asmlinkage_protect4(ret, arg1, arg2, arg3, arg4) \
+ __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \
+ "m" (arg4))
+#define __asmlinkage_protect5(ret, arg1, arg2, arg3, arg4, arg5) \
+ __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \
+ "m" (arg4), "m" (arg5))
+#define __asmlinkage_protect6(ret, arg1, arg2, arg3, arg4, arg5, arg6) \
+ __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \
+ "m" (arg4), "m" (arg5), "m" (arg6))
+
#endif
extern void via_irq_disable(int);
extern void via_nubus_irq_startup(int irq);
extern void via_nubus_irq_shutdown(int irq);
-extern void via1_irq(unsigned int irq, struct irq_desc *desc);
+extern void via1_irq(struct irq_desc *desc);
extern void via1_set_head(int);
extern int via2_scsi_drq_pending(void);
#include <uapi/asm/unistd.h>
-#define NR_syscalls 356
+#define NR_syscalls 375
#define __ARCH_WANT_OLD_READDIR
#define __ARCH_WANT_OLD_STAT
#define __NR_memfd_create 353
#define __NR_bpf 354
#define __NR_execveat 355
+#define __NR_socket 356
+#define __NR_socketpair 357
+#define __NR_bind 358
+#define __NR_connect 359
+#define __NR_listen 360
+#define __NR_accept4 361
+#define __NR_getsockopt 362
+#define __NR_setsockopt 363
+#define __NR_getsockname 364
+#define __NR_getpeername 365
+#define __NR_sendto 366
+#define __NR_sendmsg 367
+#define __NR_recvfrom 368
+#define __NR_recvmsg 369
+#define __NR_shutdown 370
+#define __NR_recvmmsg 371
+#define __NR_sendmmsg 372
+#define __NR_userfaultfd 373
+#define __NR_membarrier 374
#endif /* _UAPI_ASM_M68K_UNISTD_H_ */
.long sys_memfd_create
.long sys_bpf
.long sys_execveat /* 355 */
-
+ .long sys_socket
+ .long sys_socketpair
+ .long sys_bind
+ .long sys_connect
+ .long sys_listen /* 360 */
+ .long sys_accept4
+ .long sys_getsockopt
+ .long sys_setsockopt
+ .long sys_getsockname
+ .long sys_getpeername /* 365 */
+ .long sys_sendto
+ .long sys_sendmsg
+ .long sys_recvfrom
+ .long sys_recvmsg
+ .long sys_shutdown /* 370 */
+ .long sys_recvmmsg
+ .long sys_sendmmsg
+ .long sys_userfaultfd
+ .long sys_membarrier
* Baboon interrupt handler. This works a lot like a VIA.
*/
-static void baboon_irq(unsigned int irq, struct irq_desc *desc)
+static void baboon_irq(struct irq_desc *desc)
{
int irq_bit, irq_num;
unsigned char events;
* Handle miscellaneous OSS interrupts.
*/
-static void oss_irq(unsigned int __irq, struct irq_desc *desc)
+static void oss_irq(struct irq_desc *desc)
{
int events = oss->irq_pending &
(OSS_IP_IOPSCC | OSS_IP_SCSI | OSS_IP_IOPISM);
* Unlike the VIA/RBV this is on its own autovector interrupt level.
*/
-static void oss_nubus_irq(unsigned int irq, struct irq_desc *desc)
+static void oss_nubus_irq(struct irq_desc *desc)
{
int events, irq_bit, i;
* PSC interrupt handler. It's a lot like the VIA interrupt handler.
*/
-static void psc_irq(unsigned int __irq, struct irq_desc *desc)
+static void psc_irq(struct irq_desc *desc)
{
unsigned int offset = (unsigned int)irq_desc_get_handler_data(desc);
unsigned int irq = irq_desc_get_irq(desc);
* via6522.c :-), disable/pending masks added.
*/
-void via1_irq(unsigned int irq, struct irq_desc *desc)
+void via1_irq(struct irq_desc *desc)
{
int irq_num;
unsigned char irq_bit, events;
} while (events >= irq_bit);
}
-static void via2_irq(unsigned int irq, struct irq_desc *desc)
+static void via2_irq(struct irq_desc *desc)
{
int irq_num;
unsigned char irq_bit, events;
* VIA2 dispatcher as a fast interrupt handler.
*/
-void via_nubus_irq(unsigned int irq, struct irq_desc *desc)
+static void via_nubus_irq(struct irq_desc *desc)
{
int slot_irq;
unsigned char slot_bit, events;
generic-y += unaligned.h
generic-y += user.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
"MOV D0.5,%0\n"
"MOV D1Ar1,%1\n"
"MOV D1RtP,%2\n"
- "MOV D0Ar2,%3\n"
"SWAP A0StP,D0.5\n"
"SWAP PC,D1RtP\n"
"MOV A0StP,D0.5\n"
:
- : "r" (isp), "r" (irq), "r" (desc->handle_irq),
- "r" (desc)
+ : "r" (isp), "r" (desc), "r" (desc->handle_irq)
: "memory", "cc", "D1Ar1", "D0Ar2", "D1Ar3", "D0Ar4",
"D1Ar5", "D0Ar6", "D0Re0", "D1Re0", "D0.4", "D1RtP",
"D0.5"
generic-y += preempt.h
generic-y += syscalls.h
generic-y += trace_clock.h
+generic-y += word-at-a-time.h
void pcibios_fixup_bus(struct pci_bus *bus)
{
- /* Fixup the bus */
+ /* When called from the generic PCI probe, read PCI<->PCI bridge
+ * bases. This is -not- called when generating the PCI tree from
+ * the OF device-tree.
+ */
+ if (bus->self != NULL)
+ pci_read_bridge_bases(bus);
+
+ /* Now fixup the bus bus */
pcibios_setup_bus_self(bus);
/* Now fixup devices on that bus */
/* create chained handlers for the 4 IC requests to the MIPS IRQ ctrl */
#define DISP(name, base, addr) \
-static void au1000_##name##_dispatch(unsigned int irq, struct irq_desc *d) \
+static void au1000_##name##_dispatch(struct irq_desc *d) \
{ \
unsigned long r = __raw_readl((void __iomem *)KSEG1ADDR(addr)); \
if (likely(r)) \
DISP(ic1r0, AU1000_INTC1_INT_BASE, AU1000_IC1_PHYS_ADDR + IC_REQ0INT)
DISP(ic1r1, AU1000_INTC1_INT_BASE, AU1000_IC1_PHYS_ADDR + IC_REQ1INT)
-static void alchemy_gpic_dispatch(unsigned int irq, struct irq_desc *d)
+static void alchemy_gpic_dispatch(struct irq_desc *d)
{
int i = __raw_readl(AU1300_GPIC_ADDR + AU1300_GPIC_PRIENC);
generic_handle_irq(ALCHEMY_GPIC_INT_BASE + i);
/*
* DB1200/PB1200 CPLD IRQ muxer
*/
-static void bcsr_csc_handler(unsigned int irq, struct irq_desc *d)
+static void bcsr_csc_handler(struct irq_desc *d)
{
unsigned short bisr = __raw_readw(bcsr_virt + BCSR_REG_INTSTAT);
struct irq_chip *chip = irq_desc_get_chip(d);
.name = "ar2315-ahb-error",
};
-static void ar2315_misc_irq_handler(unsigned irq, struct irq_desc *desc)
+static void ar2315_misc_irq_handler(struct irq_desc *desc)
{
u32 pending = ar2315_rst_reg_read(AR2315_ISR) &
ar2315_rst_reg_read(AR2315_IMR);
.name = "ar5312-ahb-error",
};
-static void ar5312_misc_irq_handler(unsigned irq, struct irq_desc *desc)
+static void ar5312_misc_irq_handler(struct irq_desc *desc)
{
u32 pending = ar5312_rst_reg_read(AR5312_ISR) &
ar5312_rst_reg_read(AR5312_IMR);
#include "common.h"
#include "machtypes.h"
-static void ath79_misc_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ath79_misc_irq_handler(struct irq_desc *desc)
{
void __iomem *base = ath79_reset_base;
u32 pending;
irq_set_chained_handler(ATH79_CPU_IRQ(6), ath79_misc_irq_handler);
}
-static void ar934x_ip2_irq_dispatch(unsigned int irq, struct irq_desc *desc)
+static void ar934x_ip2_irq_dispatch(struct irq_desc *desc)
{
u32 status;
irq_set_chained_handler(ATH79_CPU_IRQ(2), ar934x_ip2_irq_dispatch);
}
-static void qca955x_ip2_irq_dispatch(unsigned int irq, struct irq_desc *desc)
+static void qca955x_ip2_irq_dispatch(struct irq_desc *desc)
{
u32 status;
}
}
-static void qca955x_ip3_irq_dispatch(unsigned int irq, struct irq_desc *desc)
+static void qca955x_ip3_irq_dispatch(struct irq_desc *desc)
{
u32 status;
return 0;
}
-IRQCHIP_DECLARE(ath79_misc_intc, "qca,ar7100-misc-intc",
- ath79_misc_intc_of_init);
+
+static int __init ar7100_misc_intc_of_init(
+ struct device_node *node, struct device_node *parent)
+{
+ ath79_misc_irq_chip.irq_mask_ack = ar71xx_misc_irq_mask;
+ return ath79_misc_intc_of_init(node, parent);
+}
+
+IRQCHIP_DECLARE(ar7100_misc_intc, "qca,ar7100-misc-intc",
+ ar7100_misc_intc_of_init);
+
+static int __init ar7240_misc_intc_of_init(
+ struct device_node *node, struct device_node *parent)
+{
+ ath79_misc_irq_chip.irq_ack = ar724x_misc_irq_ack;
+ return ath79_misc_intc_of_init(node, parent);
+}
+
+IRQCHIP_DECLARE(ar7240_misc_intc, "qca,ar7240-misc-intc",
+ ar7240_misc_intc_of_init);
static int __init ar79_cpu_intc_of_init(
struct device_node *node, struct device_node *parent)
if (irqd_get_trigger_type(irq_data) &
IRQ_TYPE_EDGE_BOTH)
cvmx_write_csr(host_data->raw_reg, 1ull << i);
- generic_handle_irq_desc(irq, desc);
+ generic_handle_irq_desc(desc);
}
}
while ((boot_mem_map.nr_map < BOOT_MEM_MAP_MAX)
&& (total < MAX_MEMORY)) {
memory = cvmx_bootmem_phy_alloc(mem_alloc_size,
- __pa_symbol(&__init_end), -1,
+ __pa_symbol(&_end), -1,
0x100000,
CVMX_BOOTMEM_FLAG_NO_LOCKING);
if (memory >= 0) {
generic-y += serial.h
generic-y += trace_clock.h
generic-y += user.h
+generic-y += word-at-a-time.h
generic-y += xor.h
#ifndef cpu_has_tlb
#define cpu_has_tlb (cpu_data[0].options & MIPS_CPU_TLB)
#endif
+#ifndef cpu_has_ftlb
+#define cpu_has_ftlb (cpu_data[0].options & MIPS_CPU_FTLB)
+#endif
#ifndef cpu_has_tlbinv
#define cpu_has_tlbinv (cpu_data[0].options & MIPS_CPU_TLBINV)
#endif
#define MIPS_CPU_CDMM 0x4000000000ull /* CPU has Common Device Memory Map */
#define MIPS_CPU_BP_GHIST 0x8000000000ull /* R12K+ Branch Prediction Global History */
#define MIPS_CPU_SP 0x10000000000ull /* Small (1KB) page support */
+#define MIPS_CPU_FTLB 0x20000000000ull /* CPU has Fixed-page-size TLB */
/*
* CPU ASE encodings
*/
#define ioremap_nocache(offset, size) \
__ioremap_mode((offset), (size), _CACHE_UNCACHED)
+#define ioremap_uc ioremap_nocache
/*
* ioremap_cachable - map bus memory into CPU space
#define KVM_PRIVATE_MEM_SLOTS 0
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
+#define KVM_HALT_POLL_NS_DEFAULT 500000
u32 msa_disabled_exits;
u32 flush_dcache_exits;
u32 halt_successful_poll;
+ u32 halt_attempted_poll;
u32 halt_wakeup;
};
back_to_back_c0_hazard();
}
+/**
+ * maar_init() - initialise MAARs
+ *
+ * Performs initialisation of MAARs for the current CPU, making use of the
+ * platforms implementation of platform_maar_init where necessary and
+ * duplicating the setup it provides on secondary CPUs.
+ */
+extern void maar_init(void);
+
/**
* struct maar_config - MAAR configuration data
* @lower: The lowest address that the MAAR pair will affect. Must be
BUILD_CM_R_(gic_status, MIPS_CM_GCB_OFS + 0xd0)
BUILD_CM_R_(cpc_status, MIPS_CM_GCB_OFS + 0xf0)
BUILD_CM_RW(l2_config, MIPS_CM_GCB_OFS + 0x130)
+BUILD_CM_RW(sys_config2, MIPS_CM_GCB_OFS + 0x150)
/* Core Local & Core Other register accessor functions */
BUILD_CM_Cx_RW(reset_release, 0x00)
#define CM_GCR_L2_CONFIG_ASSOC_SHF 0
#define CM_GCR_L2_CONFIG_ASSOC_MSK (_ULCAST_(0xff) << 0)
+/* GCR_SYS_CONFIG2 register fields */
+#define CM_GCR_SYS_CONFIG2_MAXVPW_SHF 0
+#define CM_GCR_SYS_CONFIG2_MAXVPW_MSK (_ULCAST_(0xf) << 0)
+
/* GCR_Cx_COHERENCE register fields */
#define CM_GCR_Cx_COHERENCE_COHDOMAINEN_SHF 0
#define CM_GCR_Cx_COHERENCE_COHDOMAINEN_MSK (_ULCAST_(0xff) << 0)
return read_gcr_rev();
}
+/**
+ * mips_cm_max_vp_width() - return the width in bits of VP indices
+ *
+ * Return: the width, in bits, of VP indices in fields that combine core & VP
+ * indices.
+ */
+static inline unsigned int mips_cm_max_vp_width(void)
+{
+ extern int smp_num_siblings;
+
+ if (mips_cm_revision() >= CM_REV_CM3)
+ return read_gcr_sys_config2() & CM_GCR_SYS_CONFIG2_MAXVPW_MSK;
+
+ return smp_num_siblings;
+}
+
+/**
+ * mips_cm_vp_id() - calculate the hardware VP ID for a CPU
+ * @cpu: the CPU whose VP ID to calculate
+ *
+ * Hardware such as the GIC uses identifiers for VPs which may not match the
+ * CPU numbers used by Linux. This function calculates the hardware VP
+ * identifier corresponding to a given CPU.
+ *
+ * Return: the VP ID for the CPU.
+ */
+static inline unsigned int mips_cm_vp_id(unsigned int cpu)
+{
+ unsigned int core = cpu_data[cpu].core;
+ unsigned int vp = cpu_vpe_id(&cpu_data[cpu]);
+
+ return (core * mips_cm_max_vp_width()) + vp;
+}
+
#endif /* __MIPS_ASM_MIPS_CM_H__ */
/* Bits specific to the MIPS32/64 PRA. */
#define MIPS_CONF_MT (_ULCAST_(7) << 7)
+#define MIPS_CONF_MT_TLB (_ULCAST_(1) << 7)
+#define MIPS_CONF_MT_FTLB (_ULCAST_(4) << 7)
#define MIPS_CONF_AR (_ULCAST_(7) << 10)
#define MIPS_CONF_AT (_ULCAST_(3) << 13)
#define MIPS_CONF_M (_ULCAST_(1) << 31)
#include <asm/mach-netlogic/multi-node.h>
struct irq_desc;
-void nlm_smp_function_ipi_handler(unsigned int irq, struct irq_desc *desc);
-void nlm_smp_resched_ipi_handler(unsigned int irq, struct irq_desc *desc);
+void nlm_smp_function_ipi_handler(struct irq_desc *desc);
+void nlm_smp_resched_ipi_handler(struct irq_desc *desc);
void nlm_smp_irq_init(int hwcpuid);
void nlm_boot_secondary_cpus(void);
int nlm_wakeup_secondary_cpus(void);
#define __SWAB_64_THRU_32__
-#if (defined(__mips_isa_rev) && (__mips_isa_rev >= 2)) || \
- defined(_MIPS_ARCH_LOONGSON3A)
+#if !defined(__mips16) && \
+ ((defined(__mips_isa_rev) && (__mips_isa_rev >= 2)) || \
+ defined(_MIPS_ARCH_LOONGSON3A))
-static inline __attribute__((nomips16)) __attribute_const__
- __u16 __arch_swab16(__u16 x)
+static inline __attribute_const__ __u16 __arch_swab16(__u16 x)
{
__asm__(
" .set push \n"
" .set arch=mips32r2 \n"
- " .set nomips16 \n"
" wsbh %0, %1 \n"
" .set pop \n"
: "=r" (x)
}
#define __arch_swab16 __arch_swab16
-static inline __attribute__((nomips16)) __attribute_const__
- __u32 __arch_swab32(__u32 x)
+static inline __attribute_const__ __u32 __arch_swab32(__u32 x)
{
__asm__(
" .set push \n"
" .set arch=mips32r2 \n"
- " .set nomips16 \n"
" wsbh %0, %1 \n"
" rotr %0, %0, 16 \n"
" .set pop \n"
* 64-bit kernel on r2 CPUs.
*/
#ifdef __mips64
-static inline __attribute__((nomips16)) __attribute_const__
- __u64 __arch_swab64(__u64 x)
+static inline __attribute_const__ __u64 __arch_swab64(__u64 x)
{
__asm__(
" .set push \n"
" .set arch=mips64r2 \n"
- " .set nomips16 \n"
" dsbh %0, %1 \n"
" dshd %0, %0 \n"
" .set pop \n"
}
#define __arch_swab64 __arch_swab64
#endif /* __mips64 */
-#endif /* MIPS R2 or newer or Loongson 3A */
+#endif /* (not __mips16) and (MIPS R2 or newer or Loongson 3A) */
#endif /* _ASM_SWAB_H */
#define __NR_memfd_create (__NR_Linux + 354)
#define __NR_bpf (__NR_Linux + 355)
#define __NR_execveat (__NR_Linux + 356)
+#define __NR_userfaultfd (__NR_Linux + 357)
+#define __NR_membarrier (__NR_Linux + 358)
/*
* Offset of the last Linux o32 flavoured syscall
*/
-#define __NR_Linux_syscalls 356
+#define __NR_Linux_syscalls 358
#endif /* _MIPS_SIM == _MIPS_SIM_ABI32 */
#define __NR_O32_Linux 4000
-#define __NR_O32_Linux_syscalls 356
+#define __NR_O32_Linux_syscalls 358
#if _MIPS_SIM == _MIPS_SIM_ABI64
#define __NR_memfd_create (__NR_Linux + 314)
#define __NR_bpf (__NR_Linux + 315)
#define __NR_execveat (__NR_Linux + 316)
+#define __NR_userfaultfd (__NR_Linux + 317)
+#define __NR_membarrier (__NR_Linux + 318)
/*
* Offset of the last Linux 64-bit flavoured syscall
*/
-#define __NR_Linux_syscalls 316
+#define __NR_Linux_syscalls 318
#endif /* _MIPS_SIM == _MIPS_SIM_ABI64 */
#define __NR_64_Linux 5000
-#define __NR_64_Linux_syscalls 316
+#define __NR_64_Linux_syscalls 318
#if _MIPS_SIM == _MIPS_SIM_NABI32
#define __NR_memfd_create (__NR_Linux + 318)
#define __NR_bpf (__NR_Linux + 319)
#define __NR_execveat (__NR_Linux + 320)
+#define __NR_userfaultfd (__NR_Linux + 321)
+#define __NR_membarrier (__NR_Linux + 322)
/*
* Offset of the last N32 flavoured syscall
*/
-#define __NR_Linux_syscalls 320
+#define __NR_Linux_syscalls 322
#endif /* _MIPS_SIM == _MIPS_SIM_NABI32 */
#define __NR_N32_Linux 6000
-#define __NR_N32_Linux_syscalls 320
+#define __NR_N32_Linux_syscalls 322
#endif /* _UAPI_ASM_UNISTD_H */
#include <linux/power/jz4740-battery.h>
#include <linux/power/gpio-charger.h>
+#include <asm/mach-jz4740/gpio.h>
#include <asm/mach-jz4740/jz4740_fb.h>
#include <asm/mach-jz4740/jz4740_mmc.h>
#include <asm/mach-jz4740/jz4740_nand.h>
#include <linux/seq_file.h>
#include <asm/mach-jz4740/base.h>
+#include <asm/mach-jz4740/gpio.h>
#define JZ4740_GPIO_BASE_A (32*0)
#define JZ4740_GPIO_BASE_B (32*1)
writel(mask, reg);
}
-static void jz_gpio_irq_demux_handler(unsigned int irq, struct irq_desc *desc)
+static void jz_gpio_irq_demux_handler(struct irq_desc *desc)
{
uint32_t flag;
unsigned int gpio_irq;
mfc0 \dest, CP0_CONFIG, 3
andi \dest, \dest, MIPS_CONF3_MT
beqz \dest, \nomt
+ nop
.endm
.section .text.cps-vec
END(excep_ejtag)
LEAF(mips_cps_core_init)
-#ifdef CONFIG_MIPS_MT
+#ifdef CONFIG_MIPS_MT_SMP
/* Check that the core implements the MT ASE */
has_mt t0, 3f
- nop
.set push
.set mips64r2
PTR_ADDU t0, t0, t1
/* Calculate this VPEs ID. If the core doesn't support MT use 0 */
+ li t9, 0
+#ifdef CONFIG_MIPS_MT_SMP
has_mt ta2, 1f
- li t9, 0
/* Find the number of VPEs present in the core */
mfc0 t1, CP0_MVPCONF0
/* Retrieve the VPE ID from EBase.CPUNum */
mfc0 t9, $15, 1
and t9, t9, t1
+#endif
1: /* Calculate a pointer to this VPEs struct vpe_boot_config */
li t1, VPEBOOTCFG_SIZE
PTR_L ta3, COREBOOTCFG_VPECONFIG(t0)
PTR_ADDU v0, v0, ta3
-#ifdef CONFIG_MIPS_MT
+#ifdef CONFIG_MIPS_MT_SMP
/* If the core doesn't support MT then return */
bnez ta2, 1f
2: .set pop
-#endif /* CONFIG_MIPS_MT */
+#endif /* CONFIG_MIPS_MT_SMP */
/* Return */
jr ra
static inline unsigned int decode_config0(struct cpuinfo_mips *c)
{
unsigned int config0;
- int isa;
+ int isa, mt;
config0 = read_c0_config();
/*
* Look for Standard TLB or Dual VTLB and FTLB
*/
- if ((((config0 & MIPS_CONF_MT) >> 7) == 1) ||
- (((config0 & MIPS_CONF_MT) >> 7) == 4))
+ mt = config0 & MIPS_CONF_MT;
+ if (mt == MIPS_CONF_MT_TLB)
c->options |= MIPS_CPU_TLB;
+ else if (mt == MIPS_CONF_MT_FTLB)
+ c->options |= MIPS_CPU_TLB | MIPS_CPU_FTLB;
isa = (config0 & MIPS_CONF_AT) >> 13;
switch (isa) {
if (cpu_has_tlb) {
if (((config4 & MIPS_CONF4_IE) >> 29) == 2)
c->options |= MIPS_CPU_TLBINV;
+
/*
- * This is a bit ugly. R6 has dropped that field from
- * config4 and the only valid configuration is VTLB+FTLB so
- * set a good value for mmuextdef for that case.
+ * R6 has dropped the MMUExtDef field from config4.
+ * On R6 the fields always describe the FTLB, and only if it is
+ * present according to Config.MT.
*/
- if (cpu_has_mips_r6)
+ if (!cpu_has_mips_r6)
+ mmuextdef = config4 & MIPS_CONF4_MMUEXTDEF;
+ else if (cpu_has_ftlb)
mmuextdef = MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT;
else
- mmuextdef = config4 & MIPS_CONF4_MMUEXTDEF;
+ mmuextdef = 0;
switch (mmuextdef) {
case MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT:
.set pop
/*
* task_struct *resume(task_struct *prev, task_struct *next,
- * struct thread_info *next_ti, int usedfpu)
+ * struct thread_info *next_ti)
*/
.align 7
LEAF(resume)
cpu_save_nonscratch a0
LONG_S ra, THREAD_REG31(a0)
- /*
- * check if we need to save FPU registers
- */
- .set push
- .set noreorder
- beqz a3, 1f
- PTR_L t3, TASK_THREAD_INFO(a0)
- .set pop
-
- /*
- * clear saved user stack CU1 bit
- */
- LONG_L t0, ST_OFF(t3)
- li t1, ~ST0_CU1
- and t0, t0, t1
- LONG_S t0, ST_OFF(t3)
-
- .set push
- .set arch=mips64r2
- fpu_save_double a0 t0 t1 # c0_status passed in t0
- # clobbers t1
- .set pop
-1:
-
#if CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
/* Check if we need to store CVMSEG state */
dmfc0 t0, $11,7 /* CvmMemCtl */
*/
#define ST_OFF (_THREAD_SIZE - 32 - PT_SIZE + PT_STATUS)
-/*
- * FPU context is saved iff the process has used it's FPU in the current
- * time slice as indicated by TIF_USEDFPU. In any case, the CU1 bit for user
- * space STATUS register should be 0, so that a process *always* starts its
- * userland with FPU disabled after each context switch.
- *
- * FPU will be enabled as soon as the process accesses FPU again, through
- * do_cpu() trap.
- */
-
/*
* task_struct *resume(task_struct *prev, task_struct *next,
- * struct thread_info *next_ti, int usedfpu)
+ * struct thread_info *next_ti)
*/
LEAF(resume)
mfc0 t1, CP0_STATUS
cpu_save_nonscratch a0
sw ra, THREAD_REG31(a0)
- beqz a3, 1f
-
- PTR_L t3, TASK_THREAD_INFO(a0)
-
- /*
- * clear saved user stack CU1 bit
- */
- lw t0, ST_OFF(t3)
- li t1, ~ST0_CU1
- and t0, t0, t1
- sw t0, ST_OFF(t3)
-
- fpu_save_single a0, t0 # clobbers t0
-
-1:
-
#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
PTR_LA t8, __stack_chk_guard
LONG_L t9, TASK_STACK_CANARY(a1)
lw t1, PT_EPC(sp) # skip syscall on return
subu v0, v0, __NR_O32_Linux # check syscall number
- sltiu t0, v0, __NR_O32_Linux_syscalls + 1
addiu t1, 4 # skip to next instruction
sw t1, PT_EPC(sp)
- beqz t0, illegal_syscall
-
- sll t0, v0, 2
- la t1, sys_call_table
- addu t1, t0
- lw t2, (t1) # syscall routine
- beqz t2, illegal_syscall
sw a3, PT_R26(sp) # save a3 for syscall restarting
li t1, _TIF_WORK_SYSCALL_ENTRY
and t0, t1
bnez t0, syscall_trace_entry # -> yes
+syscall_common:
+ sltiu t0, v0, __NR_O32_Linux_syscalls + 1
+ beqz t0, illegal_syscall
+
+ sll t0, v0, 2
+ la t1, sys_call_table
+ addu t1, t0
+ lw t2, (t1) # syscall routine
+
+ beqz t2, illegal_syscall
jalr t2 # Do The Real Thing (TM)
syscall_trace_entry:
SAVE_STATIC
- move s0, t2
+ move s0, v0
move a0, sp
/*
1: jal syscall_trace_enter
- bltz v0, 2f # seccomp failed? Skip syscall
+ bltz v0, 1f # seccomp failed? Skip syscall
+
+ move v0, s0 # restore syscall
- move t0, s0
RESTORE_STATIC
lw a0, PT_R4(sp) # Restore argument registers
lw a1, PT_R5(sp)
lw a2, PT_R6(sp)
lw a3, PT_R7(sp)
- jalr t0
-
- li t0, -EMAXERRNO - 1 # error?
- sltu t0, t0, v0
- sw t0, PT_R7(sp) # set error flag
- beqz t0, 1f
-
- lw t1, PT_R2(sp) # syscall number
- negu v0 # error
- sw t1, PT_R0(sp) # save it for syscall restarting
-1: sw v0, PT_R2(sp) # result
+ j syscall_common
-2: j syscall_exit
+1: j syscall_exit
/* ------------------------------------------------------------------------ */
PTR sys_memfd_create
PTR sys_bpf /* 4355 */
PTR sys_execveat
+ PTR sys_userfaultfd
+ PTR sys_membarrier
.set at
#endif
- dsubu t0, v0, __NR_64_Linux # check syscall number
- sltiu t0, t0, __NR_64_Linux_syscalls + 1
#if !defined(CONFIG_MIPS32_O32) && !defined(CONFIG_MIPS32_N32)
ld t1, PT_EPC(sp) # skip syscall on return
daddiu t1, 4 # skip to next instruction
sd t1, PT_EPC(sp)
#endif
- beqz t0, illegal_syscall
-
- dsll t0, v0, 3 # offset into table
- ld t2, (sys_call_table - (__NR_64_Linux * 8))(t0)
- # syscall routine
sd a3, PT_R26(sp) # save a3 for syscall restarting
and t0, t1, t0
bnez t0, syscall_trace_entry
+syscall_common:
+ dsubu t2, v0, __NR_64_Linux
+ sltiu t0, t2, __NR_64_Linux_syscalls + 1
+ beqz t0, illegal_syscall
+
+ dsll t0, t2, 3 # offset into table
+ dla t2, sys_call_table
+ daddu t0, t2, t0
+ ld t2, (t0) # syscall routine
+ beqz t2, illegal_syscall
+
jalr t2 # Do The Real Thing (TM)
li t0, -EMAXERRNO - 1 # error?
syscall_trace_entry:
SAVE_STATIC
- move s0, t2
+ move s0, v0
move a0, sp
move a1, v0
jal syscall_trace_enter
- bltz v0, 2f # seccomp failed? Skip syscall
+ bltz v0, 1f # seccomp failed? Skip syscall
- move t0, s0
+ move v0, s0
RESTORE_STATIC
ld a0, PT_R4(sp) # Restore argument registers
ld a1, PT_R5(sp)
ld a3, PT_R7(sp)
ld a4, PT_R8(sp)
ld a5, PT_R9(sp)
- jalr t0
-
- li t0, -EMAXERRNO - 1 # error?
- sltu t0, t0, v0
- sd t0, PT_R7(sp) # set error flag
- beqz t0, 1f
-
- ld t1, PT_R2(sp) # syscall number
- dnegu v0 # error
- sd t1, PT_R0(sp) # save it for syscall restarting
-1: sd v0, PT_R2(sp) # result
+ j syscall_common
-2: j syscall_exit
+1: j syscall_exit
illegal_syscall:
/* This also isn't a 64-bit syscall, throw an error. */
PTR sys_memfd_create
PTR sys_bpf /* 5315 */
PTR sys_execveat
+ PTR sys_userfaultfd
+ PTR sys_membarrier
.size sys_call_table,.-sys_call_table
and t0, t1, t0
bnez t0, n32_syscall_trace_entry
+syscall_common:
jalr t2 # Do The Real Thing (TM)
li t0, -EMAXERRNO - 1 # error?
move a1, v0
jal syscall_trace_enter
- bltz v0, 2f # seccomp failed? Skip syscall
+ bltz v0, 1f # seccomp failed? Skip syscall
- move t0, s0
+ move t2, s0
RESTORE_STATIC
ld a0, PT_R4(sp) # Restore argument registers
ld a1, PT_R5(sp)
ld a3, PT_R7(sp)
ld a4, PT_R8(sp)
ld a5, PT_R9(sp)
- jalr t0
+ j syscall_common
- li t0, -EMAXERRNO - 1 # error?
- sltu t0, t0, v0
- sd t0, PT_R7(sp) # set error flag
- beqz t0, 1f
-
- ld t1, PT_R2(sp) # syscall number
- dnegu v0 # error
- sd t1, PT_R0(sp) # save it for syscall restarting
-1: sd v0, PT_R2(sp) # result
-
-2: j syscall_exit
+1: j syscall_exit
not_n32_scall:
/* This is not an n32 compatibility syscall, pass it on to
PTR sys_memfd_create
PTR sys_bpf
PTR compat_sys_execveat /* 6320 */
+ PTR sys_userfaultfd
+ PTR sys_membarrier
.size sysn32_call_table,.-sysn32_call_table
and t0, t1, t0
bnez t0, trace_a_syscall
+syscall_common:
jalr t2 # Do The Real Thing (TM)
li t0, -EMAXERRNO - 1 # error?
1: jal syscall_trace_enter
- bltz v0, 2f # seccomp failed? Skip syscall
+ bltz v0, 1f # seccomp failed? Skip syscall
- move t0, s0
+ move t2, s0
RESTORE_STATIC
ld a0, PT_R4(sp) # Restore argument registers
ld a1, PT_R5(sp)
ld a5, PT_R9(sp)
ld a6, PT_R10(sp)
ld a7, PT_R11(sp) # For indirect syscalls
- jalr t0
+ j syscall_common
- li t0, -EMAXERRNO - 1 # error?
- sltu t0, t0, v0
- sd t0, PT_R7(sp) # set error flag
- beqz t0, 1f
-
- ld t1, PT_R2(sp) # syscall number
- dnegu v0 # error
- sd t1, PT_R0(sp) # save it for syscall restarting
-1: sd v0, PT_R2(sp) # result
-
-2: j syscall_exit
+1: j syscall_exit
/* ------------------------------------------------------------------------ */
PTR sys_memfd_create
PTR sys_bpf /* 4355 */
PTR compat_sys_execveat
+ PTR sys_userfaultfd
+ PTR sys_membarrier
.size sys32_call_table,.-sys32_call_table
if (end <= reserved_end)
continue;
#ifdef CONFIG_BLK_DEV_INITRD
- /* mapstart should be after initrd_end */
+ /* Skip zones before initrd and initrd itself */
if (initrd_end && end <= (unsigned long)PFN_UP(__pa(initrd_end)))
continue;
#endif
max_low_pfn = PFN_DOWN(HIGHMEM_START);
}
+#ifdef CONFIG_BLK_DEV_INITRD
+ /*
+ * mapstart should be after initrd_end
+ */
+ if (initrd_end)
+ mapstart = max(mapstart, (unsigned long)PFN_UP(__pa(initrd_end)));
+#endif
+
/*
* Initialize the boot-time allocator with low memory only.
*/
#include <asm/mmu_context.h>
#include <asm/time.h>
#include <asm/setup.h>
+#include <asm/maar.h>
cpumask_t cpu_callin_map; /* Bitmask of started secondaries */
mips_clockevent_init();
mp_ops->init_secondary();
cpu_report();
+ maar_init();
/*
* XXX parity protection should be folded in here when it's converted
{ "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU },
{ "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
+ { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU },
{ "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU },
{NULL}
};
}
if (memsize == 0)
memsize = 256;
+
+ loongson_sysconf.nr_uarts = 1;
+
pr_info("memsize=%u, highmemsize=%u\n", memsize, highmemsize);
#else
struct boot_params *boot_p;
else
#endif
#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
- if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
+ if (dev->coherent_dma_mask < DMA_BIT_MASK(sizeof(phys_addr_t) * 8))
dma_flag = __GFP_DMA;
else
#endif
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/fixmap.h>
+#include <asm/maar.h>
/*
* We have up to 8 empty zeroed pages so we can map one of the right colour
#endif
}
+unsigned __weak platform_maar_init(unsigned num_pairs)
+{
+ struct maar_config cfg[BOOT_MEM_MAP_MAX];
+ unsigned i, num_configured, num_cfg = 0;
+ phys_addr_t skip;
+
+ for (i = 0; i < boot_mem_map.nr_map; i++) {
+ switch (boot_mem_map.map[i].type) {
+ case BOOT_MEM_RAM:
+ case BOOT_MEM_INIT_RAM:
+ break;
+ default:
+ continue;
+ }
+
+ skip = 0x10000 - (boot_mem_map.map[i].addr & 0xffff);
+
+ cfg[num_cfg].lower = boot_mem_map.map[i].addr;
+ cfg[num_cfg].lower += skip;
+
+ cfg[num_cfg].upper = cfg[num_cfg].lower;
+ cfg[num_cfg].upper += boot_mem_map.map[i].size - 1;
+ cfg[num_cfg].upper -= skip;
+
+ cfg[num_cfg].attrs = MIPS_MAAR_S;
+ num_cfg++;
+ }
+
+ num_configured = maar_config(cfg, num_cfg, num_pairs);
+ if (num_configured < num_cfg)
+ pr_warn("Not enough MAAR pairs (%u) for all bootmem regions (%u)\n",
+ num_pairs, num_cfg);
+
+ return num_configured;
+}
+
+void maar_init(void)
+{
+ unsigned num_maars, used, i;
+ phys_addr_t lower, upper, attr;
+ static struct {
+ struct maar_config cfgs[3];
+ unsigned used;
+ } recorded = { { { 0 } }, 0 };
+
+ if (!cpu_has_maar)
+ return;
+
+ /* Detect the number of MAARs */
+ write_c0_maari(~0);
+ back_to_back_c0_hazard();
+ num_maars = read_c0_maari() + 1;
+
+ /* MAARs should be in pairs */
+ WARN_ON(num_maars % 2);
+
+ /* Set MAARs using values we recorded already */
+ if (recorded.used) {
+ used = maar_config(recorded.cfgs, recorded.used, num_maars / 2);
+ BUG_ON(used != recorded.used);
+ } else {
+ /* Configure the required MAARs */
+ used = platform_maar_init(num_maars / 2);
+ }
+
+ /* Disable any further MAARs */
+ for (i = (used * 2); i < num_maars; i++) {
+ write_c0_maari(i);
+ back_to_back_c0_hazard();
+ write_c0_maar(0);
+ back_to_back_c0_hazard();
+ }
+
+ if (recorded.used)
+ return;
+
+ pr_info("MAAR configuration:\n");
+ for (i = 0; i < num_maars; i += 2) {
+ write_c0_maari(i);
+ back_to_back_c0_hazard();
+ upper = read_c0_maar();
+
+ write_c0_maari(i + 1);
+ back_to_back_c0_hazard();
+ lower = read_c0_maar();
+
+ attr = lower & upper;
+ lower = (lower & MIPS_MAAR_ADDR) << 4;
+ upper = ((upper & MIPS_MAAR_ADDR) << 4) | 0xffff;
+
+ pr_info(" [%d]: ", i / 2);
+ if (!(attr & MIPS_MAAR_V)) {
+ pr_cont("disabled\n");
+ continue;
+ }
+
+ pr_cont("%pa-%pa", &lower, &upper);
+
+ if (attr & MIPS_MAAR_S)
+ pr_cont(" speculate");
+
+ pr_cont("\n");
+
+ /* Record the setup for use on secondary CPUs */
+ if (used <= ARRAY_SIZE(recorded.cfgs)) {
+ recorded.cfgs[recorded.used].lower = lower;
+ recorded.cfgs[recorded.used].upper = upper;
+ recorded.cfgs[recorded.used].attrs = attr;
+ recorded.used++;
+ }
+ }
+}
+
#ifndef CONFIG_NEED_MULTIPLE_NODES
int page_is_ram(unsigned long pagenr)
{
#endif
}
-unsigned __weak platform_maar_init(unsigned num_pairs)
-{
- struct maar_config cfg[BOOT_MEM_MAP_MAX];
- unsigned i, num_configured, num_cfg = 0;
- phys_addr_t skip;
-
- for (i = 0; i < boot_mem_map.nr_map; i++) {
- switch (boot_mem_map.map[i].type) {
- case BOOT_MEM_RAM:
- case BOOT_MEM_INIT_RAM:
- break;
- default:
- continue;
- }
-
- skip = 0x10000 - (boot_mem_map.map[i].addr & 0xffff);
-
- cfg[num_cfg].lower = boot_mem_map.map[i].addr;
- cfg[num_cfg].lower += skip;
-
- cfg[num_cfg].upper = cfg[num_cfg].lower;
- cfg[num_cfg].upper += boot_mem_map.map[i].size - 1;
- cfg[num_cfg].upper -= skip;
-
- cfg[num_cfg].attrs = MIPS_MAAR_S;
- num_cfg++;
- }
-
- num_configured = maar_config(cfg, num_cfg, num_pairs);
- if (num_configured < num_cfg)
- pr_warn("Not enough MAAR pairs (%u) for all bootmem regions (%u)\n",
- num_pairs, num_cfg);
-
- return num_configured;
-}
-
-static void maar_init(void)
-{
- unsigned num_maars, used, i;
-
- if (!cpu_has_maar)
- return;
-
- /* Detect the number of MAARs */
- write_c0_maari(~0);
- back_to_back_c0_hazard();
- num_maars = read_c0_maari() + 1;
-
- /* MAARs should be in pairs */
- WARN_ON(num_maars % 2);
-
- /* Configure the required MAARs */
- used = platform_maar_init(num_maars / 2);
-
- /* Disable any further MAARs */
- for (i = (used * 2); i < num_maars; i++) {
- write_c0_maari(i);
- back_to_back_c0_hazard();
- write_c0_maar(0);
- back_to_back_c0_hazard();
- }
-}
-
void __init mem_init(void)
{
#ifdef CONFIG_HIGHMEM
LEAF(sk_load_word)
is_offset_negative(word)
- .globl sk_load_word_positive
-sk_load_word_positive:
+FEXPORT(sk_load_word_positive)
is_offset_in_header(4, word)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
+ .set reorder
lw $r_A, 0(t1)
+ .set noreorder
#ifdef CONFIG_CPU_LITTLE_ENDIAN
+# if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
wsbh t0, $r_A
rotr $r_A, t0, 16
+# else
+ sll t0, $r_A, 24
+ srl t1, $r_A, 24
+ srl t2, $r_A, 8
+ or t0, t0, t1
+ andi t2, t2, 0xff00
+ andi t1, $r_A, 0xff00
+ or t0, t0, t2
+ sll t1, t1, 8
+ or $r_A, t0, t1
+# endif
#endif
jr $r_ra
move $r_ret, zero
LEAF(sk_load_half)
is_offset_negative(half)
- .globl sk_load_half_positive
-sk_load_half_positive:
+FEXPORT(sk_load_half_positive)
is_offset_in_header(2, half)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
+ .set reorder
lh $r_A, 0(t1)
+ .set noreorder
#ifdef CONFIG_CPU_LITTLE_ENDIAN
+# if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
wsbh t0, $r_A
seh $r_A, t0
+# else
+ sll t0, $r_A, 24
+ andi t1, $r_A, 0xff00
+ sra t0, t0, 16
+ srl t1, t1, 8
+ or $r_A, t0, t1
+# endif
#endif
jr $r_ra
move $r_ret, zero
LEAF(sk_load_byte)
is_offset_negative(byte)
- .globl sk_load_byte_positive
-sk_load_byte_positive:
+FEXPORT(sk_load_byte_positive)
is_offset_in_header(1, byte)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp)
bpf_slow_path_common(4)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
+# if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
wsbh t0, $r_s0
jr $r_ra
rotr $r_A, t0, 16
-#endif
+# else
+ sll t0, $r_s0, 24
+ srl t1, $r_s0, 24
+ srl t2, $r_s0, 8
+ or t0, t0, t1
+ andi t2, t2, 0xff00
+ andi t1, $r_s0, 0xff00
+ or t0, t0, t2
+ sll t1, t1, 8
+ jr $r_ra
+ or $r_A, t0, t1
+# endif
+#else
jr $r_ra
- move $r_A, $r_s0
+ move $r_A, $r_s0
+#endif
END(bpf_slow_path_word)
NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp)
bpf_slow_path_common(2)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
+# if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
jr $r_ra
wsbh $r_A, $r_s0
-#endif
+# else
+ sll t0, $r_s0, 8
+ andi t1, $r_s0, 0xff00
+ andi t0, t0, 0xff00
+ srl t1, t1, 8
+ jr $r_ra
+ or $r_A, t0, t1
+# endif
+#else
jr $r_ra
move $r_A, $r_s0
+#endif
END(bpf_slow_path_half)
}
/* IRQ_IPI_SMP_FUNCTION Handler */
-void nlm_smp_function_ipi_handler(unsigned int __irq, struct irq_desc *desc)
+void nlm_smp_function_ipi_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
clear_c0_eimr(irq);
}
/* IRQ_IPI_SMP_RESCHEDULE handler */
-void nlm_smp_resched_ipi_handler(unsigned int __irq, struct irq_desc *desc)
+void nlm_smp_resched_ipi_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
clear_c0_eimr(irq);
return 0;
}
-static void ar2315_pci_irq_handler(unsigned irq, struct irq_desc *desc)
+static void ar2315_pci_irq_handler(struct irq_desc *desc)
{
struct ar2315_pci_ctrl *apc = irq_desc_get_handler_data(desc);
u32 pending = ar2315_pci_reg_read(apc, AR2315_PCI_ISR) &
.write = ar71xx_pci_write_config,
};
-static void ar71xx_pci_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ar71xx_pci_irq_handler(struct irq_desc *desc)
{
struct ar71xx_pci_controller *apc;
void __iomem *base = ath79_reset_base;
.write = ar724x_pci_write,
};
-static void ar724x_pci_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ar724x_pci_irq_handler(struct irq_desc *desc)
{
struct ar724x_pci_controller *apc;
void __iomem *base;
rt3883_pci_w32(rpc, val, RT3883_PCI_REG_CFGDATA);
}
-static void rt3883_pci_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void rt3883_pci_irq_handler(struct irq_desc *desc)
{
struct rt3883_pci_controller *rpc;
u32 pending;
void pcibios_fixup_bus(struct pci_bus *bus)
{
+ struct pci_dev *dev = bus->self;
+
+ if (pci_has_flag(PCI_PROBE_ONLY) && dev &&
+ (dev->class >> 8) == PCI_CLASS_BRIDGE_PCI) {
+ pci_read_bridge_bases(bus);
+ }
}
EXPORT_SYMBOL(PCIBIOS_MIN_IO);
return CP0_LEGACY_COMPARE_IRQ;
}
-static void ralink_intc_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ralink_intc_irq_handler(struct irq_desc *desc)
{
u32 pending = rt_intc_r32(INTC_REG_STATUS0);
generic-y += preempt.h
generic-y += sections.h
generic-y += trace_clock.h
+generic-y += word-at-a-time.h
struct pci_dev *dev;
if (bus->self) {
+ pci_read_bridge_bases(bus);
pcibios_fixup_bridge_resources(bus->self);
}
generic-y += unaligned.h
generic-y += user.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
endif
ifdef CONFIG_CPU_BIG_ENDIAN
BOOTCFLAGS += -mbig-endian
+else
+BOOTCFLAGS += -mlittle-endian
+BOOTCFLAGS += $(call cc-option,-mabi=elfv2)
endif
BOOTAFLAGS := -D__ASSEMBLY__ $(BOOTCFLAGS) -traditional -nostdinc
#ifdef CONFIG_KVM_MMIO
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
#endif
+#define KVM_HALT_POLL_NS_DEFAULT 500000
/* These values are internal and can be increased later */
#define KVM_NR_IRQCHIPS 1
u32 dec_exits;
u32 ext_intr_exits;
u32 halt_successful_poll;
+ u32 halt_attempted_poll;
u32 halt_wakeup;
u32 dbell_exits;
u32 gdbell_exits;
#ifdef CONFIG_QUICC_ENGINE
void qe_ic_init(struct device_node *node, unsigned int flags,
- void (*low_handler)(unsigned int irq, struct irq_desc *desc),
- void (*high_handler)(unsigned int irq, struct irq_desc *desc));
+ void (*low_handler)(struct irq_desc *desc),
+ void (*high_handler)(struct irq_desc *desc));
unsigned int qe_ic_get_low_irq(struct qe_ic *qe_ic);
unsigned int qe_ic_get_high_irq(struct qe_ic *qe_ic);
#else
static inline void qe_ic_init(struct device_node *node, unsigned int flags,
- void (*low_handler)(unsigned int irq, struct irq_desc *desc),
- void (*high_handler)(unsigned int irq, struct irq_desc *desc))
+ void (*low_handler)(struct irq_desc *desc),
+ void (*high_handler)(struct irq_desc *desc))
{}
static inline unsigned int qe_ic_get_low_irq(struct qe_ic *qe_ic)
{ return 0; }
int qe_ic_set_priority(unsigned int virq, unsigned int priority);
int qe_ic_set_high_priority(unsigned int virq, unsigned int priority, int high);
-static inline void qe_ic_cascade_low_ipic(unsigned int irq,
- struct irq_desc *desc)
+static inline void qe_ic_cascade_low_ipic(struct irq_desc *desc)
{
struct qe_ic *qe_ic = irq_desc_get_handler_data(desc);
unsigned int cascade_irq = qe_ic_get_low_irq(qe_ic);
generic_handle_irq(cascade_irq);
}
-static inline void qe_ic_cascade_high_ipic(unsigned int irq,
- struct irq_desc *desc)
+static inline void qe_ic_cascade_high_ipic(struct irq_desc *desc)
{
struct qe_ic *qe_ic = irq_desc_get_handler_data(desc);
unsigned int cascade_irq = qe_ic_get_high_irq(qe_ic);
generic_handle_irq(cascade_irq);
}
-static inline void qe_ic_cascade_low_mpic(unsigned int irq,
- struct irq_desc *desc)
+static inline void qe_ic_cascade_low_mpic(struct irq_desc *desc)
{
struct qe_ic *qe_ic = irq_desc_get_handler_data(desc);
unsigned int cascade_irq = qe_ic_get_low_irq(qe_ic);
chip->irq_eoi(&desc->irq_data);
}
-static inline void qe_ic_cascade_high_mpic(unsigned int irq,
- struct irq_desc *desc)
+static inline void qe_ic_cascade_high_mpic(struct irq_desc *desc)
{
struct qe_ic *qe_ic = irq_desc_get_handler_data(desc);
unsigned int cascade_irq = qe_ic_get_high_irq(qe_ic);
chip->irq_eoi(&desc->irq_data);
}
-static inline void qe_ic_cascade_muxed_mpic(unsigned int irq,
- struct irq_desc *desc)
+static inline void qe_ic_cascade_muxed_mpic(struct irq_desc *desc)
{
struct qe_ic *qe_ic = irq_desc_get_handler_data(desc);
unsigned int cascade_irq;
SYSCALL_SPU(bpf)
COMPAT_SYS(execveat)
PPC64ONLY(switch_endian)
+SYSCALL_SPU(userfaultfd)
+SYSCALL_SPU(membarrier)
extern int tsi108_setup_pci(struct device_node *dev, u32 cfg_phys, int primary);
extern void tsi108_pci_int_init(struct device_node *node);
-extern void tsi108_irq_cascade(unsigned int irq, struct irq_desc *desc);
+extern void tsi108_irq_cascade(struct irq_desc *desc);
extern void tsi108_clear_pci_cfg_error(void);
#endif /* _ASM_POWERPC_TSI108_PCI_H */
#include <uapi/asm/unistd.h>
-#define __NR_syscalls 364
+#define __NR_syscalls 366
#define __NR__exit __NR_exit
#define NR_syscalls __NR_syscalls
return (val + c->high_bits) & ~rhs;
}
+static inline unsigned long zero_bytemask(unsigned long mask)
+{
+ return ~1ul << __fls(mask);
+}
+
#else
#ifdef CONFIG_64BIT
#define __NR_bpf 361
#define __NR_execveat 362
#define __NR_switch_endian 363
+#define __NR_userfaultfd 364
+#define __NR_membarrier 365
#endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */
chip = irq_data_get_irq_chip(data);
- cpumask_and(mask, data->affinity, map);
+ cpumask_and(mask, irq_data_get_affinity_mask(data), map);
if (cpumask_any(mask) >= nr_cpu_ids) {
pr_warn("Breaking affinity for irq %i\n", irq);
cpumask_copy(mask, map);
void pcibios_fixup_bus(struct pci_bus *bus)
{
- /* Fixup the bus */
+ /* When called from the generic PCI probe, read PCI<->PCI bridge
+ * bases. This is -not- called when generating the PCI tree from
+ * the OF device-tree.
+ */
+ pci_read_bridge_bases(bus);
+
+ /* Now fixup the bus bus */
pcibios_setup_bus_self(bus);
/* Now fixup devices on that bus */
#include <asm/udbg.h>
#include <asm/mmu_context.h>
#include <asm/epapr_hcalls.h>
+#include <asm/code-patching.h>
#define DBG(fmt...)
* This is called very early on the boot process, after a minimal
* MMU environment has been set up but before MMU_init is called.
*/
+extern unsigned int memset_nocache_branch; /* Insn to be replaced by NOP */
+
notrace void __init machine_init(u64 dt_ptr)
{
lockdep_init();
/* Enable early debugging if any specified (see udbg.h) */
udbg_early_init();
+ patch_instruction((unsigned int *)&memcpy, PPC_INST_NOP);
+ patch_instruction(&memset_nocache_branch, PPC_INST_NOP);
+
/* Do some early initialization based on the flat device tree */
early_init_devtree(__va(dt_ptr));
{ "ext_intr", VCPU_STAT(ext_intr_exits) },
{ "queue_intr", VCPU_STAT(queue_intr) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll), },
+ { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "pf_storage", VCPU_STAT(pf_storage) },
{ "sp_storage", VCPU_STAT(sp_storage) },
unsigned long size = kvmppc_get_gpr(vcpu, 4);
unsigned long addr = kvmppc_get_gpr(vcpu, 5);
u64 buf;
+ int srcu_idx;
int ret;
if (!is_power_of_2(size) || (size > sizeof(buf)))
return H_TOO_HARD;
+ srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, size, &buf);
+ srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
if (ret != 0)
return H_TOO_HARD;
unsigned long addr = kvmppc_get_gpr(vcpu, 5);
unsigned long val = kvmppc_get_gpr(vcpu, 6);
u64 buf;
+ int srcu_idx;
int ret;
switch (size) {
return H_TOO_HARD;
}
+ srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, size, &buf);
+ srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
if (ret != 0)
return H_TOO_HARD;
while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
(vc->vcore_state == VCORE_RUNNING ||
- vc->vcore_state == VCORE_EXITING))
+ vc->vcore_state == VCORE_EXITING ||
+ vc->vcore_state == VCORE_PIGGYBACK))
kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
+ if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
+ kvmppc_vcore_end_preempt(vc);
+
if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
kvmppc_remove_runnable(vc, vcpu);
vcpu->stat.signal_exits++;
bl kvmhv_accumulate_time
#endif
+ mr r3, r12
/* Increment exit count, poke other threads to exit */
bl kvmhv_commence_exit
nop
{ "dec", VCPU_STAT(dec_exits) },
{ "ext_intr", VCPU_STAT(ext_intr_exits) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
+ { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "doorbell", VCPU_STAT(dbell_exits) },
{ "guest doorbell", VCPU_STAT(gdbell_exits) },
* Use dcbz on the complete cache lines in the destination
* to set them to zero. This requires that the destination
* area is cacheable. -- paulus
+ *
+ * During early init, cache might not be active yet, so dcbz cannot be used.
+ * We therefore skip the optimised bloc that uses dcbz. This jump is
+ * replaced by a nop once cache is active. This is done in machine_init()
*/
_GLOBAL(memset)
rlwimi r4,r4,8,16,23
subf r6,r0,r6
cmplwi 0,r4,0
bne 2f /* Use normal procedure if r4 is not zero */
+_GLOBAL(memset_nocache_branch)
+ b 2f /* Skip optimised bloc until cache is enabled */
clrlwi r7,r6,32-LG_CACHELINE_BYTES
add r8,r7,r5
* the destination area is cacheable.
* We only use this version if the source and dest don't overlap.
* -- paulus.
+ *
+ * During early init, cache might not be active yet, so dcbz cannot be used.
+ * We therefore jump to generic_memcpy which doesn't use dcbz. This jump is
+ * replaced by a nop once cache is active. This is done in machine_init()
*/
_GLOBAL(memmove)
cmplw 0,r3,r4
/* fall through */
_GLOBAL(memcpy)
+ b generic_memcpy
add r7,r3,r5 /* test if the src & dst overlap */
add r8,r4,r5
cmplw 0,r4,r7
BUG_ON(index >= 4096);
vpn = hpt_vpn(ea, vsid, ssize);
- hash = hpt_hash(vpn, shift, ssize);
hpte_slot_array = get_hpte_slot_array(pmdp);
if (psize == MMU_PAGE_4K) {
/*
valid = hpte_valid(hpte_slot_array, index);
if (valid) {
/* update the hpte bits */
+ hash = hpt_hash(vpn, shift, ssize);
hidx = hpte_hash_index(hpte_slot_array, index);
if (hidx & _PTEIDX_SECONDARY)
hash = ~hash;
if (!valid) {
unsigned long hpte_group;
+ hash = hpt_hash(vpn, shift, ssize);
/* insert new entry */
pa = pmd_pfn(__pmd(old_pmd)) << PAGE_SHIFT;
new_pmd |= _PAGE_HASHPTE;
return irq_linear_revmap(cpld_pic_host, cpld_irq);
}
-static void
-cpld_pic_cascade(unsigned int irq, struct irq_desc *desc)
+static void cpld_pic_cascade(struct irq_desc *desc)
{
+ unsigned int irq;
+
irq = cpld_pic_get_irq(0, PCI_IGNORE, &cpld_regs->pci_status,
&cpld_regs->pci_mask);
if (irq != NO_IRQ) {
.irq_mask_ack = media5200_irq_mask,
};
-void media5200_irq_cascade(unsigned int virq, struct irq_desc *desc)
+static void media5200_irq_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
int sub_virq, val;
.irq_set_type = mpc52xx_gpt_irq_set_type,
};
-void mpc52xx_gpt_irq_cascade(unsigned int virq, struct irq_desc *desc)
+static void mpc52xx_gpt_irq_cascade(struct irq_desc *desc)
{
struct mpc52xx_gpt_priv *gpt = irq_desc_get_handler_data(desc);
int sub_virq;
ctrl_reg |= (type << (22 - (l2irq * 2)));
out_be32(&intr->ctrl, ctrl_reg);
- __irq_set_handler_locked(d->irq, handler);
+ irq_set_handler_locked(d, handler);
return 0;
}
.irq_disable = pq2ads_pci_mask_irq
};
-static void pq2ads_pci_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void pq2ads_pci_irq_demux(struct irq_desc *desc)
{
struct pq2ads_pci_pic *priv = irq_desc_get_handler_data(desc);
u32 stat, mask, pend;
return of_platform_bus_probe(NULL, mpc85xx_common_ids, NULL);
}
#ifdef CONFIG_CPM2
-static void cpm2_cascade(unsigned int irq, struct irq_desc *desc)
+static void cpm2_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
int cascade_irq;
}
#ifdef CONFIG_PPC_I8259
-static void mpc85xx_8259_cascade_handler(unsigned int irq,
- struct irq_desc *desc)
+static void mpc85xx_8259_cascade_handler(struct irq_desc *desc)
{
unsigned int cascade_irq = i8259_irq();
generic_handle_irq(cascade_irq);
/* check for any interrupts from the shared IRQ line */
- handle_fasteoi_irq(irq, desc);
+ handle_fasteoi_irq(desc);
}
static irqreturn_t mpc85xx_8259_cascade_action(int irq, void *dev_id)
#endif
#ifdef CONFIG_PPC_I8259
-static void mpc85xx_8259_cascade(unsigned int irq, struct irq_desc *desc)
+static void mpc85xx_8259_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq = i8259_irq();
(irq_hw_number_t)i);
}
-void socrates_fpga_pic_cascade(unsigned int irq, struct irq_desc *desc)
+static void socrates_fpga_pic_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
+ unsigned int irq = irq_desc_get_irq(desc);
unsigned int cascade_irq;
/*
#include <asm/i8259.h>
#ifdef CONFIG_PPC_I8259
-static void mpc86xx_8259_cascade(unsigned int irq, struct irq_desc *desc)
+static void mpc86xx_8259_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq = i8259_irq();
panic("Restart failed\n");
}
-static void cpm_cascade(unsigned int irq, struct irq_desc *desc)
+static void cpm_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
int cascade_irq = cpm_get_irq();
dcr_write(msic->dcr_host, dcr_n, val);
}
-static void axon_msi_cascade(unsigned int irq, struct irq_desc *desc)
+static void axon_msi_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct axon_msic *msic = irq_desc_get_handler_data(desc);
{
}
-static void iic_ioexc_cascade(unsigned int irq, struct irq_desc *desc)
+static void iic_ioexc_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct cbe_iic_regs __iomem *node_iic =
(void __iomem *)irq_desc_get_handler_data(desc);
+ unsigned int irq = irq_desc_get_irq(desc);
unsigned int base = (irq & 0xffffff00) | IIC_IRQ_TYPE_IOEXC;
unsigned long bits, ack;
int cascade;
.xlate = spider_host_xlate,
};
-static void spider_irq_cascade(unsigned int irq, struct irq_desc *desc)
+static void spider_irq_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct spider_pic *pic = irq_desc_get_handler_data(desc);
if (ppc_md.progress) ppc_md.progress("Linux/PPC "UTS_RELEASE"\n", 0x0);
}
-static void chrp_8259_cascade(unsigned int irq, struct irq_desc *desc)
+static void chrp_8259_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq = i8259_irq();
return irq_linear_revmap(h, irq);
}
-static void hlwd_pic_irq_cascade(unsigned int cascade_virq,
- struct irq_desc *desc)
+static void hlwd_pic_irq_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct irq_domain *irq_domain = irq_desc_get_handler_data(desc);
static phys_addr_t pci_membase;
static u_char *restart;
-static void mvme5100_8259_cascade(unsigned int irq, struct irq_desc *desc)
+static void mvme5100_8259_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq = i8259_irq();
static void pasemi_msi_teardown_msi_irqs(struct pci_dev *pdev)
{
struct msi_desc *entry;
+ irq_hw_number_t hwirq;
pr_debug("pasemi_msi_teardown_msi_irqs, pdev %p\n", pdev);
if (entry->irq == NO_IRQ)
continue;
+ hwirq = virq_to_hw(entry->irq);
irq_set_msi_desc(entry->irq, NULL);
- msi_bitmap_free_hwirqs(&msi_mpic->msi_bitmap,
- virq_to_hw(entry->irq), ALLOC_CHUNK);
irq_dispose_mapping(entry->irq);
+ msi_bitmap_free_hwirqs(&msi_mpic->msi_bitmap, hwirq, ALLOC_CHUNK);
}
return;
struct iommu_table *tbl = NULL;
long rc;
+ /*
+ * crashkernel= specifies the kdump kernel's maximum memory at
+ * some offset and there is no guaranteed the result is a power
+ * of 2, which will cause errors later.
+ */
+ const u64 max_memory = __rounddown_pow_of_two(memory_hotplug_max());
+
+ /*
+ * In memory constrained environments, e.g. kdump kernel, the
+ * DMA window can be larger than available memory, which will
+ * cause errors later.
+ */
+ const u64 window_size = min((u64)pe->table_group.tce32_size, max_memory);
+
rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,
IOMMU_PAGE_SHIFT_4K,
- pe->table_group.tce32_size,
+ window_size,
POWERNV_IOMMU_DEFAULT_LEVELS, &tbl);
if (rc) {
pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
struct pci_controller *hose = pci_bus_to_host(pdev->bus);
struct pnv_phb *phb = hose->private_data;
struct msi_desc *entry;
+ irq_hw_number_t hwirq;
if (WARN_ON(!phb))
return;
for_each_pci_msi_entry(entry, pdev) {
if (entry->irq == NO_IRQ)
continue;
+ hwirq = virq_to_hw(entry->irq);
irq_set_msi_desc(entry->irq, NULL);
- msi_bitmap_free_hwirqs(&phb->msi_bmp,
- virq_to_hw(entry->irq) - phb->msi_base, 1);
irq_dispose_mapping(entry->irq);
+ msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, 1);
}
}
#endif /* CONFIG_PCI_MSI */
dn = dlpar_configure_connector(cpu_to_be32(drc_index), parent);
of_node_put(parent);
- if (!dn)
+ if (!dn) {
+ dlpar_release_drc(drc_index);
return -EINVAL;
+ }
rc = dlpar_attach_node(dn);
if (rc) {
fwnmi_active = 1;
}
-static void pseries_8259_cascade(unsigned int irq, struct irq_desc *desc)
+static void pseries_8259_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq = i8259_irq();
irqd_set_trigger_type(d, flow_type);
if (flow_type & IRQ_TYPE_LEVEL_LOW)
- __irq_set_handler_locked(d->irq, handle_level_irq);
+ irq_set_handler_locked(d, handle_level_irq);
else
- __irq_set_handler_locked(d->irq, handle_edge_irq);
+ irq_set_handler_locked(d, handle_edge_irq);
/* internal IRQ senses are LEVEL_LOW
* EXT IRQ and Port C IRQ senses are programmable
{
struct msi_desc *entry;
struct fsl_msi *msi_data;
+ irq_hw_number_t hwirq;
for_each_pci_msi_entry(entry, pdev) {
if (entry->irq == NO_IRQ)
continue;
+ hwirq = virq_to_hw(entry->irq);
msi_data = irq_get_chip_data(entry->irq);
irq_set_msi_desc(entry->irq, NULL);
- msi_bitmap_free_hwirqs(&msi_data->bitmap,
- virq_to_hw(entry->irq), 1);
irq_dispose_mapping(entry->irq);
+ msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1);
}
return;
* should be masked out.
*/
-void gef_pic_cascade(unsigned int irq, struct irq_desc *desc)
+static void gef_pic_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq;
#ifndef __GEF_PIC_H__
#define __GEF_PIC_H__
-
-void gef_pic_cascade(unsigned int, struct irq_desc *);
unsigned int gef_pic_get_irq(void);
void gef_pic_init(struct device_node *);
irqd_set_trigger_type(d, flow_type);
if (flow_type & IRQ_TYPE_LEVEL_LOW) {
- __irq_set_handler_locked(d->irq, handle_level_irq);
+ irq_set_handler_locked(d, handle_level_irq);
d->chip = &ipic_level_irq_chip;
} else {
- __irq_set_handler_locked(d->irq, handle_edge_irq);
+ irq_set_handler_locked(d, handle_edge_irq);
d->chip = &ipic_edge_irq_chip;
}
unsigned int siel = in_be32(&siu_reg->sc_siel);
siel |= mpc8xx_irqd_to_bit(d);
out_be32(&siu_reg->sc_siel, siel);
- __irq_set_handler_locked(d->irq, handle_edge_irq);
+ irq_set_handler_locked(d, handle_edge_irq);
}
return 0;
}
}
/* IRQ handler for a secondary MPIC cascaded from another IRQ controller */
-static void mpic_cascade(unsigned int irq, struct irq_desc *desc)
+static void mpic_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct mpic *mpic = irq_desc_get_handler_data(desc);
static void u3msi_teardown_msi_irqs(struct pci_dev *pdev)
{
struct msi_desc *entry;
+ irq_hw_number_t hwirq;
for_each_pci_msi_entry(entry, pdev) {
if (entry->irq == NO_IRQ)
continue;
+ hwirq = virq_to_hw(entry->irq);
irq_set_msi_desc(entry->irq, NULL);
- msi_bitmap_free_hwirqs(&msi_mpic->msi_bitmap,
- virq_to_hw(entry->irq), 1);
irq_dispose_mapping(entry->irq);
+ msi_bitmap_free_hwirqs(&msi_mpic->msi_bitmap, hwirq, 1);
}
return;
{
struct msi_desc *entry;
struct ppc4xx_msi *msi_data = &ppc4xx_msi;
+ irq_hw_number_t hwirq;
dev_dbg(&dev->dev, "PCIE-MSI: tearing down msi irqs\n");
for_each_pci_msi_entry(entry, dev) {
if (entry->irq == NO_IRQ)
continue;
+ hwirq = virq_to_hw(entry->irq);
irq_set_msi_desc(entry->irq, NULL);
- msi_bitmap_free_hwirqs(&msi_data->bitmap,
- virq_to_hw(entry->irq), 1);
irq_dispose_mapping(entry->irq);
+ msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1);
}
}
}
void __init qe_ic_init(struct device_node *node, unsigned int flags,
- void (*low_handler)(unsigned int irq, struct irq_desc *desc),
- void (*high_handler)(unsigned int irq, struct irq_desc *desc))
+ void (*low_handler)(struct irq_desc *desc),
+ void (*high_handler)(struct irq_desc *desc))
{
struct qe_ic *qe_ic;
struct resource res;
init_pci_source();
}
-void tsi108_irq_cascade(unsigned int irq, struct irq_desc *desc)
+void tsi108_irq_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq = get_pci_source();
.xlate = irq_domain_xlate_twocell,
};
-void uic_irq_cascade(unsigned int virq, struct irq_desc *desc)
+static void uic_irq_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct irq_data *idata = irq_desc_get_irq_data(desc);
if (hw_irq == XICS_IPI || hw_irq == XICS_IRQ_SPURIOUS)
return;
- server = xics_get_irq_server(d->irq, d->affinity, 0);
+ server = xics_get_irq_server(d->irq, irq_data_get_affinity_mask(d), 0);
server = ics_opal_mangle_server(server);
rc = opal_set_xive(hw_irq, server, DEFAULT_PRIORITY);
if (hw_irq == XICS_IPI || hw_irq == XICS_IRQ_SPURIOUS)
return;
- server = xics_get_irq_server(d->irq, d->affinity, 0);
+ server = xics_get_irq_server(d->irq, irq_data_get_affinity_mask(d), 0);
call_status = rtas_call(ibm_set_xive, 3, 1, NULL, hw_irq, server,
DEFAULT_PRIORITY);
/*
* Support code for cascading to 8259 interrupt controllers
*/
-static void xilinx_i8259_cascade(unsigned int irq, struct irq_desc *desc)
+static void xilinx_i8259_cascade(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned int cascade_irq = i8259_irq();
KBUILD_CFLAGS := -m64 -D__KERNEL__ $(LINUX_INCLUDE) -O2
KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
-KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks
+KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks -msoft-float
KBUILD_CFLAGS += $(call cc-option,-mpacked-stack)
KBUILD_CFLAGS += $(call cc-option,-ffreestanding)
CONFIG_SCSI_DEBUG=m
CONFIG_ZFCP=y
CONFIG_SCSI_VIRTIO=m
-CONFIG_SCSI_DH=m
+CONFIG_SCSI_DH=y
CONFIG_SCSI_DH_RDAC=m
CONFIG_SCSI_DH_HP_SW=m
CONFIG_SCSI_DH_EMC=m
CONFIG_SCSI_DEBUG=m
CONFIG_ZFCP=y
CONFIG_SCSI_VIRTIO=m
-CONFIG_SCSI_DH=m
+CONFIG_SCSI_DH=y
CONFIG_SCSI_DH_RDAC=m
CONFIG_SCSI_DH_HP_SW=m
CONFIG_SCSI_DH_EMC=m
CONFIG_SCSI_DEBUG=m
CONFIG_ZFCP=y
CONFIG_SCSI_VIRTIO=m
-CONFIG_SCSI_DH=m
+CONFIG_SCSI_DH=y
CONFIG_SCSI_DH_RDAC=m
CONFIG_SCSI_DH_HP_SW=m
CONFIG_SCSI_DH_EMC=m
# CONFIG_SWAP is not set
CONFIG_NO_HZ=y
CONFIG_HIGH_RES_TIMERS=y
-CONFIG_RCU_FAST_NO_HZ=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_CC_OPTIMIZE_FOR_SIZE=y
# CONFIG_COMPAT_BRK is not set
# CONFIG_MONWRITER is not set
# CONFIG_S390_VMUR is not set
# CONFIG_HID is not set
-CONFIG_MEMSTICK=y
-CONFIG_MEMSTICK_DEBUG=y
-CONFIG_MEMSTICK_UNSAFE_RESUME=y
-CONFIG_MSPRO_BLOCK=y
# CONFIG_IOMMU_SUPPORT is not set
CONFIG_EXT2_FS=y
CONFIG_EXT3_FS=y
generic-y += mm-arch-hooks.h
generic-y += preempt.h
generic-y += trace_clock.h
+generic-y += word-at-a-time.h
*/
#define KVM_NR_IRQCHIPS 1
#define KVM_IRQCHIP_NUM_PINS 4096
+#define KVM_HALT_POLL_NS_DEFAULT 0
#define SIGP_CTRL_C 0x80
#define SIGP_CTRL_SCN_MASK 0x3f
u32 exit_validity;
u32 exit_instruction;
u32 halt_successful_poll;
+ u32 halt_attempted_poll;
u32 halt_wakeup;
u32 instruction_lctl;
u32 instruction_lctlg;
int __node_distance(int a, int b);
void numa_update_cpu_topology(void);
-extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+extern cpumask_t node_to_cpumask_map[MAX_NUMNODES];
extern int numa_debug_enabled;
#else
#define cpumask_of_node cpumask_of_node
static inline const struct cpumask *cpumask_of_node(int node)
{
- return node_to_cpumask_map[node];
+ return &node_to_cpumask_map[node];
}
/*
#include <uapi/asm/unistd.h>
-
#define __IGNORE_time
-/* Ignore system calls that are also reachable via sys_socketcall */
-#define __IGNORE_recvmmsg
-#define __IGNORE_sendmmsg
-#define __IGNORE_socket
-#define __IGNORE_socketpair
-#define __IGNORE_bind
-#define __IGNORE_connect
-#define __IGNORE_listen
-#define __IGNORE_accept4
-#define __IGNORE_getsockopt
-#define __IGNORE_setsockopt
-#define __IGNORE_getsockname
-#define __IGNORE_getpeername
-#define __IGNORE_sendto
-#define __IGNORE_sendmsg
-#define __IGNORE_recvfrom
-#define __IGNORE_recvmsg
-#define __IGNORE_shutdown
-
#define __ARCH_WANT_OLD_READDIR
#define __ARCH_WANT_SYS_ALARM
#define __ARCH_WANT_SYS_GETHOSTNAME
#define __NR_s390_pci_mmio_write 352
#define __NR_s390_pci_mmio_read 353
#define __NR_execveat 354
-#define NR_syscalls 355
+#define __NR_userfaultfd 355
+#define __NR_membarrier 356
+#define __NR_recvmmsg 357
+#define __NR_sendmmsg 358
+#define __NR_socket 359
+#define __NR_socketpair 360
+#define __NR_bind 361
+#define __NR_connect 362
+#define __NR_listen 363
+#define __NR_accept4 364
+#define __NR_getsockopt 365
+#define __NR_setsockopt 366
+#define __NR_getsockname 367
+#define __NR_getpeername 368
+#define __NR_sendto 369
+#define __NR_sendmsg 370
+#define __NR_recvfrom 371
+#define __NR_recvmsg 372
+#define __NR_shutdown 373
+#define NR_syscalls 374
/*
* There are some system calls that are not present on 64 bit, some
DEFINE(__LC_PASTE, offsetof(struct _lowcore, paste));
DEFINE(__LC_FP_CREG_SAVE_AREA, offsetof(struct _lowcore, fpt_creg_save_area));
DEFINE(__LC_LAST_BREAK, offsetof(struct _lowcore, breaking_event_addr));
+ DEFINE(__LC_PERCPU_OFFSET, offsetof(struct _lowcore, percpu_offset));
DEFINE(__LC_VDSO_PER_CPU, offsetof(struct _lowcore, vdso_per_cpu_data));
DEFINE(__LC_GMAP, offsetof(struct _lowcore, gmap));
DEFINE(__LC_PGM_TDB, offsetof(struct _lowcore, pgm_tdb));
struct ucontext32 uc;
} rt_sigframe32;
+static inline void sigset_to_sigset32(unsigned long *set64,
+ compat_sigset_word *set32)
+{
+ set32[0] = (compat_sigset_word) set64[0];
+ set32[1] = (compat_sigset_word)(set64[0] >> 32);
+}
+
+static inline void sigset32_to_sigset(compat_sigset_word *set32,
+ unsigned long *set64)
+{
+ set64[0] = (unsigned long) set32[0] | ((unsigned long) set32[1] << 32);
+}
+
int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from)
{
int err;
{
struct pt_regs *regs = task_pt_regs(current);
sigframe32 __user *frame = (sigframe32 __user *)regs->gprs[15];
+ compat_sigset_t cset;
sigset_t set;
- if (__copy_from_user(&set.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE32))
+ if (__copy_from_user(&cset.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE32))
goto badframe;
+ sigset32_to_sigset(cset.sig, set.sig);
set_current_blocked(&set);
save_fpu_regs();
if (restore_sigregs32(regs, &frame->sregs))
{
struct pt_regs *regs = task_pt_regs(current);
rt_sigframe32 __user *frame = (rt_sigframe32 __user *)regs->gprs[15];
+ compat_sigset_t cset;
sigset_t set;
- if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
+ if (__copy_from_user(&cset, &frame->uc.uc_sigmask, sizeof(cset)))
goto badframe;
+ sigset32_to_sigset(cset.sig, set.sig);
set_current_blocked(&set);
if (compat_restore_altstack(&frame->uc.uc_stack))
goto badframe;
return -EFAULT;
/* Create struct sigcontext32 on the signal stack */
- memcpy(&sc.oldmask, &set->sig, _SIGMASK_COPY_SIZE32);
+ sigset_to_sigset32(set->sig, sc.oldmask);
sc.sregs = (__u32)(unsigned long __force) &frame->sregs;
if (__copy_to_user(&frame->sc, &sc, sizeof(frame->sc)))
return -EFAULT;
static int setup_rt_frame32(struct ksignal *ksig, sigset_t *set,
struct pt_regs *regs)
{
+ compat_sigset_t cset;
rt_sigframe32 __user *frame;
unsigned long restorer;
size_t frame_size;
store_sigregs();
/* Create ucontext on the signal stack. */
+ sigset_to_sigset32(set->sig, cset.sig);
if (__put_user(uc_flags, &frame->uc.uc_flags) ||
__put_user(0, &frame->uc.uc_link) ||
__compat_save_altstack(&frame->uc.uc_stack, regs->gprs[15]) ||
save_sigregs32(regs, &frame->uc.uc_mcontext) ||
- __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)) ||
+ __copy_to_user(&frame->uc.uc_sigmask, &cset, sizeof(cset)) ||
save_sigregs_ext32(regs, &frame->uc.uc_mcontext_ext))
return -EFAULT;
* the regular system call wrappers.
*/
#define COMPAT_SYSCALL_WRAPx(x, name, ...) \
- asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \
- asmlinkage long compat_sys##name(__MAP(x,__SC_COMPAT_TYPE,__VA_ARGS__));\
- asmlinkage long compat_sys##name(__MAP(x,__SC_COMPAT_TYPE,__VA_ARGS__)) \
- { \
- return sys##name(__MAP(x,__SC_COMPAT_CAST,__VA_ARGS__)); \
- }
+asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \
+asmlinkage long notrace compat_sys##name(__MAP(x,__SC_COMPAT_TYPE,__VA_ARGS__));\
+asmlinkage long notrace compat_sys##name(__MAP(x,__SC_COMPAT_TYPE,__VA_ARGS__)) \
+{ \
+ return sys##name(__MAP(x,__SC_COMPAT_CAST,__VA_ARGS__)); \
+}
-COMPAT_SYSCALL_WRAP1(exit, int, error_code);
-COMPAT_SYSCALL_WRAP1(close, unsigned int, fd);
COMPAT_SYSCALL_WRAP2(creat, const char __user *, pathname, umode_t, mode);
COMPAT_SYSCALL_WRAP2(link, const char __user *, oldname, const char __user *, newname);
COMPAT_SYSCALL_WRAP1(unlink, const char __user *, pathname);
COMPAT_SYSCALL_WRAP3(mknod, const char __user *, filename, umode_t, mode, unsigned, dev);
COMPAT_SYSCALL_WRAP2(chmod, const char __user *, filename, umode_t, mode);
COMPAT_SYSCALL_WRAP1(oldumount, char __user *, name);
-COMPAT_SYSCALL_WRAP1(alarm, unsigned int, seconds);
COMPAT_SYSCALL_WRAP2(access, const char __user *, filename, int, mode);
-COMPAT_SYSCALL_WRAP1(nice, int, increment);
-COMPAT_SYSCALL_WRAP2(kill, int, pid, int, sig);
COMPAT_SYSCALL_WRAP2(rename, const char __user *, oldname, const char __user *, newname);
COMPAT_SYSCALL_WRAP2(mkdir, const char __user *, pathname, umode_t, mode);
COMPAT_SYSCALL_WRAP1(rmdir, const char __user *, pathname);
-COMPAT_SYSCALL_WRAP1(dup, unsigned int, fildes);
COMPAT_SYSCALL_WRAP1(pipe, int __user *, fildes);
COMPAT_SYSCALL_WRAP1(brk, unsigned long, brk);
COMPAT_SYSCALL_WRAP2(signal, int, sig, __sighandler_t, handler);
COMPAT_SYSCALL_WRAP1(acct, const char __user *, name);
COMPAT_SYSCALL_WRAP2(umount, char __user *, name, int, flags);
-COMPAT_SYSCALL_WRAP2(setpgid, pid_t, pid, pid_t, pgid);
-COMPAT_SYSCALL_WRAP1(umask, int, mask);
COMPAT_SYSCALL_WRAP1(chroot, const char __user *, filename);
-COMPAT_SYSCALL_WRAP2(dup2, unsigned int, oldfd, unsigned int, newfd);
COMPAT_SYSCALL_WRAP3(sigsuspend, int, unused1, int, unused2, old_sigset_t, mask);
COMPAT_SYSCALL_WRAP2(sethostname, char __user *, name, int, len);
COMPAT_SYSCALL_WRAP2(symlink, const char __user *, old, const char __user *, new);
COMPAT_SYSCALL_WRAP2(swapon, const char __user *, specialfile, int, swap_flags);
COMPAT_SYSCALL_WRAP4(reboot, int, magic1, int, magic2, unsigned int, cmd, void __user *, arg);
COMPAT_SYSCALL_WRAP2(munmap, unsigned long, addr, size_t, len);
-COMPAT_SYSCALL_WRAP2(fchmod, unsigned int, fd, umode_t, mode);
-COMPAT_SYSCALL_WRAP2(getpriority, int, which, int, who);
-COMPAT_SYSCALL_WRAP3(setpriority, int, which, int, who, int, niceval);
COMPAT_SYSCALL_WRAP3(syslog, int, type, char __user *, buf, int, len);
COMPAT_SYSCALL_WRAP1(swapoff, const char __user *, specialfile);
-COMPAT_SYSCALL_WRAP1(fsync, unsigned int, fd);
COMPAT_SYSCALL_WRAP2(setdomainname, char __user *, name, int, len);
COMPAT_SYSCALL_WRAP1(newuname, struct new_utsname __user *, name);
COMPAT_SYSCALL_WRAP3(mprotect, unsigned long, start, size_t, len, unsigned long, prot);
COMPAT_SYSCALL_WRAP3(init_module, void __user *, umod, unsigned long, len, const char __user *, uargs);
COMPAT_SYSCALL_WRAP2(delete_module, const char __user *, name_user, unsigned int, flags);
COMPAT_SYSCALL_WRAP4(quotactl, unsigned int, cmd, const char __user *, special, qid_t, id, void __user *, addr);
-COMPAT_SYSCALL_WRAP1(getpgid, pid_t, pid);
-COMPAT_SYSCALL_WRAP1(fchdir, unsigned int, fd);
COMPAT_SYSCALL_WRAP2(bdflush, int, func, long, data);
COMPAT_SYSCALL_WRAP3(sysfs, int, option, unsigned long, arg1, unsigned long, arg2);
-COMPAT_SYSCALL_WRAP1(s390_personality, unsigned int, personality);
COMPAT_SYSCALL_WRAP5(llseek, unsigned int, fd, unsigned long, high, unsigned long, low, loff_t __user *, result, unsigned int, whence);
-COMPAT_SYSCALL_WRAP2(flock, unsigned int, fd, unsigned int, cmd);
COMPAT_SYSCALL_WRAP3(msync, unsigned long, start, size_t, len, int, flags);
-COMPAT_SYSCALL_WRAP1(getsid, pid_t, pid);
-COMPAT_SYSCALL_WRAP1(fdatasync, unsigned int, fd);
COMPAT_SYSCALL_WRAP2(mlock, unsigned long, start, size_t, len);
COMPAT_SYSCALL_WRAP2(munlock, unsigned long, start, size_t, len);
-COMPAT_SYSCALL_WRAP1(mlockall, int, flags);
COMPAT_SYSCALL_WRAP2(sched_setparam, pid_t, pid, struct sched_param __user *, param);
COMPAT_SYSCALL_WRAP2(sched_getparam, pid_t, pid, struct sched_param __user *, param);
COMPAT_SYSCALL_WRAP3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param);
-COMPAT_SYSCALL_WRAP1(sched_getscheduler, pid_t, pid);
-COMPAT_SYSCALL_WRAP1(sched_get_priority_max, int, policy);
-COMPAT_SYSCALL_WRAP1(sched_get_priority_min, int, policy);
COMPAT_SYSCALL_WRAP5(mremap, unsigned long, addr, unsigned long, old_len, unsigned long, new_len, unsigned long, flags, unsigned long, new_addr);
COMPAT_SYSCALL_WRAP3(poll, struct pollfd __user *, ufds, unsigned int, nfds, int, timeout);
COMPAT_SYSCALL_WRAP5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5);
COMPAT_SYSCALL_WRAP2(capget, cap_user_header_t, header, cap_user_data_t, dataptr);
COMPAT_SYSCALL_WRAP2(capset, cap_user_header_t, header, const cap_user_data_t, data);
COMPAT_SYSCALL_WRAP3(lchown, const char __user *, filename, uid_t, user, gid_t, group);
-COMPAT_SYSCALL_WRAP2(setreuid, uid_t, ruid, uid_t, euid);
-COMPAT_SYSCALL_WRAP2(setregid, gid_t, rgid, gid_t, egid);
COMPAT_SYSCALL_WRAP2(getgroups, int, gidsetsize, gid_t __user *, grouplist);
COMPAT_SYSCALL_WRAP2(setgroups, int, gidsetsize, gid_t __user *, grouplist);
-COMPAT_SYSCALL_WRAP3(fchown, unsigned int, fd, uid_t, user, gid_t, group);
-COMPAT_SYSCALL_WRAP3(setresuid, uid_t, ruid, uid_t, euid, uid_t, suid);
COMPAT_SYSCALL_WRAP3(getresuid, uid_t __user *, ruid, uid_t __user *, euid, uid_t __user *, suid);
-COMPAT_SYSCALL_WRAP3(setresgid, gid_t, rgid, gid_t, egid, gid_t, sgid);
COMPAT_SYSCALL_WRAP3(getresgid, gid_t __user *, rgid, gid_t __user *, egid, gid_t __user *, sgid);
COMPAT_SYSCALL_WRAP3(chown, const char __user *, filename, uid_t, user, gid_t, group);
-COMPAT_SYSCALL_WRAP1(setuid, uid_t, uid);
-COMPAT_SYSCALL_WRAP1(setgid, gid_t, gid);
-COMPAT_SYSCALL_WRAP1(setfsuid, uid_t, uid);
-COMPAT_SYSCALL_WRAP1(setfsgid, gid_t, gid);
COMPAT_SYSCALL_WRAP2(pivot_root, const char __user *, new_root, const char __user *, put_old);
COMPAT_SYSCALL_WRAP3(mincore, unsigned long, start, size_t, len, unsigned char __user *, vec);
COMPAT_SYSCALL_WRAP3(madvise, unsigned long, start, size_t, len, int, behavior);
COMPAT_SYSCALL_WRAP2(removexattr, const char __user *, path, const char __user *, name);
COMPAT_SYSCALL_WRAP2(lremovexattr, const char __user *, path, const char __user *, name);
COMPAT_SYSCALL_WRAP2(fremovexattr, int, fd, const char __user *, name);
-COMPAT_SYSCALL_WRAP1(exit_group, int, error_code);
COMPAT_SYSCALL_WRAP1(set_tid_address, int __user *, tidptr);
-COMPAT_SYSCALL_WRAP1(epoll_create, int, size);
COMPAT_SYSCALL_WRAP4(epoll_ctl, int, epfd, int, op, int, fd, struct epoll_event __user *, event);
COMPAT_SYSCALL_WRAP4(epoll_wait, int, epfd, struct epoll_event __user *, events, int, maxevents, int, timeout);
-COMPAT_SYSCALL_WRAP1(timer_getoverrun, timer_t, timer_id);
-COMPAT_SYSCALL_WRAP1(timer_delete, compat_timer_t, compat_timer_id);
COMPAT_SYSCALL_WRAP1(io_destroy, aio_context_t, ctx);
COMPAT_SYSCALL_WRAP3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, struct io_event __user *, result);
COMPAT_SYSCALL_WRAP1(mq_unlink, const char __user *, name);
COMPAT_SYSCALL_WRAP5(add_key, const char __user *, tp, const char __user *, dsc, const void __user *, pld, size_t, len, key_serial_t, id);
COMPAT_SYSCALL_WRAP4(request_key, const char __user *, tp, const char __user *, dsc, const char __user *, info, key_serial_t, id);
COMPAT_SYSCALL_WRAP5(remap_file_pages, unsigned long, start, unsigned long, size, unsigned long, prot, unsigned long, pgoff, unsigned long, flags);
-COMPAT_SYSCALL_WRAP3(ioprio_set, int, which, int, who, int, ioprio);
-COMPAT_SYSCALL_WRAP2(ioprio_get, int, which, int, who);
COMPAT_SYSCALL_WRAP3(inotify_add_watch, int, fd, const char __user *, path, u32, mask);
-COMPAT_SYSCALL_WRAP2(inotify_rm_watch, int, fd, __s32, wd);
COMPAT_SYSCALL_WRAP3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode);
COMPAT_SYSCALL_WRAP4(mknodat, int, dfd, const char __user *, filename, umode_t, mode, unsigned, dev);
COMPAT_SYSCALL_WRAP5(fchownat, int, dfd, const char __user *, filename, uid_t, user, gid_t, group, int, flag);
COMPAT_SYSCALL_WRAP6(splice, int, fd_in, loff_t __user *, off_in, int, fd_out, loff_t __user *, off_out, size_t, len, unsigned int, flags);
COMPAT_SYSCALL_WRAP4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags);
COMPAT_SYSCALL_WRAP3(getcpu, unsigned __user *, cpu, unsigned __user *, node, struct getcpu_cache __user *, cache);
-COMPAT_SYSCALL_WRAP1(eventfd, unsigned int, count);
-COMPAT_SYSCALL_WRAP2(timerfd_create, int, clockid, int, flags);
-COMPAT_SYSCALL_WRAP2(eventfd2, unsigned int, count, int, flags);
-COMPAT_SYSCALL_WRAP1(inotify_init1, int, flags);
COMPAT_SYSCALL_WRAP2(pipe2, int __user *, fildes, int, flags);
-COMPAT_SYSCALL_WRAP3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags);
-COMPAT_SYSCALL_WRAP1(epoll_create1, int, flags);
-COMPAT_SYSCALL_WRAP2(tkill, int, pid, int, sig);
-COMPAT_SYSCALL_WRAP3(tgkill, int, tgid, int, pid, int, sig);
COMPAT_SYSCALL_WRAP5(perf_event_open, struct perf_event_attr __user *, attr_uptr, pid_t, pid, int, cpu, int, group_fd, unsigned long, flags);
COMPAT_SYSCALL_WRAP5(clone, unsigned long, newsp, unsigned long, clone_flags, int __user *, parent_tidptr, int __user *, child_tidptr, unsigned long, tls);
-COMPAT_SYSCALL_WRAP2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags);
COMPAT_SYSCALL_WRAP4(prlimit64, pid_t, pid, unsigned int, resource, const struct rlimit64 __user *, new_rlim, struct rlimit64 __user *, old_rlim);
COMPAT_SYSCALL_WRAP5(name_to_handle_at, int, dfd, const char __user *, name, struct file_handle __user *, handle, int __user *, mnt_id, int, flag);
-COMPAT_SYSCALL_WRAP1(syncfs, int, fd);
-COMPAT_SYSCALL_WRAP2(setns, int, fd, int, nstype);
-COMPAT_SYSCALL_WRAP2(s390_runtime_instr, int, command, int, signum);
COMPAT_SYSCALL_WRAP5(kcmp, pid_t, pid1, pid_t, pid2, int, type, unsigned long, idx1, unsigned long, idx2);
COMPAT_SYSCALL_WRAP3(finit_module, int, fd, const char __user *, uargs, int, flags);
COMPAT_SYSCALL_WRAP3(sched_setattr, pid_t, pid, struct sched_attr __user *, attr, unsigned int, flags);
COMPAT_SYSCALL_WRAP3(bpf, int, cmd, union bpf_attr *, attr, unsigned int, size);
COMPAT_SYSCALL_WRAP3(s390_pci_mmio_write, const unsigned long, mmio_addr, const void __user *, user_buffer, const size_t, length);
COMPAT_SYSCALL_WRAP3(s390_pci_mmio_read, const unsigned long, mmio_addr, void __user *, user_buffer, const size_t, length);
+COMPAT_SYSCALL_WRAP4(socketpair, int, family, int, type, int, protocol, int __user *, usockvec);
+COMPAT_SYSCALL_WRAP3(bind, int, fd, struct sockaddr __user *, umyaddr, int, addrlen);
+COMPAT_SYSCALL_WRAP3(connect, int, fd, struct sockaddr __user *, uservaddr, int, addrlen);
+COMPAT_SYSCALL_WRAP4(accept4, int, fd, struct sockaddr __user *, upeer_sockaddr, int __user *, upeer_addrlen, int, flags);
+COMPAT_SYSCALL_WRAP3(getsockname, int, fd, struct sockaddr __user *, usockaddr, int __user *, usockaddr_len);
+COMPAT_SYSCALL_WRAP3(getpeername, int, fd, struct sockaddr __user *, usockaddr, int __user *, usockaddr_len);
+COMPAT_SYSCALL_WRAP6(sendto, int, fd, void __user *, buff, size_t, len, unsigned int, flags, struct sockaddr __user *, addr, int, addr_len);
stg %r3,__SF_EMPTY(%r15)
larl %r1,.Lpsw_idle_lpsw+4
stg %r1,__SF_EMPTY+8(%r15)
+#ifdef CONFIG_SMP
+ larl %r1,smp_cpu_mtid
+ llgf %r1,0(%r1)
+ ltgr %r1,%r1
+ jz .Lpsw_idle_stcctm
+ .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
+.Lpsw_idle_stcctm:
+#endif
STCK __CLOCK_IDLE_ENTER(%r2)
stpt __TIMER_IDLE_ENTER(%r2)
.Lpsw_idle_lpsw:
jhe 1f
mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2)
-1: # account system time going idle
+1: # calculate idle cycles
+#ifdef CONFIG_SMP
+ clg %r9,BASED(.Lcleanup_idle_insn)
+ jl 3f
+ larl %r1,smp_cpu_mtid
+ llgf %r1,0(%r1)
+ ltgr %r1,%r1
+ jz 3f
+ .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
+ larl %r3,mt_cycles
+ ag %r3,__LC_PERCPU_OFFSET
+ la %r4,__SF_EMPTY+16(%r15)
+2: lg %r0,0(%r3)
+ slg %r0,0(%r4)
+ alg %r0,64(%r4)
+ stg %r0,0(%r3)
+ la %r3,8(%r3)
+ la %r4,8(%r4)
+ brct %r1,2b
+#endif
+3: # account system time going idle
lg %r9,__LC_STEAL_TIMER
alg %r9,__CLOCK_IDLE_ENTER(%r2)
slg %r9,__LC_LAST_UPDATE_CLOCK
clg %r9,BASED(.Lcleanup_save_fpu_fpc_end)
jhe 1f
lg %r2,__LC_CURRENT
+ aghi %r2,__TASK_thread
0: # Store floating-point controls
stfpc __THREAD_FPU_fpc(%r2)
1: # Load register save area and check if VX is active
clg %r9,BASED(.Lcleanup_load_fpu_regs_vx_ctl)
jhe 6f
lg %r4,__LC_CURRENT
+ aghi %r4,__TASK_thread
lfpc __THREAD_FPU_fpc(%r4)
tm __THREAD_FPU_flags+3(%r4),FPU_USE_VX # VX-enabled task ?
lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area
cpuhw = &get_cpu_var(cpu_hw_events);
- /* check authorization for cpu counter sets */
+ /* Check authorization for cpu counter sets.
+ * If the particular CPU counter set is not authorized,
+ * return with -ENOENT in order to fall back to other
+ * PMUs that might suffice the event request.
+ */
ctrs_state = cpumf_state_ctl[hwc->config_base];
if (!(ctrs_state & cpuhw->info.auth_ctl))
- err = -EPERM;
+ err = -ENOENT;
put_cpu_var(cpu_hw_events);
return err;
*/
if (!(cpuhw->flags & PERF_EVENT_TXN))
if (validate_ctr_auth(&event->hw))
- return -EPERM;
+ return -ENOENT;
ctr_set_enable(&cpuhw->state, event->hw.config_base);
event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
state = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
state >>= CPUMF_LCCTL_ENABLE_SHIFT;
if ((state & cpuhw->info.auth_ctl) != state)
- return -EPERM;
+ return -ENOENT;
cpuhw->flags &= ~PERF_EVENT_TXN;
perf_pmu_enable(pmu);
aghi %r15,-STACK_FRAME_OVERHEAD
stg %r1,__SF_BACKCHAIN(%r15)
+ /* Store FPU registers */
+ brasl %r14,save_fpu_regs
+
/* Deactivate DAT */
stnsm __SF_EMPTY(%r15),0xfb
/* Store registers */
mvc 0x318(4,%r1),__SF_EMPTY(%r15) /* move prefix to lowcore */
- stfpc 0x31c(%r1) /* store fpu control */
- std 0,0x200(%r1) /* store f0 */
- std 1,0x208(%r1) /* store f1 */
- std 2,0x210(%r1) /* store f2 */
- std 3,0x218(%r1) /* store f3 */
- std 4,0x220(%r1) /* store f4 */
- std 5,0x228(%r1) /* store f5 */
- std 6,0x230(%r1) /* store f6 */
- std 7,0x238(%r1) /* store f7 */
- std 8,0x240(%r1) /* store f8 */
- std 9,0x248(%r1) /* store f9 */
- std 10,0x250(%r1) /* store f10 */
- std 11,0x258(%r1) /* store f11 */
- std 12,0x260(%r1) /* store f12 */
- std 13,0x268(%r1) /* store f13 */
- std 14,0x270(%r1) /* store f14 */
- std 15,0x278(%r1) /* store f15 */
stam %a0,%a15,0x340(%r1) /* store access registers */
stctg %c0,%c15,0x380(%r1) /* store control registers */
stmg %r0,%r15,0x280(%r1) /* store general registers */
lctlg %c0,%c15,0x380(%r13) /* load control registers */
lam %a0,%a15,0x340(%r13) /* load access registers */
- lfpc 0x31c(%r13) /* load fpu control */
- ld 0,0x200(%r13) /* load f0 */
- ld 1,0x208(%r13) /* load f1 */
- ld 2,0x210(%r13) /* load f2 */
- ld 3,0x218(%r13) /* load f3 */
- ld 4,0x220(%r13) /* load f4 */
- ld 5,0x228(%r13) /* load f5 */
- ld 6,0x230(%r13) /* load f6 */
- ld 7,0x238(%r13) /* load f7 */
- ld 8,0x240(%r13) /* load f8 */
- ld 9,0x248(%r13) /* load f9 */
- ld 10,0x250(%r13) /* load f10 */
- ld 11,0x258(%r13) /* load f11 */
- ld 12,0x260(%r13) /* load f12 */
- ld 13,0x268(%r13) /* load f13 */
- ld 14,0x270(%r13) /* load f14 */
- ld 15,0x278(%r13) /* load f15 */
-
/* Load old stack */
lg %r15,0x2f8(%r13)
#define NI_SYSCALL SYSCALL(sys_ni_syscall,sys_ni_syscall)
NI_SYSCALL /* 0 */
-SYSCALL(sys_exit,compat_sys_exit)
+SYSCALL(sys_exit,sys_exit)
SYSCALL(sys_fork,sys_fork)
SYSCALL(sys_read,compat_sys_s390_read)
SYSCALL(sys_write,compat_sys_s390_write)
SYSCALL(sys_open,compat_sys_open) /* 5 */
-SYSCALL(sys_close,compat_sys_close)
+SYSCALL(sys_close,sys_close)
SYSCALL(sys_restart_syscall,sys_restart_syscall)
SYSCALL(sys_creat,compat_sys_creat)
SYSCALL(sys_link,compat_sys_link)
SYSCALL(sys_ni_syscall,compat_sys_s390_getuid16) /* old getuid16 syscall*/
SYSCALL(sys_ni_syscall,compat_sys_stime) /* 25 old stime syscall */
SYSCALL(sys_ptrace,compat_sys_ptrace)
-SYSCALL(sys_alarm,compat_sys_alarm)
+SYSCALL(sys_alarm,sys_alarm)
NI_SYSCALL /* old fstat syscall */
SYSCALL(sys_pause,sys_pause)
SYSCALL(sys_utime,compat_sys_utime) /* 30 */
NI_SYSCALL /* old stty syscall */
NI_SYSCALL /* old gtty syscall */
SYSCALL(sys_access,compat_sys_access)
-SYSCALL(sys_nice,compat_sys_nice)
+SYSCALL(sys_nice,sys_nice)
NI_SYSCALL /* 35 old ftime syscall */
SYSCALL(sys_sync,sys_sync)
-SYSCALL(sys_kill,compat_sys_kill)
+SYSCALL(sys_kill,sys_kill)
SYSCALL(sys_rename,compat_sys_rename)
SYSCALL(sys_mkdir,compat_sys_mkdir)
SYSCALL(sys_rmdir,compat_sys_rmdir) /* 40 */
-SYSCALL(sys_dup,compat_sys_dup)
+SYSCALL(sys_dup,sys_dup)
SYSCALL(sys_pipe,compat_sys_pipe)
SYSCALL(sys_times,compat_sys_times)
NI_SYSCALL /* old prof syscall */
SYSCALL(sys_ioctl,compat_sys_ioctl)
SYSCALL(sys_fcntl,compat_sys_fcntl) /* 55 */
NI_SYSCALL /* intel mpx syscall */
-SYSCALL(sys_setpgid,compat_sys_setpgid)
+SYSCALL(sys_setpgid,sys_setpgid)
NI_SYSCALL /* old ulimit syscall */
NI_SYSCALL /* old uname syscall */
-SYSCALL(sys_umask,compat_sys_umask) /* 60 */
+SYSCALL(sys_umask,sys_umask) /* 60 */
SYSCALL(sys_chroot,compat_sys_chroot)
SYSCALL(sys_ustat,compat_sys_ustat)
-SYSCALL(sys_dup2,compat_sys_dup2)
+SYSCALL(sys_dup2,sys_dup2)
SYSCALL(sys_getppid,sys_getppid)
SYSCALL(sys_getpgrp,sys_getpgrp) /* 65 */
SYSCALL(sys_setsid,sys_setsid)
SYSCALL(sys_munmap,compat_sys_munmap)
SYSCALL(sys_truncate,compat_sys_truncate)
SYSCALL(sys_ftruncate,compat_sys_ftruncate)
-SYSCALL(sys_fchmod,compat_sys_fchmod)
+SYSCALL(sys_fchmod,sys_fchmod)
SYSCALL(sys_ni_syscall,compat_sys_s390_fchown16) /* 95 old fchown16 syscall*/
-SYSCALL(sys_getpriority,compat_sys_getpriority)
-SYSCALL(sys_setpriority,compat_sys_setpriority)
+SYSCALL(sys_getpriority,sys_getpriority)
+SYSCALL(sys_setpriority,sys_setpriority)
NI_SYSCALL /* old profil syscall */
SYSCALL(sys_statfs,compat_sys_statfs)
SYSCALL(sys_fstatfs,compat_sys_fstatfs) /* 100 */
SYSCALL(sys_swapoff,compat_sys_swapoff) /* 115 */
SYSCALL(sys_sysinfo,compat_sys_sysinfo)
SYSCALL(sys_s390_ipc,compat_sys_s390_ipc)
-SYSCALL(sys_fsync,compat_sys_fsync)
+SYSCALL(sys_fsync,sys_fsync)
SYSCALL(sys_sigreturn,compat_sys_sigreturn)
SYSCALL(sys_clone,compat_sys_clone) /* 120 */
SYSCALL(sys_setdomainname,compat_sys_setdomainname)
SYSCALL(sys_delete_module,compat_sys_delete_module)
NI_SYSCALL /* 130: old get_kernel_syms */
SYSCALL(sys_quotactl,compat_sys_quotactl)
-SYSCALL(sys_getpgid,compat_sys_getpgid)
-SYSCALL(sys_fchdir,compat_sys_fchdir)
+SYSCALL(sys_getpgid,sys_getpgid)
+SYSCALL(sys_fchdir,sys_fchdir)
SYSCALL(sys_bdflush,compat_sys_bdflush)
SYSCALL(sys_sysfs,compat_sys_sysfs) /* 135 */
-SYSCALL(sys_s390_personality,compat_sys_s390_personality)
+SYSCALL(sys_s390_personality,sys_s390_personality)
NI_SYSCALL /* for afs_syscall */
SYSCALL(sys_ni_syscall,compat_sys_s390_setfsuid16) /* old setfsuid16 syscall */
SYSCALL(sys_ni_syscall,compat_sys_s390_setfsgid16) /* old setfsgid16 syscall */
SYSCALL(sys_llseek,compat_sys_llseek) /* 140 */
SYSCALL(sys_getdents,compat_sys_getdents)
SYSCALL(sys_select,compat_sys_select)
-SYSCALL(sys_flock,compat_sys_flock)
+SYSCALL(sys_flock,sys_flock)
SYSCALL(sys_msync,compat_sys_msync)
SYSCALL(sys_readv,compat_sys_readv) /* 145 */
SYSCALL(sys_writev,compat_sys_writev)
-SYSCALL(sys_getsid,compat_sys_getsid)
-SYSCALL(sys_fdatasync,compat_sys_fdatasync)
+SYSCALL(sys_getsid,sys_getsid)
+SYSCALL(sys_fdatasync,sys_fdatasync)
SYSCALL(sys_sysctl,compat_sys_sysctl)
SYSCALL(sys_mlock,compat_sys_mlock) /* 150 */
SYSCALL(sys_munlock,compat_sys_munlock)
-SYSCALL(sys_mlockall,compat_sys_mlockall)
+SYSCALL(sys_mlockall,sys_mlockall)
SYSCALL(sys_munlockall,sys_munlockall)
SYSCALL(sys_sched_setparam,compat_sys_sched_setparam)
SYSCALL(sys_sched_getparam,compat_sys_sched_getparam) /* 155 */
SYSCALL(sys_sched_setscheduler,compat_sys_sched_setscheduler)
-SYSCALL(sys_sched_getscheduler,compat_sys_sched_getscheduler)
+SYSCALL(sys_sched_getscheduler,sys_sched_getscheduler)
SYSCALL(sys_sched_yield,sys_sched_yield)
-SYSCALL(sys_sched_get_priority_max,compat_sys_sched_get_priority_max)
-SYSCALL(sys_sched_get_priority_min,compat_sys_sched_get_priority_min) /* 160 */
+SYSCALL(sys_sched_get_priority_max,sys_sched_get_priority_max)
+SYSCALL(sys_sched_get_priority_min,sys_sched_get_priority_min) /* 160 */
SYSCALL(sys_sched_rr_get_interval,compat_sys_sched_rr_get_interval)
SYSCALL(sys_nanosleep,compat_sys_nanosleep)
SYSCALL(sys_mremap,compat_sys_mremap)
SYSCALL(sys_getgid,sys_getgid) /* 200 */
SYSCALL(sys_geteuid,sys_geteuid)
SYSCALL(sys_getegid,sys_getegid)
-SYSCALL(sys_setreuid,compat_sys_setreuid)
-SYSCALL(sys_setregid,compat_sys_setregid)
+SYSCALL(sys_setreuid,sys_setreuid)
+SYSCALL(sys_setregid,sys_setregid)
SYSCALL(sys_getgroups,compat_sys_getgroups) /* 205 */
SYSCALL(sys_setgroups,compat_sys_setgroups)
-SYSCALL(sys_fchown,compat_sys_fchown)
-SYSCALL(sys_setresuid,compat_sys_setresuid)
+SYSCALL(sys_fchown,sys_fchown)
+SYSCALL(sys_setresuid,sys_setresuid)
SYSCALL(sys_getresuid,compat_sys_getresuid)
-SYSCALL(sys_setresgid,compat_sys_setresgid) /* 210 */
+SYSCALL(sys_setresgid,sys_setresgid) /* 210 */
SYSCALL(sys_getresgid,compat_sys_getresgid)
SYSCALL(sys_chown,compat_sys_chown)
-SYSCALL(sys_setuid,compat_sys_setuid)
-SYSCALL(sys_setgid,compat_sys_setgid)
-SYSCALL(sys_setfsuid,compat_sys_setfsuid) /* 215 */
-SYSCALL(sys_setfsgid,compat_sys_setfsgid)
+SYSCALL(sys_setuid,sys_setuid)
+SYSCALL(sys_setgid,sys_setgid)
+SYSCALL(sys_setfsuid,sys_setfsuid) /* 215 */
+SYSCALL(sys_setfsgid,sys_setfsgid)
SYSCALL(sys_pivot_root,compat_sys_pivot_root)
SYSCALL(sys_mincore,compat_sys_mincore)
SYSCALL(sys_madvise,compat_sys_madvise)
SYSCALL(sys_lremovexattr,compat_sys_lremovexattr)
SYSCALL(sys_fremovexattr,compat_sys_fremovexattr) /* 235 */
SYSCALL(sys_gettid,sys_gettid)
-SYSCALL(sys_tkill,compat_sys_tkill)
+SYSCALL(sys_tkill,sys_tkill)
SYSCALL(sys_futex,compat_sys_futex)
SYSCALL(sys_sched_setaffinity,compat_sys_sched_setaffinity)
SYSCALL(sys_sched_getaffinity,compat_sys_sched_getaffinity) /* 240 */
-SYSCALL(sys_tgkill,compat_sys_tgkill)
+SYSCALL(sys_tgkill,sys_tgkill)
NI_SYSCALL /* reserved for TUX */
SYSCALL(sys_io_setup,compat_sys_io_setup)
SYSCALL(sys_io_destroy,compat_sys_io_destroy)
SYSCALL(sys_io_getevents,compat_sys_io_getevents) /* 245 */
SYSCALL(sys_io_submit,compat_sys_io_submit)
SYSCALL(sys_io_cancel,compat_sys_io_cancel)
-SYSCALL(sys_exit_group,compat_sys_exit_group)
-SYSCALL(sys_epoll_create,compat_sys_epoll_create)
+SYSCALL(sys_exit_group,sys_exit_group)
+SYSCALL(sys_epoll_create,sys_epoll_create)
SYSCALL(sys_epoll_ctl,compat_sys_epoll_ctl) /* 250 */
SYSCALL(sys_epoll_wait,compat_sys_epoll_wait)
SYSCALL(sys_set_tid_address,compat_sys_set_tid_address)
SYSCALL(sys_timer_create,compat_sys_timer_create)
SYSCALL(sys_timer_settime,compat_sys_timer_settime) /* 255 */
SYSCALL(sys_timer_gettime,compat_sys_timer_gettime)
-SYSCALL(sys_timer_getoverrun,compat_sys_timer_getoverrun)
-SYSCALL(sys_timer_delete,compat_sys_timer_delete)
+SYSCALL(sys_timer_getoverrun,sys_timer_getoverrun)
+SYSCALL(sys_timer_delete,sys_timer_delete)
SYSCALL(sys_clock_settime,compat_sys_clock_settime)
SYSCALL(sys_clock_gettime,compat_sys_clock_gettime) /* 260 */
SYSCALL(sys_clock_getres,compat_sys_clock_getres)
SYSCALL(sys_request_key,compat_sys_request_key)
SYSCALL(sys_keyctl,compat_sys_keyctl) /* 280 */
SYSCALL(sys_waitid,compat_sys_waitid)
-SYSCALL(sys_ioprio_set,compat_sys_ioprio_set)
-SYSCALL(sys_ioprio_get,compat_sys_ioprio_get)
+SYSCALL(sys_ioprio_set,sys_ioprio_set)
+SYSCALL(sys_ioprio_get,sys_ioprio_get)
SYSCALL(sys_inotify_init,sys_inotify_init)
SYSCALL(sys_inotify_add_watch,compat_sys_inotify_add_watch) /* 285 */
-SYSCALL(sys_inotify_rm_watch,compat_sys_inotify_rm_watch)
+SYSCALL(sys_inotify_rm_watch,sys_inotify_rm_watch)
SYSCALL(sys_migrate_pages,compat_sys_migrate_pages)
SYSCALL(sys_openat,compat_sys_openat)
SYSCALL(sys_mkdirat,compat_sys_mkdirat)
SYSCALL(sys_utimensat,compat_sys_utimensat) /* 315 */
SYSCALL(sys_signalfd,compat_sys_signalfd)
NI_SYSCALL /* 317 old sys_timer_fd */
-SYSCALL(sys_eventfd,compat_sys_eventfd)
-SYSCALL(sys_timerfd_create,compat_sys_timerfd_create)
+SYSCALL(sys_eventfd,sys_eventfd)
+SYSCALL(sys_timerfd_create,sys_timerfd_create)
SYSCALL(sys_timerfd_settime,compat_sys_timerfd_settime) /* 320 */
SYSCALL(sys_timerfd_gettime,compat_sys_timerfd_gettime)
SYSCALL(sys_signalfd4,compat_sys_signalfd4)
-SYSCALL(sys_eventfd2,compat_sys_eventfd2)
-SYSCALL(sys_inotify_init1,compat_sys_inotify_init1)
+SYSCALL(sys_eventfd2,sys_eventfd2)
+SYSCALL(sys_inotify_init1,sys_inotify_init1)
SYSCALL(sys_pipe2,compat_sys_pipe2) /* 325 */
-SYSCALL(sys_dup3,compat_sys_dup3)
-SYSCALL(sys_epoll_create1,compat_sys_epoll_create1)
+SYSCALL(sys_dup3,sys_dup3)
+SYSCALL(sys_epoll_create1,sys_epoll_create1)
SYSCALL(sys_preadv,compat_sys_preadv)
SYSCALL(sys_pwritev,compat_sys_pwritev)
SYSCALL(sys_rt_tgsigqueueinfo,compat_sys_rt_tgsigqueueinfo) /* 330 */
SYSCALL(sys_perf_event_open,compat_sys_perf_event_open)
-SYSCALL(sys_fanotify_init,compat_sys_fanotify_init)
+SYSCALL(sys_fanotify_init,sys_fanotify_init)
SYSCALL(sys_fanotify_mark,compat_sys_fanotify_mark)
SYSCALL(sys_prlimit64,compat_sys_prlimit64)
SYSCALL(sys_name_to_handle_at,compat_sys_name_to_handle_at) /* 335 */
SYSCALL(sys_open_by_handle_at,compat_sys_open_by_handle_at)
SYSCALL(sys_clock_adjtime,compat_sys_clock_adjtime)
-SYSCALL(sys_syncfs,compat_sys_syncfs)
-SYSCALL(sys_setns,compat_sys_setns)
+SYSCALL(sys_syncfs,sys_syncfs)
+SYSCALL(sys_setns,sys_setns)
SYSCALL(sys_process_vm_readv,compat_sys_process_vm_readv) /* 340 */
SYSCALL(sys_process_vm_writev,compat_sys_process_vm_writev)
-SYSCALL(sys_s390_runtime_instr,compat_sys_s390_runtime_instr)
+SYSCALL(sys_s390_runtime_instr,sys_s390_runtime_instr)
SYSCALL(sys_kcmp,compat_sys_kcmp)
SYSCALL(sys_finit_module,compat_sys_finit_module)
SYSCALL(sys_sched_setattr,compat_sys_sched_setattr) /* 345 */
SYSCALL(sys_s390_pci_mmio_write,compat_sys_s390_pci_mmio_write)
SYSCALL(sys_s390_pci_mmio_read,compat_sys_s390_pci_mmio_read)
SYSCALL(sys_execveat,compat_sys_execveat)
+SYSCALL(sys_userfaultfd,sys_userfaultfd) /* 355 */
+SYSCALL(sys_membarrier,sys_membarrier)
+SYSCALL(sys_recvmmsg,compat_sys_recvmmsg)
+SYSCALL(sys_sendmmsg,compat_sys_sendmmsg)
+SYSCALL(sys_socket,sys_socket)
+SYSCALL(sys_socketpair,compat_sys_socketpair) /* 360 */
+SYSCALL(sys_bind,sys_bind)
+SYSCALL(sys_connect,sys_connect)
+SYSCALL(sys_listen,sys_listen)
+SYSCALL(sys_accept4,sys_accept4)
+SYSCALL(sys_getsockopt,compat_sys_getsockopt) /* 365 */
+SYSCALL(sys_setsockopt,compat_sys_setsockopt)
+SYSCALL(sys_getsockname,compat_sys_getsockname)
+SYSCALL(sys_getpeername,compat_sys_getpeername)
+SYSCALL(sys_sendto,compat_sys_sendto)
+SYSCALL(sys_sendmsg,compat_sys_sendmsg) /* 370 */
+SYSCALL(sys_recvfrom,compat_sys_recvfrom)
+SYSCALL(sys_recvmsg,compat_sys_recvmsg)
+SYSCALL(sys_shutdown,sys_shutdown)
static atomic64_t virt_timer_current;
static atomic64_t virt_timer_elapsed;
-static DEFINE_PER_CPU(u64, mt_cycles[32]);
+DEFINE_PER_CPU(u64, mt_cycles[8]);
static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
return elapsed >= atomic64_read(&virt_timer_current);
}
+static void update_mt_scaling(void)
+{
+ u64 cycles_new[8], *cycles_old;
+ u64 delta, fac, mult, div;
+ int i;
+
+ stcctm5(smp_cpu_mtid + 1, cycles_new);
+ cycles_old = this_cpu_ptr(mt_cycles);
+ fac = 1;
+ mult = div = 0;
+ for (i = 0; i <= smp_cpu_mtid; i++) {
+ delta = cycles_new[i] - cycles_old[i];
+ div += delta;
+ mult *= i + 1;
+ mult += delta * fac;
+ fac *= i + 1;
+ }
+ div *= fac;
+ if (div > 0) {
+ /* Update scaling factor */
+ __this_cpu_write(mt_scaling_mult, mult);
+ __this_cpu_write(mt_scaling_div, div);
+ memcpy(cycles_old, cycles_new,
+ sizeof(u64) * (smp_cpu_mtid + 1));
+ }
+ __this_cpu_write(mt_scaling_jiffies, jiffies_64);
+}
+
/*
* Update process times based on virtual cpu times stored by entry.S
* to the lowcore fields user_timer, system_timer & steal_clock.
struct thread_info *ti = task_thread_info(tsk);
u64 timer, clock, user, system, steal;
u64 user_scaled, system_scaled;
- int i;
timer = S390_lowcore.last_update_timer;
clock = S390_lowcore.last_update_clock;
S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
- /* Do MT utilization calculation */
+ /* Update MT utilization calculation */
if (smp_cpu_mtid &&
- time_after64(jiffies_64, __this_cpu_read(mt_scaling_jiffies))) {
- u64 cycles_new[32], *cycles_old;
- u64 delta, mult, div;
-
- cycles_old = this_cpu_ptr(mt_cycles);
- if (stcctm5(smp_cpu_mtid + 1, cycles_new) < 2) {
- mult = div = 0;
- for (i = 0; i <= smp_cpu_mtid; i++) {
- delta = cycles_new[i] - cycles_old[i];
- mult += delta;
- div += (i + 1) * delta;
- }
- if (mult > 0) {
- /* Update scaling factor */
- __this_cpu_write(mt_scaling_mult, mult);
- __this_cpu_write(mt_scaling_div, div);
- memcpy(cycles_old, cycles_new,
- sizeof(u64) * (smp_cpu_mtid + 1));
- }
- }
- __this_cpu_write(mt_scaling_jiffies, jiffies_64);
- }
+ time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
+ update_mt_scaling();
user = S390_lowcore.user_timer - ti->user_timer;
S390_lowcore.steal_timer -= user;
S390_lowcore.last_update_timer = get_vtimer();
S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
+ /* Update MT utilization calculation */
+ if (smp_cpu_mtid &&
+ time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
+ update_mt_scaling();
+
system = S390_lowcore.system_timer - ti->system_timer;
S390_lowcore.steal_timer -= system;
ti->system_timer = S390_lowcore.system_timer;
{ "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
{ "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
+ { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
{ "instruction_lctl", VCPU_STAT(instruction_lctl) },
static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
{
- atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
+ atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
}
/*
cpumask_copy(&top->thread_mask, &core->mask);
cpumask_copy(&top->core_mask, &core_mc(core)->mask);
cpumask_copy(&top->book_mask, &core_book(core)->mask);
- cpumask_set_cpu(cpu, node_to_cpumask_map[core_node(core)->id]);
+ cpumask_set_cpu(cpu, &node_to_cpumask_map[core_node(core)->id]);
top->node_id = core_node(core)->id;
}
}
/* Clear all node masks */
for (i = 0; i < MAX_NUMNODES; i++)
- cpumask_clear(node_to_cpumask_map[i]);
+ cpumask_clear(&node_to_cpumask_map[i]);
/* Rebuild all masks */
toptree_for_each(core, numa, CORE)
pg_data_t *node_data[MAX_NUMNODES];
EXPORT_SYMBOL(node_data);
-cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+cpumask_t node_to_cpumask_map[MAX_NUMNODES];
EXPORT_SYMBOL(node_to_cpumask_map);
const struct numa_mode numa_mode_plain = {
static int __init numa_init_early(void)
{
/* Attach all possible CPUs to node 0 for now. */
- cpumask_copy(node_to_cpumask_map[0], cpu_possible_mask);
+ cpumask_copy(&node_to_cpumask_map[0], cpu_possible_mask);
return 0;
}
early_initcall(numa_init_early);
generic-y += trace_clock.h
generic-y += xor.h
generic-y += serial.h
+generic-y += word-at-a-time.h
static void __iomem *se7343_irq_regs;
struct irq_domain *se7343_irq_domain;
-static void se7343_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void se7343_irq_demux(struct irq_desc *desc)
{
struct irq_data *data = irq_desc_get_irq_data(desc);
struct irq_chip *chip = irq_data_get_irq_chip(data);
static void __iomem *se7722_irq_regs;
struct irq_domain *se7722_irq_domain;
-static void se7722_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void se7722_irq_demux(struct irq_desc *desc)
{
struct irq_data *data = irq_desc_get_irq_data(desc);
struct irq_chip *chip = irq_data_get_irq_chip(data);
.irq_unmask = enable_se7724_irq,
};
-static void se7724_irq_demux(unsigned int __irq, struct irq_desc *desc)
+static void se7724_irq_demux(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct fpga_irq set = get_fpga_irq(irq);
return virq;
}
-static void x3proto_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void x3proto_gpio_irq_handler(struct irq_desc *desc)
{
struct irq_data *data = irq_desc_get_irq_data(desc);
struct irq_chip *chip = irq_data_get_irq_chip(data);
.irq_unmask = hd64461_unmask_irq,
};
-static void hd64461_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void hd64461_irq_demux(struct irq_desc *desc)
{
unsigned short intv = __raw_readw(HD64461_NIRR);
unsigned int ext_irq = HD64461_IRQBASE;
}
/* Handle one or multiple IRQs from the extended interrupt controller */
-static void leon_handle_ext_irq(unsigned int irq, struct irq_desc *desc)
+static void leon_handle_ext_irq(struct irq_desc *desc)
{
unsigned int eirq;
struct irq_bucket *p;
};
/* Handle one or multiple IRQs from the PCI core */
-static void grpci1_pci_flow_irq(unsigned int irq, struct irq_desc *desc)
+static void grpci1_pci_flow_irq(struct irq_desc *desc)
{
struct grpci1_priv *priv = grpci1priv;
int i, ack = 0;
};
/* Handle one or multiple IRQs from the PCI core */
-static void grpci2_pci_flow_irq(unsigned int irq, struct irq_desc *desc)
+static void grpci2_pci_flow_irq(struct irq_desc *desc)
{
struct grpci2_priv *priv = grpci2priv;
int i, ack = 0;
#include <linux/errno.h>
#include <linux/io.h>
#include <linux/module.h>
+#include <linux/string.h>
#include <gxio/iorpc_globals.h>
#include <gxio/iorpc_mpipe.h>
/* HACK: Avoid pointless "shadow" warnings. */
#define link link_shadow
-/**
- * strscpy - Copy a C-string into a sized buffer, but only if it fits
- * @dest: Where to copy the string to
- * @src: Where to copy the string from
- * @size: size of destination buffer
- *
- * Use this routine to avoid copying too-long strings.
- * The routine returns the total number of bytes copied
- * (including the trailing NUL) or zero if the buffer wasn't
- * big enough. To ensure that programmers pay attention
- * to the return code, the destination has a single NUL
- * written at the front (if size is non-zero) when the
- * buffer is not big enough.
- */
-static size_t strscpy(char *dest, const char *src, size_t size)
-{
- size_t len = strnlen(src, size) + 1;
- if (len > size) {
- if (size)
- dest[0] = '\0';
- return 0;
- }
- memcpy(dest, src, len);
- return len;
-}
-
int gxio_mpipe_init(gxio_mpipe_context_t *context, unsigned int mpipe_index)
{
char file[32];
if (!context)
return GXIO_ERR_NO_DEVICE;
- if (strscpy(name.name, link_name, sizeof(name.name)) == 0)
+ if (strscpy(name.name, link_name, sizeof(name.name)) < 0)
return GXIO_ERR_NO_DEVICE;
return gxio_mpipe_info_instance_aux(context, name);
rv = gxio_mpipe_info_enumerate_aux(context, idx, &name, &mac);
if (rv >= 0) {
- if (strscpy(link_name, name.name, sizeof(name.name)) == 0)
+ if (strscpy(link_name, name.name, sizeof(name.name)) < 0)
return GXIO_ERR_INVAL_MEMORY_SIZE;
memcpy(link_mac, mac.mac, sizeof(mac.mac));
}
_gxio_mpipe_link_name_t name;
int rv;
- if (strscpy(name.name, link_name, sizeof(name.name)) == 0)
+ if (strscpy(name.name, link_name, sizeof(name.name)) < 0)
return GXIO_ERR_NO_DEVICE;
rv = gxio_mpipe_link_open_aux(context, name, flags);
struct word_at_a_time { /* unused */ };
#define WORD_AT_A_TIME_CONSTANTS {}
-/* Generate 0x01 byte values for non-zero bytes using a SIMD instruction. */
+/* Generate 0x01 byte values for zero bytes using a SIMD instruction. */
static inline unsigned long has_zero(unsigned long val, unsigned long *data,
const struct word_at_a_time *c)
{
#endif
}
+#ifdef __BIG_ENDIAN
+#define zero_bytemask(mask) (~1ul << (63 - __builtin_clzl(mask)))
+#else
+#define zero_bytemask(mask) ((2ul << __builtin_ctzl(mask)) - 1)
+#endif
+
#endif /* _ASM_WORD_AT_A_TIME_H */
* to Linux which just calls handle_level_irq() after clearing the
* MAC INTx Assert status bit associated with this interrupt.
*/
-static void trio_handle_level_irq(unsigned int __irq, struct irq_desc *desc)
+static void trio_handle_level_irq(struct irq_desc *desc)
{
struct pci_controller *controller = irq_desc_get_handler_data(desc);
gxio_trio_context_t *trio_context = controller->trio;
uint64_t intx = (uint64_t)irq_desc_get_chip_data(desc);
- unsigned int irq = irq_desc_get_irq(desc);
int mac = controller->mac;
unsigned int reg_offset;
uint64_t level_mask;
- handle_level_irq(irq, desc);
+ handle_level_irq(desc);
/*
* Clear the INTx Level status, otherwise future interrupts are
#include <linux/platform_device.h>
#include <linux/usb/tilegx.h>
#include <linux/init.h>
+#include <linux/module.h>
#include <linux/types.h>
static u64 ehci_dmamask = DMA_BIT_MASK(32);
generic-y += switch_to.h
generic-y += topology.h
generic-y += trace_clock.h
+generic-y += word-at-a-time.h
generic-y += xor.h
generic-y += unaligned.h
generic-y += user.h
generic-y += vga.h
+generic-y += word-at-a-time.h
generic-y += xor.h
* irq_controller_lock held, and IRQs disabled. Decode the IRQ
* and call the handler.
*/
-static void puv3_gpio_handler(unsigned int __irq, struct irq_desc *desc)
+static void puv3_gpio_handler(struct irq_desc *desc)
{
unsigned int mask, irq;
depends on X86_MCE_INTEL
config X86_LEGACY_VM86
- bool "Legacy VM86 support (obsolete)"
+ bool "Legacy VM86 support"
default n
depends on X86_32
---help---
available to accelerate real mode DOS programs. However, any
recent version of DOSEMU, X, or vbetool should be fully
functional even without kernel VM86 support, as they will all
- fall back to (pretty well performing) software emulation.
+ fall back to software emulation. Nevertheless, if you are using
+ a 16-bit DOS program where 16-bit performance matters, vm86
+ mode might be faster than emulation and you might want to
+ enable this option.
- Anything that works on a 64-bit kernel is unlikely to need
- this option, as 64-bit kernels don't, and can't, support V8086
- mode. This option is also unrelated to 16-bit protected mode
- and is not needed to run most 16-bit programs under Wine.
+ Note that any app that works on a 64-bit kernel is unlikely to
+ need this option, as 64-bit kernels don't, and can't, support
+ V8086 mode. This option is also unrelated to 16-bit protected
+ mode and is not needed to run most 16-bit programs under Wine.
- Enabling this option adds considerable attack surface to the
- kernel and slows down system calls and exception handling.
+ Enabling this option increases the complexity of the kernel
+ and slows down exception handling a tiny bit.
- Unless you use very old userspace or need the last drop of
- performance in your real mode DOS games and can't use KVM,
- say N here.
+ If unsure, say N here.
config VM86
bool
config X86_PAE
bool "PAE (Physical Address Extension) Support"
depends on X86_32 && !HIGHMEM4G
+ select SWIOTLB
---help---
PAE is required for NX support, and furthermore enables
larger swapspace support for non-overcommit purposes. It
/* Runs on exception stack */
ENTRY(nmi)
+ /*
+ * Fix up the exception frame if we're on Xen.
+ * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most
+ * one value to the stack on native, so it may clobber the rdx
+ * scratch slot, but it won't clobber any of the important
+ * slots past it.
+ *
+ * Xen is a different story, because the Xen frame itself overlaps
+ * the "NMI executing" variable.
+ */
PARAVIRT_ADJUST_EXCEPTION_FRAME
+
/*
* We allow breakpoints in NMIs. If a breakpoint occurs, then
* the iretq it performs will take us out of NMI context.
* we don't want to enable interrupts, because then we'll end
* up in an awkward situation in which IRQs are on but NMIs
* are off.
+ *
+ * We also must not push anything to the stack before switching
+ * stacks lest we corrupt the "NMI executing" variable.
*/
- SWAPGS
+ SWAPGS_UNSAFE_STACK
cld
movq %rsp, %rdx
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
#define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */
#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */
#define X86_FEATURE_HWP ( 7*32+ 10) /* "hwp" Intel HWP */
-#define X86_FEATURE_HWP_NOITFY ( 7*32+ 11) /* Intel HWP_NOTIFY */
+#define X86_FEATURE_HWP_NOTIFY ( 7*32+ 11) /* Intel HWP_NOTIFY */
#define X86_FEATURE_HWP_ACT_WINDOW ( 7*32+ 12) /* Intel HWP_ACT_WINDOW */
#define X86_FEATURE_HWP_EPP ( 7*32+13) /* Intel HWP_EPP */
#define X86_FEATURE_HWP_PKG_REQ ( 7*32+14) /* Intel HWP_PKG_REQ */
#define X86_FEATURE_AVX512PF ( 9*32+26) /* AVX-512 Prefetch */
#define X86_FEATURE_AVX512ER ( 9*32+27) /* AVX-512 Exponential and Reciprocal */
#define X86_FEATURE_AVX512CD ( 9*32+28) /* AVX-512 Conflict Detection */
+#define X86_FEATURE_SHA_NI ( 9*32+29) /* SHA1/SHA256 Instruction Extensions */
/* Extended state features, CPUID level 0x0000000d:1 (eax), word 10 */
#define X86_FEATURE_XSAVEOPT (10*32+ 0) /* XSAVEOPT */
extern void __iomem *__init efi_ioremap(unsigned long addr, unsigned long size,
u32 type, u64 attribute);
+#ifdef CONFIG_KASAN
+/*
+ * CONFIG_KASAN may redefine memset to __memset. __memset function is present
+ * only in kernel binary. Since the EFI stub linked into a separate binary it
+ * doesn't have __memset(). So we should use standard memset from
+ * arch/x86/boot/compressed/string.c. The same applies to memcpy and memmove.
+ */
+#undef memcpy
+#undef memset
+#undef memmove
+#endif
+
#endif /* CONFIG_X86_32 */
extern struct efi_scratch efi_scratch;
#define KVM_PIO_PAGE_OFFSET 1
#define KVM_COALESCED_MMIO_PAGE_OFFSET 2
+#define KVM_HALT_POLL_NS_DEFAULT 500000
#define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS
u32 nmi_window_exits;
u32 halt_exits;
u32 halt_successful_poll;
+ u32 halt_attempted_poll;
u32 halt_wakeup;
u32 request_irq_exits;
u32 irq_exits;
#define DEBUGCTLMSR_BTS_OFF_USR (1UL << 10)
#define DEBUGCTLMSR_FREEZE_LBRS_ON_PMI (1UL << 11)
+#define MSR_PEBS_FRONTEND 0x000003f7
+
#define MSR_IA32_POWER_CTL 0x000001fc
#define MSR_IA32_MC0_CTL 0x00000400
/* C1E active bits in int pending message */
#define K8_INTP_C1E_ACTIVE_MASK 0x18000000
#define MSR_K8_TSEG_ADDR 0xc0010112
+#define MSR_K8_TSEG_MASK 0xc0010113
#define K8_MTRRFIXRANGE_DRAM_ENABLE 0x00040000 /* MtrrFixDramEn bit */
#define K8_MTRRFIXRANGE_DRAM_MODIFY 0x00080000 /* MtrrFixDramModEn bit */
#define K8_MTRR_RDMEM_WRMEM_MASK 0x18181818 /* Mask: RdMem|WrMem */
struct pv_time_ops {
unsigned long long (*sched_clock)(void);
unsigned long long (*steal_clock)(int cpu);
- unsigned long (*get_tsc_khz)(void);
};
struct pv_cpu_ops {
#define PVCLOCK_TSC_STABLE_BIT (1 << 0)
#define PVCLOCK_GUEST_STOPPED (1 << 1)
+/* PVCLOCK_COUNTS_FROM_ZERO broke ABI and can't be used anymore. */
#define PVCLOCK_COUNTS_FROM_ZERO (1 << 2)
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_PVCLOCK_ABI_H */
}
#endif
-#define virt_queued_spin_lock virt_queued_spin_lock
-
-static inline bool virt_queued_spin_lock(struct qspinlock *lock)
+#ifdef CONFIG_PARAVIRT
+#define virt_spin_lock virt_spin_lock
+static inline bool virt_spin_lock(struct qspinlock *lock)
{
if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
return false;
- while (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) != 0)
- cpu_relax();
+ /*
+ * On hypervisors without PARAVIRT_SPINLOCKS support we fall
+ * back to a Test-and-Set spinlock, because fair locks have
+ * horrible lock 'holder' preemption issues.
+ */
+
+ do {
+ while (atomic_read(&lock->val) != 0)
+ cpu_relax();
+ } while (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) != 0);
return true;
}
+#endif /* CONFIG_PARAVIRT */
#include <asm-generic/qspinlock.h>
return _hypercall4(int, update_descriptor, ma, ma>>32, desc, desc>>32);
}
-static inline int
+static inline long
HYPERVISOR_memory_op(unsigned int cmd, void *arg)
{
- return _hypercall2(int, memory_op, cmd, arg);
+ return _hypercall2(long, memory_op, cmd, arg);
}
static inline int
#ifndef __ASM_X86_BITSPERLONG_H
#define __ASM_X86_BITSPERLONG_H
-#ifdef __x86_64__
+#if defined(__x86_64__) && !defined(__ILP32__)
# define __BITS_PER_LONG 64
#else
# define __BITS_PER_LONG 32
static void __init_or_module optimize_nops(struct alt_instr *a, u8 *instr)
{
+ unsigned long flags;
+
if (instr[0] != 0x90)
return;
+ local_irq_save(flags);
add_nops(instr + (a->instrlen - a->padlen), a->padlen);
+ sync_core();
+ local_irq_restore(flags);
DUMP_BYTES(instr, a->instrlen, "%p: [%d:%d) optimized NOPs: ",
instr, a->instrlen - a->padlen, a->padlen);
apic_write(APIC_LVTT, lvtt_value);
if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
+ /*
+ * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
+ * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
+ * According to Intel, MFENCE can do the serialization here.
+ */
+ asm volatile("mfence" : : : "memory");
+
printk_once(KERN_DEBUG "TSC deadline timer enabled\n");
return;
}
int pin, ioapic, irq, irq_entry;
const struct cpumask *mask;
struct irq_data *idata;
+ struct irq_chip *chip;
if (skip_ioapic_setup == 1)
return;
else
mask = apic->target_cpus();
- irq_set_affinity(irq, mask);
+ chip = irq_data_get_irq_chip(idata);
+ chip->irq_set_affinity(idata, mask, false);
}
-
}
#endif
err = assign_irq_vector(irq, data, dest);
if (err) {
- struct irq_data *top = irq_get_irq_data(irq);
-
if (assign_irq_vector(irq, data,
- irq_data_get_affinity_mask(top)))
+ irq_data_get_affinity_mask(irq_data)))
pr_err("Failed to recover vector for irq %d\n", irq);
return err;
}
else
printk(KERN_CONT "%d86", c->x86);
- printk(KERN_CONT " (fam: %02x, model: %02x", c->x86, c->x86_model);
+ printk(KERN_CONT " (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
if (c->x86_mask || c->cpuid_level >= 0)
- printk(KERN_CONT ", stepping: %02x)\n", c->x86_mask);
+ printk(KERN_CONT ", stepping: 0x%x)\n", c->x86_mask);
else
printk(KERN_CONT ")\n");
struct ms_hyperv_info ms_hyperv;
EXPORT_SYMBOL_GPL(ms_hyperv);
-static void (*hv_kexec_handler)(void);
-static void (*hv_crash_handler)(struct pt_regs *regs);
-
#if IS_ENABLED(CONFIG_HYPERV)
static void (*vmbus_handler)(void);
+static void (*hv_kexec_handler)(void);
+static void (*hv_crash_handler)(struct pt_regs *regs);
void hyperv_vector_handler(struct pt_regs *regs)
{
hv_crash_handler = NULL;
}
EXPORT_SYMBOL_GPL(hv_remove_crash_handler);
-#endif
+#ifdef CONFIG_KEXEC_CORE
static void hv_machine_shutdown(void)
{
if (kexec_in_progress && hv_kexec_handler)
hv_crash_handler(regs);
native_machine_crash_shutdown(regs);
}
-
+#endif /* CONFIG_KEXEC_CORE */
+#endif /* CONFIG_HYPERV */
static uint32_t __init ms_hyperv_platform(void)
{
no_timer_check = 1;
#endif
+#if IS_ENABLED(CONFIG_HYPERV) && defined(CONFIG_KEXEC_CORE)
machine_ops.shutdown = hv_machine_shutdown;
machine_ops.crash_shutdown = hv_machine_crash_shutdown;
+#endif
mark_tsc_unstable("running on Hyper-V");
}
EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
EXTRA_REG_LBR = 2, /* lbr_select */
EXTRA_REG_LDLAT = 3, /* ld_lat_threshold */
+ EXTRA_REG_FE = 4, /* fe_* */
EXTRA_REG_MAX /* number of entries needed */
};
INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
+ /*
+ * Note the low 8 bits eventsel code is not a continuous field, containing
+ * some #GPing bits. These are masked out.
+ */
+ INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE),
EVENT_EXTRA_END
};
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
INTEL_UEVENT_CONSTRAINT(0x148, 0x4), /* L1D_PEND_MISS.PENDING */
- INTEL_EVENT_CONSTRAINT(0xa3, 0x4), /* CYCLE_ACTIVITY.* */
+ INTEL_UEVENT_CONSTRAINT(0x8a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_MISS */
EVENT_CONSTRAINT_END
};
intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
struct perf_event *event)
{
- struct event_constraint *c1 = cpuc->event_constraint[idx];
+ struct event_constraint *c1 = NULL;
struct event_constraint *c2;
+ if (idx >= 0) /* fake does < 0 */
+ c1 = cpuc->event_constraint[idx];
+
/*
* first time only
* - static constraint: no change across incremental scheduling calls
PMU_FORMAT_ATTR(ldlat, "config1:0-15");
+PMU_FORMAT_ATTR(frontend, "config1:0-23");
+
static struct attribute *intel_arch3_formats_attr[] = {
&format_attr_event.attr,
&format_attr_umask.attr,
NULL,
};
+static struct attribute *skl_format_attr[] = {
+ &format_attr_frontend.attr,
+ NULL,
+};
+
static __initconst const struct x86_pmu core_pmu = {
.name = "core",
.handle_irq = x86_pmu_handle_irq,
x86_pmu.hw_config = hsw_hw_config;
x86_pmu.get_event_constraints = hsw_get_event_constraints;
- x86_pmu.cpu_events = hsw_events_attrs;
+ x86_pmu.format_attrs = merge_attr(intel_arch3_formats_attr,
+ skl_format_attr);
WARN_ON(!x86_pmu.format_attrs);
x86_pmu.cpu_events = hsw_events_attrs;
pr_cont("Skylake events, ");
if (!buf || bts_buffer_is_full(buf, bts))
return;
+ event->hw.itrace_started = 1;
event->hw.state = 0;
if (!buf->snapshot)
PERF_MSR_EVENT_MAX,
};
-bool test_aperfmperf(int idx)
+static bool test_aperfmperf(int idx)
{
return boot_cpu_has(X86_FEATURE_APERFMPERF);
}
-bool test_intel(int idx)
+static bool test_intel(int idx)
{
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
boot_cpu_data.x86 != 6)
{ X86_FEATURE_PLN, CR_EAX, 4, 0x00000006, 0 },
{ X86_FEATURE_PTS, CR_EAX, 6, 0x00000006, 0 },
{ X86_FEATURE_HWP, CR_EAX, 7, 0x00000006, 0 },
- { X86_FEATURE_HWP_NOITFY, CR_EAX, 8, 0x00000006, 0 },
+ { X86_FEATURE_HWP_NOTIFY, CR_EAX, 8, 0x00000006, 0 },
{ X86_FEATURE_HWP_ACT_WINDOW, CR_EAX, 9, 0x00000006, 0 },
{ X86_FEATURE_HWP_EPP, CR_EAX,10, 0x00000006, 0 },
{ X86_FEATURE_HWP_PKG_REQ, CR_EAX,11, 0x00000006, 0 },
}
#ifdef CONFIG_KEXEC_FILE
-static int get_nr_ram_ranges_callback(unsigned long start_pfn,
- unsigned long nr_pfn, void *arg)
+static int get_nr_ram_ranges_callback(u64 start, u64 end, void *arg)
{
- int *nr_ranges = arg;
+ unsigned int *nr_ranges = arg;
(*nr_ranges)++;
return 0;
ced->image = image;
- walk_system_ram_range(0, -1, &nr_ranges,
+ walk_system_ram_res(0, -1, &nr_ranges,
get_nr_ram_ranges_callback);
ced->max_nr_ranges = nr_ranges;
return (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
}
-static inline int
-execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
+static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
{
struct irq_stack *curstk, *irqstk;
- u32 *isp, *prev_esp, arg1, arg2;
+ u32 *isp, *prev_esp, arg1;
curstk = (struct irq_stack *) current_stack();
irqstk = __this_cpu_read(hardirq_stack);
asm volatile("xchgl %%ebx,%%esp \n"
"call *%%edi \n"
"movl %%ebx,%%esp \n"
- : "=a" (arg1), "=d" (arg2), "=b" (isp)
- : "0" (irq), "1" (desc), "2" (isp),
+ : "=a" (arg1), "=b" (isp)
+ : "0" (desc), "1" (isp),
"D" (desc->handle_irq)
: "memory", "cc", "ecx");
return 1;
bool handle_irq(struct irq_desc *desc, struct pt_regs *regs)
{
- unsigned int irq;
- int overflow;
-
- overflow = check_stack_overflow();
+ int overflow = check_stack_overflow();
if (IS_ERR_OR_NULL(desc))
return false;
- irq = irq_desc_get_irq(desc);
- if (user_mode(regs) || !execute_on_irq_stack(overflow, desc, irq)) {
+ if (user_mode(regs) || !execute_on_irq_stack(overflow, desc)) {
if (unlikely(overflow))
print_stack_overflow();
- generic_handle_irq_desc(irq, desc);
+ generic_handle_irq_desc(desc);
}
return true;
if (unlikely(IS_ERR_OR_NULL(desc)))
return false;
- generic_handle_irq_desc(irq_desc_get_irq(desc), desc);
+ generic_handle_irq_desc(desc);
return true;
}
if (alloc_size > PAGE_SIZE)
new_ldt->entries = vzalloc(alloc_size);
else
- new_ldt->entries = kzalloc(PAGE_SIZE, GFP_KERNEL);
+ new_ldt->entries = (void *)get_zeroed_page(GFP_KERNEL);
if (!new_ldt->entries) {
kfree(new_ldt);
if (ldt->size * LDT_ENTRY_SIZE > PAGE_SIZE)
vfree(ldt->entries);
else
- kfree(ldt->entries);
+ free_page((unsigned long)ldt->entries);
kfree(ldt);
}
#include <asm/timer.h>
#include <asm/special_insns.h>
-/* nop stub */
-void _paravirt_nop(void)
-{
-}
+/*
+ * nop stub, which must not clobber anything *including the stack* to
+ * avoid confusing the entry prologues.
+ */
+extern void _paravirt_nop(void);
+asm (".pushsection .entry.text, \"ax\"\n"
+ ".global _paravirt_nop\n"
+ "_paravirt_nop:\n\t"
+ "ret\n\t"
+ ".size _paravirt_nop, . - _paravirt_nop\n\t"
+ ".type _paravirt_nop, @function\n\t"
+ ".popsection");
/* identity function, which can be inlined */
u32 _paravirt_ident_32(u32 x)
bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)
{
- *gfp = dma_alloc_coherent_gfp_flags(*dev, *gfp);
*gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
+ *gfp = dma_alloc_coherent_gfp_flags(*dev, *gfp);
if (!*dev)
*dev = &x86_dma_fallback_dev;
return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
}
+/*
+ * Called from fs/proc with a reference on @p to find the function
+ * which called into schedule(). This needs to be done carefully
+ * because the task might wake up and we might look at a stack
+ * changing under us.
+ */
+unsigned long get_wchan(struct task_struct *p)
+{
+ unsigned long start, bottom, top, sp, fp, ip;
+ int count = 0;
+
+ if (!p || p == current || p->state == TASK_RUNNING)
+ return 0;
+
+ start = (unsigned long)task_stack_page(p);
+ if (!start)
+ return 0;
+
+ /*
+ * Layout of the stack page:
+ *
+ * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
+ * PADDING
+ * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
+ * stack
+ * ----------- bottom = start + sizeof(thread_info)
+ * thread_info
+ * ----------- start
+ *
+ * The tasks stack pointer points at the location where the
+ * framepointer is stored. The data on the stack is:
+ * ... IP FP ... IP FP
+ *
+ * We need to read FP and IP, so we need to adjust the upper
+ * bound by another unsigned long.
+ */
+ top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
+ top -= 2 * sizeof(unsigned long);
+ bottom = start + sizeof(struct thread_info);
+
+ sp = READ_ONCE(p->thread.sp);
+ if (sp < bottom || sp > top)
+ return 0;
+
+ fp = READ_ONCE(*(unsigned long *)sp);
+ do {
+ if (fp < bottom || fp > top)
+ return 0;
+ ip = READ_ONCE(*(unsigned long *)(fp + sizeof(unsigned long)));
+ if (!in_sched_functions(ip))
+ return ip;
+ fp = READ_ONCE(*(unsigned long *)fp);
+ } while (count++ < 16 && p->state != TASK_RUNNING);
+ return 0;
+}
return prev_p;
}
-
-#define top_esp (THREAD_SIZE - sizeof(unsigned long))
-#define top_ebp (THREAD_SIZE - 2*sizeof(unsigned long))
-
-unsigned long get_wchan(struct task_struct *p)
-{
- unsigned long bp, sp, ip;
- unsigned long stack_page;
- int count = 0;
- if (!p || p == current || p->state == TASK_RUNNING)
- return 0;
- stack_page = (unsigned long)task_stack_page(p);
- sp = p->thread.sp;
- if (!stack_page || sp < stack_page || sp > top_esp+stack_page)
- return 0;
- /* include/asm-i386/system.h:switch_to() pushes bp last. */
- bp = *(unsigned long *) sp;
- do {
- if (bp < stack_page || bp > top_ebp+stack_page)
- return 0;
- ip = *(unsigned long *) (bp+4);
- if (!in_sched_functions(ip))
- return ip;
- bp = *(unsigned long *) bp;
- } while (count++ < 16);
- return 0;
-}
-
}
EXPORT_SYMBOL_GPL(set_personality_ia32);
-unsigned long get_wchan(struct task_struct *p)
-{
- unsigned long stack;
- u64 fp, ip;
- int count = 0;
-
- if (!p || p == current || p->state == TASK_RUNNING)
- return 0;
- stack = (unsigned long)task_stack_page(p);
- if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
- return 0;
- fp = *(u64 *)(p->thread.sp);
- do {
- if (fp < (unsigned long)stack ||
- fp >= (unsigned long)stack+THREAD_SIZE)
- return 0;
- ip = *(u64 *)(fp+8);
- if (!in_sched_functions(ip))
- return ip;
- fp = *(u64 *)fp;
- } while (count++ < 16);
- return 0;
-}
-
long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
{
int ret = 0;
#include <asm/hypervisor.h>
#include <asm/nmi.h>
#include <asm/x86_init.h>
+#include <asm/geode.h>
unsigned int __read_mostly cpu_khz; /* TSC clocks / usec, not used here */
EXPORT_SYMBOL(cpu_khz);
static void __init check_system_tsc_reliable(void)
{
-#ifdef CONFIG_MGEODE_LX
- /* RTSC counts during suspend */
+#if defined(CONFIG_MGEODEGX1) || defined(CONFIG_MGEODE_LX) || defined(CONFIG_X86_GENERIC)
+ if (is_geode_lx()) {
+ /* RTSC counts during suspend */
#define RTSC_SUSP 0x100
- unsigned long res_low, res_high;
+ unsigned long res_low, res_high;
- rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
- /* Geode_LX - the OLPC CPU has a very reliable TSC */
- if (res_low & RTSC_SUSP)
- tsc_clocksource_reliable = 1;
+ rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
+ /* Geode_LX - the OLPC CPU has a very reliable TSC */
+ if (res_low & RTSC_SUSP)
+ tsc_clocksource_reliable = 1;
+ }
#endif
if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
tsc_clocksource_reliable = 1;
#include <linux/audit.h>
#include <linux/stddef.h>
#include <linux/slab.h>
+#include <linux/security.h>
#include <asm/uaccess.h>
#include <asm/io.h>
struct pt_regs *regs = current_pt_regs();
unsigned long err = 0;
+ err = security_mmap_addr(0);
+ if (err) {
+ /*
+ * vm86 cannot virtualize the address space, so vm86 users
+ * need to manage the low 1MB themselves using mmap. Given
+ * that BIOS places important data in the first page, vm86
+ * is essentially useless if mmap_min_addr != 0. DOSEMU,
+ * for example, won't even bother trying to use vm86 if it
+ * can't map a page at virtual address 0.
+ *
+ * To reduce the available kernel attack surface, simply
+ * disallow vm86(old) for users who cannot mmap at va 0.
+ *
+ * The implementation of security_mmap_addr will allow
+ * suitably privileged users to map va 0 even if
+ * vm.mmap_min_addr is set above 0, and we want this
+ * behavior for vm86 as well, as it ensures that legacy
+ * tools like vbetool will not fail just because of
+ * vm.mmap_min_addr.
+ */
+ pr_info_once("Denied a call to vm86(old) from %s[%d] (uid: %d). Set the vm.mmap_min_addr sysctl to 0 and/or adjust LSM mmap_min_addr policy to enable vm86 if you are using a vm86-based DOS emulator.\n",
+ current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()));
+ return -EPERM;
+ }
+
if (!vm86) {
if (!(vm86 = kzalloc(sizeof(*vm86), GFP_KERNEL)))
return -ENOMEM;
break;
reserved |= is_shadow_zero_bits_set(&vcpu->arch.mmu, spte,
- leaf);
+ iterator.level);
}
walk_shadow_page_lockless_end(vcpu);
__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
struct rsvd_bits_validate *rsvd_check,
int maxphyaddr, int level, bool nx, bool gbpages,
- bool pse)
+ bool pse, bool amd)
{
u64 exb_bit_rsvd = 0;
u64 gbpages_bit_rsvd = 0;
* Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
* leaf entries) on AMD CPUs only.
*/
- if (guest_cpuid_is_amd(vcpu))
+ if (amd)
nonleaf_bit8_rsvd = rsvd_bits(8, 8);
switch (level) {
__reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
cpuid_maxphyaddr(vcpu), context->root_level,
context->nx, guest_cpuid_has_gbpages(vcpu),
- is_pse(vcpu));
+ is_pse(vcpu), guest_cpuid_is_amd(vcpu));
}
static void
void
reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
{
+ /*
+ * Passing "true" to the last argument is okay; it adds a check
+ * on bit 8 of the SPTEs which KVM doesn't use anyway.
+ */
__reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
boot_cpu_data.x86_phys_bits,
context->shadow_root_level, context->nx,
- guest_cpuid_has_gbpages(vcpu), is_pse(vcpu));
+ guest_cpuid_has_gbpages(vcpu), is_pse(vcpu),
+ true);
}
EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask);
+static inline bool boot_cpu_is_amd(void)
+{
+ WARN_ON_ONCE(!tdp_enabled);
+ return shadow_x_mask == 0;
+}
+
/*
* the direct page table on host, use as much mmu features as
* possible, however, kvm currently does not do execution-protection.
reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
struct kvm_mmu *context)
{
- if (guest_cpuid_is_amd(vcpu))
+ if (boot_cpu_is_amd())
__reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
boot_cpu_data.x86_phys_bits,
context->shadow_root_level, false,
- cpu_has_gbpages, true);
+ cpu_has_gbpages, true, true);
else
__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
boot_cpu_data.x86_phys_bits,
static int nested = true;
module_param(nested, int, S_IRUGO);
+static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
static void svm_flush_tlb(struct kvm_vcpu *vcpu);
static void svm_complete_interrupts(struct vcpu_svm *svm);
struct vcpu_svm *svm = to_svm(vcpu);
if (svm->vmcb->control.next_rip != 0) {
- WARN_ON(!static_cpu_has(X86_FEATURE_NRIPS));
+ WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
svm->next_rip = svm->vmcb->control.next_rip;
}
set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
}
-#define MTRR_TYPE_UC_MINUS 7
-#define MTRR2PROTVAL_INVALID 0xff
-
-static u8 mtrr2protval[8];
-
-static u8 fallback_mtrr_type(int mtrr)
-{
- /*
- * WT and WP aren't always available in the host PAT. Treat
- * them as UC and UC- respectively. Everything else should be
- * there.
- */
- switch (mtrr)
- {
- case MTRR_TYPE_WRTHROUGH:
- return MTRR_TYPE_UNCACHABLE;
- case MTRR_TYPE_WRPROT:
- return MTRR_TYPE_UC_MINUS;
- default:
- BUG();
- }
-}
-
-static void build_mtrr2protval(void)
-{
- int i;
- u64 pat;
-
- for (i = 0; i < 8; i++)
- mtrr2protval[i] = MTRR2PROTVAL_INVALID;
-
- /* Ignore the invalid MTRR types. */
- mtrr2protval[2] = 0;
- mtrr2protval[3] = 0;
-
- /*
- * Use host PAT value to figure out the mapping from guest MTRR
- * values to nested page table PAT/PCD/PWT values. We do not
- * want to change the host PAT value every time we enter the
- * guest.
- */
- rdmsrl(MSR_IA32_CR_PAT, pat);
- for (i = 0; i < 8; i++) {
- u8 mtrr = pat >> (8 * i);
-
- if (mtrr2protval[mtrr] == MTRR2PROTVAL_INVALID)
- mtrr2protval[mtrr] = __cm_idx2pte(i);
- }
-
- for (i = 0; i < 8; i++) {
- if (mtrr2protval[i] == MTRR2PROTVAL_INVALID) {
- u8 fallback = fallback_mtrr_type(i);
- mtrr2protval[i] = mtrr2protval[fallback];
- BUG_ON(mtrr2protval[i] == MTRR2PROTVAL_INVALID);
- }
- }
-}
-
static __init int svm_hardware_setup(void)
{
int cpu;
} else
kvm_disable_tdp();
- build_mtrr2protval();
return 0;
err:
return target_tsc - tsc;
}
-static void svm_set_guest_pat(struct vcpu_svm *svm, u64 *g_pat)
-{
- struct kvm_vcpu *vcpu = &svm->vcpu;
-
- /* Unlike Intel, AMD takes the guest's CR0.CD into account.
- *
- * AMD does not have IPAT. To emulate it for the case of guests
- * with no assigned devices, just set everything to WB. If guests
- * have assigned devices, however, we cannot force WB for RAM
- * pages only, so use the guest PAT directly.
- */
- if (!kvm_arch_has_assigned_device(vcpu->kvm))
- *g_pat = 0x0606060606060606;
- else
- *g_pat = vcpu->arch.pat;
-}
-
-static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
-{
- u8 mtrr;
-
- /*
- * 1. MMIO: trust guest MTRR, so same as item 3.
- * 2. No passthrough: always map as WB, and force guest PAT to WB as well
- * 3. Passthrough: can't guarantee the result, try to trust guest.
- */
- if (!is_mmio && !kvm_arch_has_assigned_device(vcpu->kvm))
- return 0;
-
- if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED) &&
- kvm_read_cr0(vcpu) & X86_CR0_CD)
- return _PAGE_NOCACHE;
-
- mtrr = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
- return mtrr2protval[mtrr];
-}
-
static void init_vmcb(struct vcpu_svm *svm, bool init_event)
{
struct vmcb_control_area *control = &svm->vmcb->control;
* svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
* It also updates the guest-visible cr0 value.
*/
- (void)kvm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
+ svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
+ kvm_mmu_reset_context(&svm->vcpu);
save->cr4 = X86_CR4_PAE;
/* rdx = ?? */
clr_cr_intercept(svm, INTERCEPT_CR3_READ);
clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
save->g_pat = svm->vcpu.arch.pat;
- svm_set_guest_pat(svm, &save->g_pat);
save->cr3 = 0;
save->cr4 = 0;
}
if (!vcpu->fpu_active)
cr0 |= X86_CR0_TS;
-
- /* These are emulated via page tables. */
- cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
-
+ /*
+ * re-enable caching here because the QEMU bios
+ * does not do it - this results in some delay at
+ * reboot
+ */
+ if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
+ cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
svm->vmcb->save.cr0 = cr0;
mark_dirty(svm->vmcb, VMCB_CR);
update_cr0_intercept(svm);
case MSR_VM_IGNNE:
vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
break;
- case MSR_IA32_CR_PAT:
- if (npt_enabled) {
- if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
- return 1;
- vcpu->arch.pat = data;
- svm_set_guest_pat(svm, &svm->vmcb->save.g_pat);
- mark_dirty(svm->vmcb, VMCB_NPT);
- break;
- }
- /* fall through */
default:
return kvm_set_msr_common(vcpu, msr);
}
return true;
}
+static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
+{
+ return 0;
+}
+
static void svm_cpuid_update(struct kvm_vcpu *vcpu)
{
}
memcpy(vmx_msr_bitmap_longmode_x2apic,
vmx_msr_bitmap_longmode, PAGE_SIZE);
+ set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
+
if (enable_apicv) {
for (msr = 0x800; msr <= 0x8ff; msr++)
vmx_disable_intercept_msr_read_x2apic(msr);
u64 ipat = 0;
/* For VT-d and EPT combination
- * 1. MMIO: guest may want to apply WC, trust it.
+ * 1. MMIO: always map as UC
* 2. EPT with VT-d:
* a. VT-d without snooping control feature: can't guarantee the
- * result, try to trust guest. So the same as item 1.
+ * result, try to trust guest.
* b. VT-d with snooping control feature: snooping control feature of
* VT-d engine can guarantee the cache correctness. Just set it
* to WB to keep consistent with host. So the same as item 3.
* 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
* consistent with host MTRR
*/
- if (!is_mmio && !kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
+ if (is_mmio) {
+ cache = MTRR_TYPE_UNCACHABLE;
+ goto exit;
+ }
+
+ if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
ipat = VMX_EPT_IPAT_BIT;
cache = MTRR_TYPE_WRBACK;
goto exit;
{ "nmi_window", VCPU_STAT(nmi_window_exits) },
{ "halt_exits", VCPU_STAT(halt_exits) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
+ { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "hypercalls", VCPU_STAT(hypercalls) },
{ "request_irq", VCPU_STAT(request_irq_exits) },
vcpu->pvclock_set_guest_stopped_request = false;
}
- pvclock_flags |= PVCLOCK_COUNTS_FROM_ZERO;
-
/* If the host uses TSC clocksource, then it is stable */
if (use_master_clock)
pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
&vcpu->requests);
ka->boot_vcpu_runs_old_kvmclock = tmp;
-
- ka->kvmclock_offset = -get_kernel_ns();
}
vcpu->arch.time = data;
case MSR_IA32_LASTINTFROMIP:
case MSR_IA32_LASTINTTOIP:
case MSR_K8_SYSCFG:
+ case MSR_K8_TSEG_ADDR:
+ case MSR_K8_TSEG_MASK:
case MSR_K7_HWCR:
case MSR_VM_HSAVE_PA:
case MSR_K8_INT_PENDING_MSG:
* This is the Guest timer interrupt handler (hardware interrupt 0). We just
* call the clockevent infrastructure and it does whatever needs doing.
*/
-static void lguest_time_irq(unsigned int irq, struct irq_desc *desc)
+static void lguest_time_irq(struct irq_desc *desc)
{
unsigned long flags;
* has been zapped already via cleanup_highmem().
*/
all_end = roundup((unsigned long)_brk_end, PMD_SIZE);
- set_memory_nx(rodata_start, (all_end - rodata_start) >> PAGE_SHIFT);
+ set_memory_nx(text_end, (all_end - text_end) >> PAGE_SHIFT);
rodata_test();
node_set(node, numa_nodes_parsed);
- pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s\n",
+ pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n",
node, pxm,
(unsigned long long) start, (unsigned long long) end - 1,
- hotpluggable ? " hotplug" : "");
+ hotpluggable ? " hotplug" : "",
+ ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : "");
/* Mark hotplug range in memblock. */
if (hotpluggable && memblock_mark_hotplug(start, ma->length))
{
struct pci_dev *dev;
+ pci_read_bridge_bases(b);
list_for_each_entry(dev, &b->devices, bus_list)
pcibios_fixup_device_resources(dev);
}
return ret;
}
+/*
+ * Iterate the EFI memory map in reverse order because the regions
+ * will be mapped top-down. The end result is the same as if we had
+ * mapped things forward, but doesn't require us to change the
+ * existing implementation of efi_map_region().
+ */
+static inline void *efi_map_next_entry_reverse(void *entry)
+{
+ /* Initial call */
+ if (!entry)
+ return memmap.map_end - memmap.desc_size;
+
+ entry -= memmap.desc_size;
+ if (entry < memmap.map)
+ return NULL;
+
+ return entry;
+}
+
+/*
+ * efi_map_next_entry - Return the next EFI memory map descriptor
+ * @entry: Previous EFI memory map descriptor
+ *
+ * This is a helper function to iterate over the EFI memory map, which
+ * we do in different orders depending on the current configuration.
+ *
+ * To begin traversing the memory map @entry must be %NULL.
+ *
+ * Returns %NULL when we reach the end of the memory map.
+ */
+static void *efi_map_next_entry(void *entry)
+{
+ if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
+ /*
+ * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
+ * config table feature requires us to map all entries
+ * in the same order as they appear in the EFI memory
+ * map. That is to say, entry N must have a lower
+ * virtual address than entry N+1. This is because the
+ * firmware toolchain leaves relative references in
+ * the code/data sections, which are split and become
+ * separate EFI memory regions. Mapping things
+ * out-of-order leads to the firmware accessing
+ * unmapped addresses.
+ *
+ * Since we need to map things this way whether or not
+ * the kernel actually makes use of
+ * EFI_PROPERTIES_TABLE, let's just switch to this
+ * scheme by default for 64-bit.
+ */
+ return efi_map_next_entry_reverse(entry);
+ }
+
+ /* Initial call */
+ if (!entry)
+ return memmap.map;
+
+ entry += memmap.desc_size;
+ if (entry >= memmap.map_end)
+ return NULL;
+
+ return entry;
+}
+
/*
* Map the efi memory ranges of the runtime services and update new_mmap with
* virtual addresses.
unsigned long left = 0;
efi_memory_desc_t *md;
- for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
+ p = NULL;
+ while ((p = efi_map_next_entry(p))) {
md = p;
if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
#ifdef CONFIG_X86_64
#include <linux/memblock.h>
#include <linux/edd.h>
+#ifdef CONFIG_KEXEC_CORE
+#include <linux/kexec.h>
+#endif
+
#include <xen/xen.h>
#include <xen/events.h>
#include <xen/interface/xen.h>
/* Fast syscall setup is all done in hypercalls, so
these are all ignored. Stub them out here to stop
Xen console noise. */
+ break;
default:
if (!pmu_msr_write(msr, low, high, &ret))
.notifier_call = xen_hvm_cpu_notify,
};
+#ifdef CONFIG_KEXEC_CORE
+static void xen_hvm_shutdown(void)
+{
+ native_machine_shutdown();
+ if (kexec_in_progress)
+ xen_reboot(SHUTDOWN_soft_reset);
+}
+
+static void xen_hvm_crash_shutdown(struct pt_regs *regs)
+{
+ native_machine_crash_shutdown(regs);
+ xen_reboot(SHUTDOWN_soft_reset);
+}
+#endif
+
static void __init xen_hvm_guest_init(void)
{
if (xen_pv_domain())
x86_init.irqs.intr_init = xen_init_IRQ;
xen_hvm_init_time_ops();
xen_hvm_init_mmu_ops();
+#ifdef CONFIG_KEXEC_CORE
+ machine_ops.shutdown = xen_hvm_shutdown;
+ machine_ops.crash_shutdown = xen_hvm_crash_shutdown;
+#endif
}
#endif
static pte_t *p2m_missing_pte;
static pte_t *p2m_identity_pte;
+/*
+ * Hint at last populated PFN.
+ *
+ * Used to set HYPERVISOR_shared_info->arch.max_pfn so the toolstack
+ * can avoid scanning the whole P2M (which may be sized to account for
+ * hotplugged memory).
+ */
+static unsigned long xen_p2m_last_pfn;
+
static inline unsigned p2m_top_index(unsigned long pfn)
{
BUG_ON(pfn >= MAX_P2M_PFN);
else
HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
virt_to_mfn(p2m_top_mfn);
- HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn;
+ HYPERVISOR_shared_info->arch.max_pfn = xen_p2m_last_pfn;
HYPERVISOR_shared_info->arch.p2m_generation = 0;
HYPERVISOR_shared_info->arch.p2m_vaddr = (unsigned long)xen_p2m_addr;
HYPERVISOR_shared_info->arch.p2m_cr3 =
static struct vm_struct vm;
unsigned long p2m_limit;
+ xen_p2m_last_pfn = xen_max_p2m_pfn;
+
p2m_limit = (phys_addr_t)P2M_LIMIT * 1024 * 1024 * 1024 / PAGE_SIZE;
vm.flags = VM_ALLOC;
vm.size = ALIGN(sizeof(unsigned long) * max(xen_max_p2m_pfn, p2m_limit),
free_p2m_page(p2m);
}
+ /* Expanded the p2m? */
+ if (pfn > xen_p2m_last_pfn) {
+ xen_p2m_last_pfn = pfn;
+ HYPERVISOR_shared_info->arch.max_pfn = xen_p2m_last_pfn;
+ }
+
return true;
}
{
unsigned long max_pages, limit;
domid_t domid = DOMID_SELF;
- int ret;
+ long ret;
limit = xen_get_pages_limit();
max_pages = limit;
xen_ignore_unusable();
/* Make sure the Xen-supplied memory map is well-ordered. */
- sanitize_e820_map(xen_e820_map, xen_e820_map_entries,
+ sanitize_e820_map(xen_e820_map, ARRAY_SIZE(xen_e820_map),
&xen_e820_map_entries);
max_pages = xen_get_max_pages();
generic-y += termios.h
generic-y += topology.h
generic-y += trace_clock.h
+generic-y += word-at-a-time.h
generic-y += xor.h
void pcibios_fixup_bus(struct pci_bus *bus)
{
+ if (bus->parent) {
+ /* This is a subordinate bridge */
+ pci_read_bridge_bases(bus);
+ }
}
void pcibios_set_master(struct pci_dev *dev)
iv = bip->bip_vec + bip->bip_vcnt;
+ if (bip->bip_vcnt &&
+ bvec_gap_to_prev(bdev_get_queue(bio->bi_bdev),
+ &bip->bip_vec[bip->bip_vcnt - 1], offset))
+ return 0;
+
iv->bv_page = page;
iv->bv_len = len;
iv->bv_offset = offset;
blkg_destroy(blkg);
spin_unlock(&blkcg->lock);
}
+
+ q->root_blkg = NULL;
+ q->root_rl.blkg = NULL;
}
/*
q->limits.max_integrity_segments)
return false;
+ if (integrity_req_gap_back_merge(req, next->bio))
+ return false;
+
return true;
}
EXPORT_SYMBOL(blk_integrity_merge_rq);
#include "blk.h"
+static bool iovec_gap_to_prv(struct request_queue *q,
+ struct iovec *prv, struct iovec *cur)
+{
+ unsigned long prev_end;
+
+ if (!queue_virt_boundary(q))
+ return false;
+
+ if (prv->iov_base == NULL && prv->iov_len == 0)
+ /* prv is not set - don't check */
+ return false;
+
+ prev_end = (unsigned long)(prv->iov_base + prv->iov_len);
+
+ return (((unsigned long)cur->iov_base & queue_virt_boundary(q)) ||
+ prev_end & queue_virt_boundary(q));
+}
+
int blk_rq_append_bio(struct request_queue *q, struct request *rq,
struct bio *bio)
{
struct bio *bio;
int unaligned = 0;
struct iov_iter i;
- struct iovec iov;
+ struct iovec iov, prv = {.iov_base = NULL, .iov_len = 0};
if (!iter || !iter->count)
return -EINVAL;
/*
* Keep going so we check length of all segments
*/
- if (uaddr & queue_dma_alignment(q))
+ if ((uaddr & queue_dma_alignment(q)) ||
+ iovec_gap_to_prv(q, &prv, &iov))
unaligned = 1;
+
+ prv.iov_base = iov.iov_base;
+ prv.iov_len = iov.iov_len;
}
if (unaligned || (q->dma_pad_mask & iter->count) || map_data)
struct bio *bio,
struct bio_set *bs)
{
- struct bio *split;
- struct bio_vec bv, bvprv;
+ struct bio_vec bv, bvprv, *bvprvp = NULL;
struct bvec_iter iter;
unsigned seg_size = 0, nsegs = 0, sectors = 0;
- int prev = 0;
bio_for_each_segment(bv, bio, iter) {
- sectors += bv.bv_len >> 9;
-
- if (sectors > queue_max_sectors(q))
+ if (sectors + (bv.bv_len >> 9) > queue_max_sectors(q))
goto split;
/*
* If the queue doesn't support SG gaps and adding this
* offset would create a gap, disallow it.
*/
- if (prev && bvec_gap_to_prev(q, &bvprv, bv.bv_offset))
+ if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset))
goto split;
- if (prev && blk_queue_cluster(q)) {
+ if (bvprvp && blk_queue_cluster(q)) {
if (seg_size + bv.bv_len > queue_max_segment_size(q))
goto new_segment;
- if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv))
+ if (!BIOVEC_PHYS_MERGEABLE(bvprvp, &bv))
goto new_segment;
- if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv))
+ if (!BIOVEC_SEG_BOUNDARY(q, bvprvp, &bv))
goto new_segment;
seg_size += bv.bv_len;
bvprv = bv;
- prev = 1;
+ bvprvp = &bv;
+ sectors += bv.bv_len >> 9;
continue;
}
new_segment:
nsegs++;
bvprv = bv;
- prev = 1;
+ bvprvp = &bv;
seg_size = bv.bv_len;
+ sectors += bv.bv_len >> 9;
}
return NULL;
split:
- split = bio_clone_bioset(bio, GFP_NOIO, bs);
-
- split->bi_iter.bi_size -= iter.bi_size;
- bio->bi_iter = iter;
-
- if (bio_integrity(bio)) {
- bio_integrity_advance(bio, split->bi_iter.bi_size);
- bio_integrity_trim(split, 0, bio_sectors(split));
- }
-
- return split;
+ return bio_split(bio, sectors, GFP_NOIO, bs);
}
void blk_queue_split(struct request_queue *q, struct bio **bio,
int ll_back_merge_fn(struct request_queue *q, struct request *req,
struct bio *bio)
{
+ if (req_gap_back_merge(req, bio))
+ return 0;
+ if (blk_integrity_rq(req) &&
+ integrity_req_gap_back_merge(req, bio))
+ return 0;
if (blk_rq_sectors(req) + bio_sectors(bio) >
blk_rq_get_max_sectors(req)) {
req->cmd_flags |= REQ_NOMERGE;
int ll_front_merge_fn(struct request_queue *q, struct request *req,
struct bio *bio)
{
+
+ if (req_gap_front_merge(req, bio))
+ return 0;
+ if (blk_integrity_rq(req) &&
+ integrity_req_gap_front_merge(req, bio))
+ return 0;
if (blk_rq_sectors(req) + bio_sectors(bio) >
blk_rq_get_max_sectors(req)) {
req->cmd_flags |= REQ_NOMERGE;
return !q->mq_ops && req->special;
}
-static int req_gap_to_prev(struct request *req, struct bio *next)
-{
- struct bio *prev = req->biotail;
-
- return bvec_gap_to_prev(req->q, &prev->bi_io_vec[prev->bi_vcnt - 1],
- next->bi_io_vec[0].bv_offset);
-}
-
static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
struct request *next)
{
if (req_no_special_merge(req) || req_no_special_merge(next))
return 0;
- if (req_gap_to_prev(req, next->bio))
+ if (req_gap_back_merge(req, next->bio))
return 0;
/*
!blk_write_same_mergeable(rq->bio, bio))
return false;
- /* Only check gaps if the bio carries data */
- if (bio_has_data(bio) && req_gap_to_prev(rq, bio))
- return false;
-
return true;
}
return cpu;
}
-int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues)
+int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues,
+ const struct cpumask *online_mask)
{
unsigned int i, nr_cpus, nr_uniq_cpus, queue, first_sibling;
cpumask_var_t cpus;
cpumask_clear(cpus);
nr_cpus = nr_uniq_cpus = 0;
- for_each_online_cpu(i) {
+ for_each_cpu(i, online_mask) {
nr_cpus++;
first_sibling = get_first_sibling(i);
if (!cpumask_test_cpu(first_sibling, cpus))
queue = 0;
for_each_possible_cpu(i) {
- if (!cpu_online(i)) {
+ if (!cpumask_test_cpu(i, online_mask)) {
map[i] = 0;
continue;
}
if (!map)
return NULL;
- if (!blk_mq_update_queue_map(map, set->nr_hw_queues))
+ if (!blk_mq_update_queue_map(map, set->nr_hw_queues, cpu_online_mask))
return map;
kfree(map);
unsigned int i, first = 1;
ssize_t ret = 0;
- blk_mq_disable_hotplug();
-
for_each_cpu(i, hctx->cpumask) {
if (first)
ret += sprintf(ret + page, "%u", i);
first = 0;
}
- blk_mq_enable_hotplug();
-
ret += sprintf(ret + page, "\n");
return ret;
}
struct blk_mq_ctx *ctx;
int i;
- if (!hctx->nr_ctx || !(hctx->flags & BLK_MQ_F_SYSFS_UP))
+ if (!hctx->nr_ctx)
return;
hctx_for_each_ctx(hctx, ctx, i)
struct blk_mq_ctx *ctx;
int i, ret;
- if (!hctx->nr_ctx || !(hctx->flags & BLK_MQ_F_SYSFS_UP))
+ if (!hctx->nr_ctx)
return 0;
ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
struct blk_mq_ctx *ctx;
int i, j;
+ blk_mq_disable_hotplug();
+
queue_for_each_hw_ctx(q, hctx, i) {
blk_mq_unregister_hctx(hctx);
kobject_put(&q->mq_kobj);
kobject_put(&disk_to_dev(disk)->kobj);
+
+ q->mq_sysfs_init_done = false;
+ blk_mq_enable_hotplug();
}
static void blk_mq_sysfs_init(struct request_queue *q)
struct blk_mq_hw_ctx *hctx;
int ret, i;
+ blk_mq_disable_hotplug();
+
blk_mq_sysfs_init(q);
ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
if (ret < 0)
- return ret;
+ goto out;
kobject_uevent(&q->mq_kobj, KOBJ_ADD);
queue_for_each_hw_ctx(q, hctx, i) {
- hctx->flags |= BLK_MQ_F_SYSFS_UP;
ret = blk_mq_register_hctx(hctx);
if (ret)
break;
}
- if (ret) {
+ if (ret)
blk_mq_unregister_disk(disk);
- return ret;
- }
+ else
+ q->mq_sysfs_init_done = true;
+out:
+ blk_mq_enable_hotplug();
- return 0;
+ return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_register_disk);
struct blk_mq_hw_ctx *hctx;
int i;
+ if (!q->mq_sysfs_init_done)
+ return;
+
queue_for_each_hw_ctx(q, hctx, i)
blk_mq_unregister_hctx(hctx);
}
struct blk_mq_hw_ctx *hctx;
int i, ret = 0;
+ if (!q->mq_sysfs_init_done)
+ return ret;
+
queue_for_each_hw_ctx(q, hctx, i) {
ret = blk_mq_register_hctx(hctx);
if (ret)
}
EXPORT_SYMBOL(blk_mq_all_tag_busy_iter);
-void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx *hctx, busy_iter_fn *fn,
+void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
void *priv)
{
- struct blk_mq_tags *tags = hctx->tags;
+ struct blk_mq_hw_ctx *hctx;
+ int i;
+
+
+ queue_for_each_hw_ctx(q, hctx, i) {
+ struct blk_mq_tags *tags = hctx->tags;
+
+ /*
+ * If not software queues are currently mapped to this
+ * hardware queue, there's nothing to check
+ */
+ if (!blk_mq_hw_queue_mapped(hctx))
+ continue;
+
+ if (tags->nr_reserved_tags)
+ bt_for_each(hctx, &tags->breserved_tags, 0, fn, priv, true);
+ bt_for_each(hctx, &tags->bitmap_tags, tags->nr_reserved_tags, fn, priv,
+ false);
+ }
- if (tags->nr_reserved_tags)
- bt_for_each(hctx, &tags->breserved_tags, 0, fn, priv, true);
- bt_for_each(hctx, &tags->bitmap_tags, tags->nr_reserved_tags, fn, priv,
- false);
}
-EXPORT_SYMBOL(blk_mq_tag_busy_iter);
static unsigned int bt_unused_tags(struct blk_mq_bitmap_tags *bt)
{
extern void blk_mq_tag_init_last_tag(struct blk_mq_tags *tags, unsigned int *last_tag);
extern int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int depth);
extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
+void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
+ void *priv);
enum {
BLK_MQ_TAG_CACHE_MIN = 1,
* Ends all I/O on a request. It does not handle partial completions.
* The actual completion happens out-of-order, through a IPI handler.
**/
-void blk_mq_complete_request(struct request *rq)
+void blk_mq_complete_request(struct request *rq, int error)
{
struct request_queue *q = rq->q;
if (unlikely(blk_should_fake_timeout(q)))
return;
- if (!blk_mark_rq_complete(rq))
+ if (!blk_mark_rq_complete(rq)) {
+ rq->errors = error;
__blk_mq_complete_request(rq);
+ }
}
EXPORT_SYMBOL(blk_mq_complete_request);
* If a request wasn't started before the queue was
* marked dying, kill it here or it'll go unnoticed.
*/
- if (unlikely(blk_queue_dying(rq->q))) {
- rq->errors = -EIO;
- blk_mq_complete_request(rq);
- }
+ if (unlikely(blk_queue_dying(rq->q)))
+ blk_mq_complete_request(rq, -EIO);
return;
}
if (rq->cmd_flags & REQ_NO_TIMEOUT)
.next = 0,
.next_set = 0,
};
- struct blk_mq_hw_ctx *hctx;
int i;
- queue_for_each_hw_ctx(q, hctx, i) {
- /*
- * If not software queues are currently mapped to this
- * hardware queue, there's nothing to check
- */
- if (!blk_mq_hw_queue_mapped(hctx))
- continue;
-
- blk_mq_tag_busy_iter(hctx, blk_mq_check_expired, &data);
- }
+ blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
if (data.next_set) {
data.next = blk_rq_timeout(round_jiffies_up(data.next));
mod_timer(&q->timeout, data.next);
} else {
+ struct blk_mq_hw_ctx *hctx;
+
queue_for_each_hw_ctx(q, hctx, i) {
/* the hctx may be unmapped, so check it here */
if (blk_mq_hw_queue_mapped(hctx))
}
}
-static void blk_mq_map_swqueue(struct request_queue *q)
+static void blk_mq_map_swqueue(struct request_queue *q,
+ const struct cpumask *online_mask)
{
unsigned int i;
struct blk_mq_hw_ctx *hctx;
struct blk_mq_ctx *ctx;
struct blk_mq_tag_set *set = q->tag_set;
+ /*
+ * Avoid others reading imcomplete hctx->cpumask through sysfs
+ */
+ mutex_lock(&q->sysfs_lock);
+
queue_for_each_hw_ctx(q, hctx, i) {
cpumask_clear(hctx->cpumask);
hctx->nr_ctx = 0;
*/
queue_for_each_ctx(q, ctx, i) {
/* If the cpu isn't online, the cpu is mapped to first hctx */
- if (!cpu_online(i))
+ if (!cpumask_test_cpu(i, online_mask))
continue;
hctx = q->mq_ops->map_queue(q, i);
cpumask_set_cpu(i, hctx->cpumask);
- cpumask_set_cpu(i, hctx->tags->cpumask);
ctx->index_hw = hctx->nr_ctx;
hctx->ctxs[hctx->nr_ctx++] = ctx;
}
+ mutex_unlock(&q->sysfs_lock);
+
queue_for_each_hw_ctx(q, hctx, i) {
struct blk_mq_ctxmap *map = &hctx->ctx_map;
hctx->next_cpu = cpumask_first(hctx->cpumask);
hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
}
+
+ queue_for_each_ctx(q, ctx, i) {
+ if (!cpumask_test_cpu(i, online_mask))
+ continue;
+
+ hctx = q->mq_ops->map_queue(q, i);
+ cpumask_set_cpu(i, hctx->tags->cpumask);
+ }
}
static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
kfree(hctx);
}
+ kfree(q->mq_map);
+ q->mq_map = NULL;
+
kfree(q->queue_hw_ctx);
/* ctx kobj stays in queue_ctx */
if (blk_mq_init_hw_queues(q, set))
goto err_hctxs;
+ get_online_cpus();
mutex_lock(&all_q_mutex);
- list_add_tail(&q->all_q_node, &all_q_list);
- mutex_unlock(&all_q_mutex);
+ list_add_tail(&q->all_q_node, &all_q_list);
blk_mq_add_queue_tag_set(set, q);
+ blk_mq_map_swqueue(q, cpu_online_mask);
- blk_mq_map_swqueue(q);
+ mutex_unlock(&all_q_mutex);
+ put_online_cpus();
return q;
{
struct blk_mq_tag_set *set = q->tag_set;
+ mutex_lock(&all_q_mutex);
+ list_del_init(&q->all_q_node);
+ mutex_unlock(&all_q_mutex);
+
blk_mq_del_queue_tag_set(q);
blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
blk_mq_free_hw_queues(q, set);
percpu_ref_exit(&q->mq_usage_counter);
-
- kfree(q->mq_map);
-
- q->mq_map = NULL;
-
- mutex_lock(&all_q_mutex);
- list_del_init(&q->all_q_node);
- mutex_unlock(&all_q_mutex);
}
/* Basically redo blk_mq_init_queue with queue frozen */
-static void blk_mq_queue_reinit(struct request_queue *q)
+static void blk_mq_queue_reinit(struct request_queue *q,
+ const struct cpumask *online_mask)
{
WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
blk_mq_sysfs_unregister(q);
- blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
+ blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues, online_mask);
/*
* redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
* involves free and re-allocate memory, worthy doing?)
*/
- blk_mq_map_swqueue(q);
+ blk_mq_map_swqueue(q, online_mask);
blk_mq_sysfs_register(q);
}
unsigned long action, void *hcpu)
{
struct request_queue *q;
+ int cpu = (unsigned long)hcpu;
+ /*
+ * New online cpumask which is going to be set in this hotplug event.
+ * Declare this cpumasks as global as cpu-hotplug operation is invoked
+ * one-by-one and dynamically allocating this could result in a failure.
+ */
+ static struct cpumask online_new;
/*
- * Before new mappings are established, hotadded cpu might already
- * start handling requests. This doesn't break anything as we map
- * offline CPUs to first hardware queue. We will re-init the queue
- * below to get optimal settings.
+ * Before hotadded cpu starts handling requests, new mappings must
+ * be established. Otherwise, these requests in hw queue might
+ * never be dispatched.
+ *
+ * For example, there is a single hw queue (hctx) and two CPU queues
+ * (ctx0 for CPU0, and ctx1 for CPU1).
+ *
+ * Now CPU1 is just onlined and a request is inserted into
+ * ctx1->rq_list and set bit0 in pending bitmap as ctx1->index_hw is
+ * still zero.
+ *
+ * And then while running hw queue, flush_busy_ctxs() finds bit0 is
+ * set in pending bitmap and tries to retrieve requests in
+ * hctx->ctxs[0]->rq_list. But htx->ctxs[0] is a pointer to ctx0,
+ * so the request in ctx1->rq_list is ignored.
*/
- if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
- action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
+ switch (action & ~CPU_TASKS_FROZEN) {
+ case CPU_DEAD:
+ case CPU_UP_CANCELED:
+ cpumask_copy(&online_new, cpu_online_mask);
+ break;
+ case CPU_UP_PREPARE:
+ cpumask_copy(&online_new, cpu_online_mask);
+ cpumask_set_cpu(cpu, &online_new);
+ break;
+ default:
return NOTIFY_OK;
+ }
mutex_lock(&all_q_mutex);
}
list_for_each_entry(q, &all_q_list, all_q_node)
- blk_mq_queue_reinit(q);
+ blk_mq_queue_reinit(q, &online_new);
list_for_each_entry(q, &all_q_list, all_q_node)
blk_mq_unfreeze_queue(q);
* CPU -> queue mappings
*/
extern unsigned int *blk_mq_make_queue_map(struct blk_mq_tag_set *set);
-extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues);
+extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues,
+ const struct cpumask *online_mask);
extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);
/*
struct bio *bio_orig = bio->bi_private;
struct bio_vec *bvec, *org_vec;
int i;
+ int start = bio_orig->bi_iter.bi_idx;
/*
* free up bounce indirect pages used
*/
bio_for_each_segment_all(bvec, bio, i) {
- org_vec = bio_orig->bi_io_vec + i;
+ org_vec = bio_orig->bi_io_vec + i + start;
+
if (bvec->bv_page == org_vec->bv_page)
continue;
srlen = cert->raw_serial_size;
q = cert->raw_serial;
}
- if (srlen > 1 && *q == 0) {
- srlen--;
- q++;
- }
ret = -ENOMEM;
desc = kmalloc(sulen + 2 + srlen * 2 + 1, GFP_KERNEL);
char *xbuf[XBUFSIZE];
char *xoutbuf[XBUFSIZE];
int ret = -ENOMEM;
+ unsigned int ivsize = crypto_skcipher_ivsize(tfm);
if (testmgr_alloc_buf(xbuf))
goto out_nobuf;
continue;
if (template[i].iv)
- memcpy(iv, template[i].iv, MAX_IVLEN);
+ memcpy(iv, template[i].iv, ivsize);
else
memset(iv, 0, MAX_IVLEN);
continue;
if (template[i].iv)
- memcpy(iv, template[i].iv, MAX_IVLEN);
+ memcpy(iv, template[i].iv, ivsize);
else
memset(iv, 0, MAX_IVLEN);
capbuf[OSC_QUERY_DWORD] = OSC_QUERY_ENABLE;
capbuf[OSC_SUPPORT_DWORD] = OSC_SB_PR3_SUPPORT; /* _PR3 is in use */
-#if defined(CONFIG_ACPI_PROCESSOR_AGGREGATOR) ||\
- defined(CONFIG_ACPI_PROCESSOR_AGGREGATOR_MODULE)
- capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PAD_SUPPORT;
-#endif
-
-#if defined(CONFIG_ACPI_PROCESSOR) || defined(CONFIG_ACPI_PROCESSOR_MODULE)
- capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PPC_OST_SUPPORT;
-#endif
+ if (IS_ENABLED(CONFIG_ACPI_PROCESSOR_AGGREGATOR))
+ capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PAD_SUPPORT;
+ if (IS_ENABLED(CONFIG_ACPI_PROCESSOR))
+ capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PPC_OST_SUPPORT;
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_HOTPLUG_OST_SUPPORT;
goto err_exit;
mutex_lock(&ec->mutex);
+ result = -ENODATA;
list_for_each_entry(handler, &ec->list, node) {
if (value == handler->query_bit) {
+ result = 0;
q->handler = acpi_ec_get_query_handler(handler);
ec_dbg_evt("Query(0x%02x) scheduled",
q->handler->query_bit);
static int int340x_thermal_handler_attach(struct acpi_device *adev,
const struct acpi_device_id *id)
{
-#if defined(CONFIG_INT340X_THERMAL) || defined(CONFIG_INT340X_THERMAL_MODULE)
- acpi_create_platform_device(adev);
-#elif defined(INTEL_SOC_DTS_THERMAL) || defined(INTEL_SOC_DTS_THERMAL_MODULE)
+ if (IS_ENABLED(CONFIG_INT340X_THERMAL))
+ acpi_create_platform_device(adev);
/* Intel SoC DTS thermal driver needs INT3401 to set IRQ descriptor */
- if (id->driver_data == INT3401_DEVICE)
+ else if (IS_ENABLED(CONFIG_INTEL_SOC_DTS_THERMAL) &&
+ id->driver_data == INT3401_DEVICE)
acpi_create_platform_device(adev);
-#endif
return 1;
}
/* Interrupt Line values above 0xF are forbidden */
if (dev->irq > 0 && (dev->irq <= 0xF) &&
+ acpi_isa_irq_available(dev->irq) &&
(acpi_isa_irq_to_gsi(dev->irq, &dev_gsi) == 0)) {
dev_warn(&dev->dev, "PCI INT %c: no GSI - using ISA IRQ %d\n",
pin_name(dev->pin), dev->irq);
PIRQ_PENALTY_PCI_POSSIBLE;
}
}
- /* Add a penalty for the SCI */
- acpi_irq_penalty[acpi_gbl_FADT.sci_interrupt] += PIRQ_PENALTY_PCI_USING;
+
return 0;
}
irq = link->irq.possible[i];
}
}
+ if (acpi_irq_penalty[irq] >= PIRQ_PENALTY_ISA_ALWAYS) {
+ printk(KERN_ERR PREFIX "No IRQ available for %s [%s]. "
+ "Try pci=noacpi or acpi=off\n",
+ acpi_device_name(link->device),
+ acpi_device_bid(link->device));
+ return -ENODEV;
+ }
/* Attempt to enable the link device at this IRQ. */
if (acpi_pci_link_set(link, irq)) {
}
}
+bool acpi_isa_irq_available(int irq)
+{
+ return irq >= 0 && (irq >= ARRAY_SIZE(acpi_irq_penalty) ||
+ acpi_irq_penalty[irq] < PIRQ_PENALTY_ISA_ALWAYS);
+}
+
/*
* Penalize IRQ used by ACPI SCI. If ACPI SCI pin attributes conflict with
* PCI IRQ attributes, mark ACPI SCI as ISA_ALWAYS so it won't be use for
kfree(he_dev->rbpl_virt);
kfree(he_dev->rbpl_table);
-
- if (he_dev->rbpl_pool)
- dma_pool_destroy(he_dev->rbpl_pool);
+ dma_pool_destroy(he_dev->rbpl_pool);
if (he_dev->rbrq_base)
dma_free_coherent(&he_dev->pci_dev->dev, CONFIG_RBRQ_SIZE * sizeof(struct he_rbrq),
dma_free_coherent(&he_dev->pci_dev->dev, CONFIG_TBRQ_SIZE * sizeof(struct he_tbrq),
he_dev->tpdrq_base, he_dev->tpdrq_phys);
- if (he_dev->tpd_pool)
- dma_pool_destroy(he_dev->tpd_pool);
+ dma_pool_destroy(he_dev->tpd_pool);
if (he_dev->pci_dev) {
pci_read_config_word(he_dev->pci_dev, PCI_COMMAND, &command);
continue;
}
- skb = alloc_skb(size + 1, GFP_ATOMIC);
+ /* Use netdev_alloc_skb() because it adds NET_SKB_PAD of
+ * headroom, and ensures we can route packets back out an
+ * Ethernet interface (for example) without having to
+ * reallocate. Adding NET_IP_ALIGN also ensures that both
+ * PPPoATM and PPPoEoBR2684 packets end up aligned. */
+ skb = netdev_alloc_skb_ip_align(NULL, size + 1);
if (!skb) {
if (net_ratelimit())
dev_warn(&card->dev->dev, "Failed to allocate sk_buff for RX\n");
/* Allocate RX skbs for any ports which need them */
if (card->using_dma && card->atmdev[port] &&
!card->rx_skb[port]) {
- struct sk_buff *skb = alloc_skb(RX_DMA_SIZE, GFP_ATOMIC);
+ /* Unlike the MMIO case (qv) we can't add NET_IP_ALIGN
+ * here; the FPGA can only DMA to addresses which are
+ * aligned to 4 bytes. */
+ struct sk_buff *skb = dev_alloc_skb(RX_DMA_SIZE);
if (skb) {
SKB_CB(skb)->dma_addr =
dma_map_single(&card->dev->dev, skb->data,
if (sibling == cpu) /* skip itself */
continue;
+
sib_cpu_ci = get_cpu_cacheinfo(sibling);
+ if (!sib_cpu_ci->info_list)
+ continue;
+
sib_leaf = sib_cpu_ci->info_list + index;
cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map);
cpumask_clear_cpu(sibling, &this_leaf->shared_cpu_map);
static void free_cache_attributes(unsigned int cpu)
{
+ if (!per_cpu_cacheinfo(cpu))
+ return;
+
cache_shared_cpu_map_remove(cpu);
kfree(per_cpu_cacheinfo(cpu));
break;
case CPU_DEAD:
cache_remove_dev(cpu);
- if (per_cpu_cacheinfo(cpu))
- free_cache_attributes(cpu);
+ free_cache_attributes(cpu);
break;
}
return notifier_from_errno(rc);
unsigned int virq, irq_hw_number_t hwirq,
msi_alloc_info_t *arg)
{
- struct irq_data *data;
-
- irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
- info->chip, info->chip_data);
-
- /*
- * Save the MSI descriptor in handler_data so that the
- * irq_write_msi_msg callback can retrieve it (and the
- * associated device).
- */
- data = irq_domain_get_irq_data(domain, virq);
- data->handler_data = arg->desc;
-
- return 0;
+ return irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
+ info->chip, info->chip_data);
}
#else
#define platform_msi_set_desc NULL
static void platform_msi_write_msg(struct irq_data *data, struct msi_msg *msg)
{
- struct msi_desc *desc = irq_data_get_irq_handler_data(data);
+ struct msi_desc *desc = irq_data_get_msi_desc(data);
struct platform_msi_priv_data *priv_data;
priv_data = desc->platform.msi_priv_data;
u32 microvolt[3] = {0};
int count, ret;
- count = of_property_count_u32_elems(opp->np, "opp-microvolt");
- if (!count)
+ /* Missing property isn't a problem, but an invalid entry is */
+ if (!of_find_property(opp->np, "opp-microvolt", NULL))
return 0;
+ count = of_property_count_u32_elems(opp->np, "opp-microvolt");
+ if (count < 0) {
+ dev_err(dev, "%s: Invalid opp-microvolt property (%d)\n",
+ __func__, count);
+ return count;
+ }
+
/* There can be one or three elements here */
if (count != 1 && count != 3) {
dev_err(dev, "%s: Invalid number of elements in opp-microvolt property (%d)\n",
* share a common logic which is isolated here.
*
* Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
- * copy operation, returns 0 if no modifcation was done OR modification was
+ * copy operation, returns 0 if no modification was done OR modification was
* successful.
*
* Locking: The internal device_opp and opp structures are RCU protected.
* mutex locking or synchronize_rcu() blocking calls cannot be used.
*
* Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
- * copy operation, returns 0 if no modifcation was done OR modification was
+ * copy operation, returns 0 if no modification was done OR modification was
* successful.
*/
int dev_pm_opp_enable(struct device *dev, unsigned long freq)
* mutex locking or synchronize_rcu() blocking calls cannot be used.
*
* Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
- * copy operation, returns 0 if no modifcation was done OR modification was
+ * copy operation, returns 0 if no modification was done OR modification was
* successful.
*/
int dev_pm_opp_disable(struct device *dev, unsigned long freq)
/* Calculate the length of a fixed format */
static size_t regmap_calc_reg_len(int max_val, char *buf, size_t buf_size)
{
- snprintf(buf, buf_size, "%x", max_val);
- return strlen(buf);
+ return snprintf(NULL, 0, "%x", max_val);
}
static ssize_t regmap_name_read_file(struct file *file,
/* If we're in the region the user is trying to read */
if (p >= *ppos) {
/* ...but not beyond it */
- if (buf_pos >= count - 1 - tot_len)
+ if (buf_pos + tot_len + 1 >= count)
break;
/* Format the register */
{
const bool write = cmd->rq->cmd_flags & REQ_WRITE;
struct loop_device *lo = cmd->rq->q->queuedata;
- int ret = -EIO;
+ int ret = 0;
- if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY))
+ if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
+ ret = -EIO;
goto failed;
+ }
ret = do_req_filebacked(lo, cmd->rq);
-
failed:
- if (ret)
- cmd->rq->errors = -EIO;
- blk_mq_complete_request(cmd->rq);
+ blk_mq_complete_request(cmd->rq, ret ? -EIO : 0);
}
static void loop_queue_write_work(struct work_struct *work)
case NULL_IRQ_SOFTIRQ:
switch (queue_mode) {
case NULL_Q_MQ:
- blk_mq_complete_request(cmd->rq);
+ blk_mq_complete_request(cmd->rq, cmd->rq->errors);
break;
case NULL_Q_RQ:
blk_complete_request(cmd->rq);
.complete = null_softirq_done_fn,
};
+static void cleanup_queue(struct nullb_queue *nq)
+{
+ kfree(nq->tag_map);
+ kfree(nq->cmds);
+}
+
+static void cleanup_queues(struct nullb *nullb)
+{
+ int i;
+
+ for (i = 0; i < nullb->nr_queues; i++)
+ cleanup_queue(&nullb->queues[i]);
+
+ kfree(nullb->queues);
+}
+
static void null_del_dev(struct nullb *nullb)
{
list_del_init(&nullb->list);
if (queue_mode == NULL_Q_MQ)
blk_mq_free_tag_set(&nullb->tag_set);
put_disk(nullb->disk);
+ cleanup_queues(nullb);
kfree(nullb);
}
return 0;
}
-static void cleanup_queue(struct nullb_queue *nq)
-{
- kfree(nq->tag_map);
- kfree(nq->cmds);
-}
-
-static void cleanup_queues(struct nullb *nullb)
-{
- int i;
-
- for (i = 0; i < nullb->nr_queues; i++)
- cleanup_queue(&nullb->queues[i]);
-
- kfree(nullb->queues);
-}
-
static int setup_queues(struct nullb *nullb)
{
nullb->queues = kzalloc(submit_queues * sizeof(struct nullb_queue),
blk_queue_physical_block_size(nullb->q, bs);
size = gb * 1024 * 1024 * 1024ULL;
- sector_div(size, bs);
- set_capacity(disk, size);
+ set_capacity(disk, size >> 9);
disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
disk->major = null_major;
spin_unlock_irqrestore(req->q->queue_lock, flags);
return;
}
+
if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
if (cmd_rq->ctx == CMD_CTX_CANCELLED)
- req->errors = -EINTR;
- else
- req->errors = status;
+ status = -EINTR;
} else {
- req->errors = nvme_error_status(status);
+ status = nvme_error_status(status);
}
- } else
- req->errors = 0;
+ }
+
if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
u32 result = le32_to_cpup(&cqe->result);
req->special = (void *)(uintptr_t)result;
}
nvme_free_iod(nvmeq->dev, iod);
- blk_mq_complete_request(req);
+ blk_mq_complete_request(req, status);
}
/* length is in bytes. gfp flags indicates whether we may sleep. */
if (ns && ns->ms && !blk_integrity_rq(req)) {
if (!(ns->pi_type && ns->ms == 8) &&
req->cmd_type != REQ_TYPE_DRV_PRIV) {
- req->errors = -EFAULT;
- blk_mq_complete_request(req);
+ blk_mq_complete_request(req, -EFAULT);
return BLK_MQ_RQ_QUEUE_OK;
}
}
list_sort(NULL, &dev->namespaces, ns_cmp);
}
+static void nvme_set_irq_hints(struct nvme_dev *dev)
+{
+ struct nvme_queue *nvmeq;
+ int i;
+
+ for (i = 0; i < dev->online_queues; i++) {
+ nvmeq = dev->queues[i];
+
+ if (!nvmeq->tags || !(*nvmeq->tags))
+ continue;
+
+ irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
+ blk_mq_tags_cpumask(*nvmeq->tags));
+ }
+}
+
static void nvme_dev_scan(struct work_struct *work)
{
struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
return;
nvme_scan_namespaces(dev, le32_to_cpup(&ctrl->nn));
kfree(ctrl);
+ nvme_set_irq_hints(dev);
}
/*
.compat_ioctl = nvme_dev_ioctl,
};
-static void nvme_set_irq_hints(struct nvme_dev *dev)
-{
- struct nvme_queue *nvmeq;
- int i;
-
- for (i = 0; i < dev->online_queues; i++) {
- nvmeq = dev->queues[i];
-
- if (!nvmeq->tags || !(*nvmeq->tags))
- continue;
-
- irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
- blk_mq_tags_cpumask(*nvmeq->tags));
- }
-}
-
static int nvme_dev_start(struct nvme_dev *dev)
{
int result;
if (result)
goto free_tags;
- nvme_set_irq_hints(dev);
-
dev->event_limit = 1;
return result;
} else {
nvme_unfreeze_queues(dev);
nvme_dev_add(dev);
- nvme_set_irq_hints(dev);
}
return 0;
}
do {
virtqueue_disable_cb(vq);
while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {
- blk_mq_complete_request(vbr->req);
+ blk_mq_complete_request(vbr->req, vbr->req->errors);
req_done = true;
}
if (unlikely(virtqueue_is_broken(vq)))
static int xen_blkif_disconnect(struct xen_blkif *blkif)
{
+ struct pending_req *req, *n;
+ int i = 0, j;
+
if (blkif->xenblkd) {
kthread_stop(blkif->xenblkd);
wake_up(&blkif->shutdown_wq);
/* Remove all persistent grants and the cache of ballooned pages. */
xen_blkbk_free_caches(blkif);
+ /* Check that there is no request in use */
+ list_for_each_entry_safe(req, n, &blkif->pending_free, free_list) {
+ list_del(&req->free_list);
+
+ for (j = 0; j < MAX_INDIRECT_SEGMENTS; j++)
+ kfree(req->segments[j]);
+
+ for (j = 0; j < MAX_INDIRECT_PAGES; j++)
+ kfree(req->indirect_pages[j]);
+
+ kfree(req);
+ i++;
+ }
+
+ WARN_ON(i != (XEN_BLKIF_REQS_PER_PAGE * blkif->nr_ring_pages));
+ blkif->nr_ring_pages = 0;
+
return 0;
}
static void xen_blkif_free(struct xen_blkif *blkif)
{
- struct pending_req *req, *n;
- int i = 0, j;
xen_blkif_disconnect(blkif);
xen_vbd_free(&blkif->vbd);
BUG_ON(!list_empty(&blkif->free_pages));
BUG_ON(!RB_EMPTY_ROOT(&blkif->persistent_gnts));
- /* Check that there is no request in use */
- list_for_each_entry_safe(req, n, &blkif->pending_free, free_list) {
- list_del(&req->free_list);
-
- for (j = 0; j < MAX_INDIRECT_SEGMENTS; j++)
- kfree(req->segments[j]);
-
- for (j = 0; j < MAX_INDIRECT_PAGES; j++)
- kfree(req->indirect_pages[j]);
-
- kfree(req);
- i++;
- }
-
- WARN_ON(i != (XEN_BLKIF_REQS_PER_PAGE * blkif->nr_ring_pages));
-
kmem_cache_free(xen_blkif_cachep, blkif);
}
RING_IDX i, rp;
unsigned long flags;
struct blkfront_info *info = (struct blkfront_info *)dev_id;
+ int error;
spin_lock_irqsave(&info->io_lock, flags);
continue;
}
- req->errors = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO;
+ error = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO;
switch (bret->operation) {
case BLKIF_OP_DISCARD:
if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
struct request_queue *rq = info->rq;
printk(KERN_WARNING "blkfront: %s: %s op failed\n",
info->gd->disk_name, op_name(bret->operation));
- req->errors = -EOPNOTSUPP;
+ error = -EOPNOTSUPP;
info->feature_discard = 0;
info->feature_secdiscard = 0;
queue_flag_clear(QUEUE_FLAG_DISCARD, rq);
queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq);
}
- blk_mq_complete_request(req);
+ blk_mq_complete_request(req, error);
break;
case BLKIF_OP_FLUSH_DISKCACHE:
case BLKIF_OP_WRITE_BARRIER:
if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
printk(KERN_WARNING "blkfront: %s: %s op failed\n",
info->gd->disk_name, op_name(bret->operation));
- req->errors = -EOPNOTSUPP;
+ error = -EOPNOTSUPP;
}
if (unlikely(bret->status == BLKIF_RSP_ERROR &&
info->shadow[id].req.u.rw.nr_segments == 0)) {
printk(KERN_WARNING "blkfront: %s: empty %s op failed\n",
info->gd->disk_name, op_name(bret->operation));
- req->errors = -EOPNOTSUPP;
+ error = -EOPNOTSUPP;
}
- if (unlikely(req->errors)) {
- if (req->errors == -EOPNOTSUPP)
- req->errors = 0;
+ if (unlikely(error)) {
+ if (error == -EOPNOTSUPP)
+ error = 0;
info->feature_flush = 0;
xlvbd_flush(info);
}
dev_dbg(&info->xbdev->dev, "Bad return from blkdev data "
"request: %x\n", bret->status);
- blk_mq_complete_request(req);
+ blk_mq_complete_request(req, error);
break;
default:
BUG();
* allocate new zcomp and initialize it. return compressing
* backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
* if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
- * case of allocation error.
+ * case of allocation error, or any other error potentially
+ * returned by functions zcomp_strm_{multi,single}_create.
*/
struct zcomp *zcomp_create(const char *compress, int max_strm)
{
struct zcomp *comp;
struct zcomp_backend *backend;
+ int error;
backend = find_backend(compress);
if (!backend)
comp->backend = backend;
if (max_strm > 1)
- zcomp_strm_multi_create(comp, max_strm);
+ error = zcomp_strm_multi_create(comp, max_strm);
else
- zcomp_strm_single_create(comp);
- if (!comp->stream) {
+ error = zcomp_strm_single_create(comp);
+ if (error) {
kfree(comp);
- return ERR_PTR(-ENOMEM);
+ return ERR_PTR(error);
}
return comp;
}
config ARM_CCI500_PMU
bool "ARM CCI500 PMU support"
- default y
depends on (ARM && CPU_V7) || ARM64
depends on PERF_EVENTS
select ARM_CCI_PMU
if (IS_ERR(ctx->csr_base))
return PTR_ERR(ctx->csr_base);
- ctx->irq = platform_get_irq(pdev, 0);
- if (ctx->irq < 0) {
+ rc = platform_get_irq(pdev, 0);
+ if (rc < 0) {
dev_err(&pdev->dev, "No IRQ resource\n");
- return ctx->irq;
+ return rc;
}
+ ctx->irq = rc;
dev_dbg(&pdev->dev, "APM X-Gene RNG BASE %p ALARM IRQ %d",
ctx->csr_base, ctx->irq);
hlist_for_each_entry_safe(orphan, tmp2, &clk_orphan_list, child_node) {
if (orphan->num_parents && orphan->ops->get_parent) {
i = orphan->ops->get_parent(orphan->hw);
- if (!strcmp(core->name, orphan->parent_names[i]))
+ if (i >= 0 && i < orphan->num_parents &&
+ !strcmp(core->name, orphan->parent_names[i]))
clk_core_reparent(orphan, core);
continue;
}
#include <linux/err.h>
#include <linux/device.h>
#include <linux/of_address.h>
+#include <linux/slab.h>
static DEFINE_SPINLOCK(clklock);
config COMMON_CLK_HI6220
bool "Hi6220 Clock Driver"
- depends on (ARCH_HISI || COMPILE_TEST) && MAILBOX
+ depends on ARCH_HISI || COMPILE_TEST
default ARCH_HISI
help
Build the Hisilicon Hi6220 clock driver based on the common clock framework.
+
+config STUB_CLK_HI6220
+ bool "Hi6220 Stub Clock Driver"
+ depends on COMMON_CLK_HI6220 && MAILBOX
+ help
+ Build the Hisilicon Hi6220 stub clock driver.
obj-$(CONFIG_ARCH_HI3xxx) += clk-hi3620.o
obj-$(CONFIG_ARCH_HIP04) += clk-hip04.o
obj-$(CONFIG_ARCH_HIX5HD2) += clk-hix5hd2.o
-obj-$(CONFIG_COMMON_CLK_HI6220) += clk-hi6220.o clk-hi6220-stub.o
+obj-$(CONFIG_COMMON_CLK_HI6220) += clk-hi6220.o
+obj-$(CONFIG_STUB_CLK_HI6220) += clk-hi6220-stub.o
"aclk_cpu",
"aclk_peri",
"hclk_peri",
+ "pclk_cpu",
+ "pclk_peri",
};
static void __init rk3188_common_clk_init(struct device_node *np)
rockchip_clk_register_branches(common_clk_branches,
ARRAY_SIZE(common_clk_branches));
- rockchip_clk_protect_critical(rk3188_critical_clocks,
- ARRAY_SIZE(rk3188_critical_clocks));
rockchip_register_softrst(np, 9, reg_base + RK2928_SOFTRST_CON(0),
ROCKCHIP_SOFTRST_HIWORD_MASK);
mux_armclk_p, ARRAY_SIZE(mux_armclk_p),
&rk3066_cpuclk_data, rk3066_cpuclk_rates,
ARRAY_SIZE(rk3066_cpuclk_rates));
+ rockchip_clk_protect_critical(rk3188_critical_clocks,
+ ARRAY_SIZE(rk3188_critical_clocks));
}
CLK_OF_DECLARE(rk3066a_cru, "rockchip,rk3066a-cru", rk3066a_clk_init);
pr_warn("%s: missing clocks to reparent aclk_cpu_pre to gpll\n",
__func__);
}
+
+ rockchip_clk_protect_critical(rk3188_critical_clocks,
+ ARRAY_SIZE(rk3188_critical_clocks));
}
CLK_OF_DECLARE(rk3188a_cru, "rockchip,rk3188a-cru", rk3188a_clk_init);
GATE(0, "sclk_timer00", "xin24m", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(24), 0, GFLAGS),
};
+static const char *const rk3368_critical_clocks[] __initconst = {
+ "pclk_pd_pmu",
+};
+
static void __init rk3368_clk_init(struct device_node *np)
{
void __iomem *reg_base;
RK3368_GRF_SOC_STATUS0);
rockchip_clk_register_branches(rk3368_clk_branches,
ARRAY_SIZE(rk3368_clk_branches));
+ rockchip_clk_protect_critical(rk3368_critical_clocks,
+ ARRAY_SIZE(rk3368_critical_clocks));
rockchip_clk_register_armclk(ARMCLKB, "armclkb",
mux_armclkb_p, ARRAY_SIZE(mux_armclkb_p),
* the values for DIV_COPY and DIV_HPM dividers need not be set.
*/
div0 = cfg_data->div0;
- if (test_bit(CLK_CPU_HAS_DIV1, &cpuclk->flags)) {
+ if (cpuclk->flags & CLK_CPU_HAS_DIV1) {
div1 = cfg_data->div1;
if (readl(base + E4210_SRC_CPU) & E4210_MUX_HPM_MASK)
div1 = readl(base + E4210_DIV_CPU1) &
alt_div = DIV_ROUND_UP(alt_prate, tmp_rate) - 1;
WARN_ON(alt_div >= MAX_DIV);
- if (test_bit(CLK_CPU_NEEDS_DEBUG_ALT_DIV, &cpuclk->flags)) {
+ if (cpuclk->flags & CLK_CPU_NEEDS_DEBUG_ALT_DIV) {
/*
* In Exynos4210, ATB clock parent is also mout_core. So
* ATB clock also needs to be mantained at safe speed.
writel(div0, base + E4210_DIV_CPU0);
wait_until_divider_stable(base + E4210_DIV_STAT_CPU0, DIV_MASK_ALL);
- if (test_bit(CLK_CPU_HAS_DIV1, &cpuclk->flags)) {
+ if (cpuclk->flags & CLK_CPU_HAS_DIV1) {
writel(div1, base + E4210_DIV_CPU1);
wait_until_divider_stable(base + E4210_DIV_STAT_CPU1,
DIV_MASK_ALL);
unsigned long mux_reg;
/* find out the divider values to use for clock data */
- if (test_bit(CLK_CPU_NEEDS_DEBUG_ALT_DIV, &cpuclk->flags)) {
+ if (cpuclk->flags & CLK_CPU_NEEDS_DEBUG_ALT_DIV) {
while ((cfg_data->prate * 1000) != ndata->new_rate) {
if (cfg_data->prate == 0)
return -EINVAL;
writel(mux_reg & ~(1 << 16), base + E4210_SRC_CPU);
wait_until_mux_stable(base + E4210_STAT_CPU, 16, 1);
- if (test_bit(CLK_CPU_NEEDS_DEBUG_ALT_DIV, &cpuclk->flags)) {
+ if (cpuclk->flags & CLK_CPU_NEEDS_DEBUG_ALT_DIV) {
div |= (cfg_data->div0 & E4210_DIV0_ATB_MASK);
div_mask |= E4210_DIV0_ATB_MASK;
}
.get_rate = clk_fs660c32_dig_get_rate,
};
-static const struct clkgen_quadfs_data st_fs660c32_C_407 = {
+static const struct clkgen_quadfs_data st_fs660c32_C = {
.nrst_present = true,
.nrst = { CLKGEN_FIELD(0x2f0, 0x1, 0),
CLKGEN_FIELD(0x2f0, 0x1, 1),
.get_rate = clk_fs660c32_dig_get_rate,
};
-static const struct clkgen_quadfs_data st_fs660c32_D_407 = {
+static const struct clkgen_quadfs_data st_fs660c32_D = {
.nrst_present = true,
.nrst = { CLKGEN_FIELD(0x2a0, 0x1, 0),
CLKGEN_FIELD(0x2a0, 0x1, 1),
},
{
.compatible = "st,stih407-quadfs660-C",
- .data = &st_fs660c32_C_407
+ .data = &st_fs660c32_C
},
{
.compatible = "st,stih407-quadfs660-D",
- .data = &st_fs660c32_D_407
+ .data = &st_fs660c32_D
},
{}
};
.ops = &stm_pll3200c32_ops,
};
-static const struct clkgen_pll_data st_pll3200c32_407_c0_0 = {
+static const struct clkgen_pll_data st_pll3200c32_cx_0 = {
/* 407 C0 PLL0 */
.pdn_status = CLKGEN_FIELD(0x2a0, 0x1, 8),
.locked_status = CLKGEN_FIELD(0x2a0, 0x1, 24),
.ops = &stm_pll3200c32_ops,
};
-static const struct clkgen_pll_data st_pll3200c32_407_c0_1 = {
+static const struct clkgen_pll_data st_pll3200c32_cx_1 = {
/* 407 C0 PLL1 */
.pdn_status = CLKGEN_FIELD(0x2c8, 0x1, 8),
.locked_status = CLKGEN_FIELD(0x2c8, 0x1, 24),
.data = &st_pll3200c32_407_a0,
},
{
- .compatible = "st,stih407-plls-c32-c0_0",
- .data = &st_pll3200c32_407_c0_0,
+ .compatible = "st,plls-c32-cx_0",
+ .data = &st_pll3200c32_cx_0,
},
{
- .compatible = "st,stih407-plls-c32-c0_1",
- .data = &st_pll3200c32_407_c0_1,
+ .compatible = "st,plls-c32-cx_1",
+ .data = &st_pll3200c32_cx_1,
},
{
.compatible = "st,stih407-plls-c32-a9",
struct dev_pm_opp *opp;
int i, uv;
+ rcu_read_lock();
+
opp = dev_pm_opp_find_freq_ceil(td->soc->dev, &rate);
- if (IS_ERR(opp))
+ if (IS_ERR(opp)) {
+ rcu_read_unlock();
return PTR_ERR(opp);
+ }
uv = dev_pm_opp_get_voltage(opp);
+ rcu_read_unlock();
+
for (i = 0; i < td->i2c_lut_size; i++) {
if (regulator_list_voltage(td->vdd_reg, td->i2c_lut[i]) == uv)
return i;
DT_CLK(NULL, "gpio2_ick", "gpio2_ick"),
DT_CLK(NULL, "wdt3_ick", "wdt3_ick"),
DT_CLK(NULL, "uart3_ick", "uart3_ick"),
- DT_CLK(NULL, "uart4_ick", "uart4_ick"),
DT_CLK(NULL, "gpt9_ick", "gpt9_ick"),
DT_CLK(NULL, "gpt8_ick", "gpt8_ick"),
DT_CLK(NULL, "gpt7_ick", "gpt7_ick"),
static struct ti_dt_clk omap36xx_clks[] = {
DT_CLK(NULL, "omap_192m_alwon_fck", "omap_192m_alwon_fck"),
DT_CLK(NULL, "uart4_fck", "uart4_fck"),
+ DT_CLK(NULL, "uart4_ick", "uart4_ick"),
{ .node_name = NULL },
};
#include "clock.h"
-#define DRA7_DPLL_ABE_DEFFREQ 180633600
#define DRA7_DPLL_GMAC_DEFFREQ 1000000000
#define DRA7_DPLL_USB_DEFFREQ 960000000
int __init dra7xx_dt_clk_init(void)
{
int rc;
- struct clk *abe_dpll_mux, *sys_clkin2, *dpll_ck, *hdcp_ck;
+ struct clk *dpll_ck, *hdcp_ck;
ti_dt_clocks_register(dra7xx_clks);
omap2_clk_disable_autoidle_all();
- abe_dpll_mux = clk_get_sys(NULL, "abe_dpll_sys_clk_mux");
- sys_clkin2 = clk_get_sys(NULL, "sys_clkin2");
- dpll_ck = clk_get_sys(NULL, "dpll_abe_ck");
-
- rc = clk_set_parent(abe_dpll_mux, sys_clkin2);
- if (!rc)
- rc = clk_set_rate(dpll_ck, DRA7_DPLL_ABE_DEFFREQ);
- if (rc)
- pr_err("%s: failed to configure ABE DPLL!\n", __func__);
-
- dpll_ck = clk_get_sys(NULL, "dpll_abe_m2x2_ck");
- rc = clk_set_rate(dpll_ck, DRA7_DPLL_ABE_DEFFREQ * 2);
- if (rc)
- pr_err("%s: failed to configure ABE DPLL m2x2!\n", __func__);
-
dpll_ck = clk_get_sys(NULL, "dpll_gmac_ck");
rc = clk_set_rate(dpll_ck, DRA7_DPLL_GMAC_DEFFREQ);
if (rc)
}
}
- if (unlikely(!clk->enable_reg)) {
+ if (unlikely(IS_ERR(clk->enable_reg))) {
pr_err("%s: %s missing enable_reg\n", __func__,
clk_hw_get_name(hw));
ret = -EINVAL;
u32 v;
clk = to_clk_hw_omap(hw);
- if (!clk->enable_reg) {
+ if (IS_ERR(clk->enable_reg)) {
/*
* 'independent' here refers to a clock which is not
* controlled by its parent.
bc_timer.freq = clk_get_rate(timer_clk);
irq = irq_of_parse_and_map(np, 0);
- if (irq == NO_IRQ) {
+ if (!irq) {
pr_err("Failed to map interrupts for '%s'\n", TIMER_NAME);
return;
}
int irq, error;
irq = irq_of_parse_and_map(np, 0);
- if (irq == NO_IRQ) {
+ if (!irq) {
pr_err("%s: failed to map interrupts\n", __func__);
return;
}
{
struct acpi_cpufreq_data *data = policy->driver_data;
+ if (unlikely(!data))
+ return -ENODEV;
+
return cpufreq_show_cpus(data->freqdomain_cpus, buf);
}
pr_debug("get_cur_freq_on_cpu (%d)\n", cpu);
- policy = cpufreq_cpu_get(cpu);
+ policy = cpufreq_cpu_get_raw(cpu);
if (unlikely(!policy))
return 0;
data = policy->driver_data;
- cpufreq_cpu_put(policy);
if (unlikely(!data || !data->freq_table))
return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_init);
-/* Only for cpufreq core internal use */
-static struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
+struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
{
struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
}
+EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
unsigned int cpufreq_generic_get(unsigned int cpu)
{
* since this is a core component, and is essential for the
* subsequent light-weight ->init() to succeed.
*/
- if (cpufreq_driver->exit)
+ if (cpufreq_driver->exit) {
cpufreq_driver->exit(policy);
+ policy->freq_table = NULL;
+ }
}
/**
config CRYPTO_DEV_VMX
bool "Support for VMX cryptographic acceleration instructions"
- depends on PPC64
+ depends on PPC64 && VSX
help
Support for VMX cryptographic acceleration instructions.
int mv_cesa_queue_req(struct crypto_async_request *req);
+/*
+ * Helper function that indicates whether a crypto request needs to be
+ * cleaned up or not after being enqueued using mv_cesa_queue_req().
+ */
+static inline int mv_cesa_req_needs_cleanup(struct crypto_async_request *req,
+ int ret)
+{
+ /*
+ * The queue still had some space, the request was queued
+ * normally, so there's no need to clean it up.
+ */
+ if (ret == -EINPROGRESS)
+ return false;
+
+ /*
+ * The queue had not space left, but since the request is
+ * flagged with CRYPTO_TFM_REQ_MAY_BACKLOG, it was added to
+ * the backlog and will be processed later. There's no need to
+ * clean it up.
+ */
+ if (ret == -EBUSY && req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
+ return false;
+
+ /* Request wasn't queued, we need to clean it up */
+ return true;
+}
+
/* TDMA functions */
static inline void mv_cesa_req_dma_iter_init(struct mv_cesa_dma_iter *iter,
{
struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req);
struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq);
-
creq->req.base.engine = engine;
if (creq->req.base.type == CESA_DMA_REQ)
return ret;
ret = mv_cesa_queue_req(&req->base);
- if (ret && ret != -EINPROGRESS)
+ if (mv_cesa_req_needs_cleanup(&req->base, ret))
mv_cesa_ablkcipher_cleanup(req);
return ret;
return ret;
ret = mv_cesa_queue_req(&req->base);
- if (ret && ret != -EINPROGRESS)
+ if (mv_cesa_req_needs_cleanup(&req->base, ret))
mv_cesa_ablkcipher_cleanup(req);
return ret;
return ret;
ret = mv_cesa_queue_req(&req->base);
- if (ret && ret != -EINPROGRESS)
+ if (mv_cesa_req_needs_cleanup(&req->base, ret))
mv_cesa_ablkcipher_cleanup(req);
return ret;
return 0;
ret = mv_cesa_queue_req(&req->base);
- if (ret && ret != -EINPROGRESS) {
+ if (mv_cesa_req_needs_cleanup(&req->base, ret))
mv_cesa_ahash_cleanup(req);
- return ret;
- }
return ret;
}
return 0;
ret = mv_cesa_queue_req(&req->base);
- if (ret && ret != -EINPROGRESS)
+ if (mv_cesa_req_needs_cleanup(&req->base, ret))
mv_cesa_ahash_cleanup(req);
return ret;
return 0;
ret = mv_cesa_queue_req(&req->base);
- if (ret && ret != -EINPROGRESS)
+ if (mv_cesa_req_needs_cleanup(&req->base, ret))
mv_cesa_ahash_cleanup(req);
return ret;
struct pci_dev *parent = pdev->bus->self;
uint16_t bridge_ctl = 0;
+ if (accel_dev->is_vf)
+ return;
+
dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n",
accel_dev->accel_id);
sg_miter_next(&mo);
oo = 0;
}
- } while (mo.length > 0);
+ } while (oleft > 0);
if (areq->info) {
for (i = 0; i < 4 && i < ivsize / 4; i++) {
{
struct devfreq *tmp_devfreq;
- if (unlikely(IS_ERR_OR_NULL(dev))) {
+ if (IS_ERR_OR_NULL(dev)) {
pr_err("DEVFREQ: %s: Invalid parameters\n", __func__);
return ERR_PTR(-EINVAL);
}
{
struct devfreq_governor *tmp_governor;
- if (unlikely(IS_ERR_OR_NULL(name))) {
+ if (IS_ERR_OR_NULL(name)) {
pr_err("DEVFREQ: %s: Invalid parameters\n", __func__);
return ERR_PTR(-EINVAL);
}
return err;
/*
- * Adjust the freuqency with user freq and QoS.
+ * Adjust the frequency with user freq and QoS.
*
- * List from the highest proiority
- * max_freq (probably called by thermal when it's too hot)
+ * List from the highest priority
+ * max_freq
* min_freq
*/
devfreq->profile->max_state *
devfreq->profile->max_state,
GFP_KERNEL);
- devfreq->time_in_state = devm_kzalloc(dev, sizeof(unsigned int) *
+ devfreq->time_in_state = devm_kzalloc(dev, sizeof(unsigned long) *
devfreq->profile->max_state,
GFP_KERNEL);
devfreq->last_stat_updated = jiffies;
if (err) {
put_device(&devfreq->dev);
mutex_unlock(&devfreq->lock);
- goto err_dev;
+ goto err_out;
}
mutex_unlock(&devfreq->lock);
err_init:
list_del(&devfreq->node);
device_unregister(&devfreq->dev);
-err_dev:
kfree(devfreq);
err_out:
return ERR_PTR(err);
ret = PTR_ERR(governor);
goto out;
}
- if (df->governor == governor)
+ if (df->governor == governor) {
+ ret = 0;
goto out;
+ }
if (df->governor) {
ret = df->governor->event_handler(df, DEVFREQ_GOV_STOP, NULL);
case PPMU_PMNCNT3:
pmcnt_high = __raw_readl(info->ppmu.base + PPMU_V2_PMCNT3_HIGH);
pmcnt_low = __raw_readl(info->ppmu.base + PPMU_V2_PMCNT3_LOW);
- load_count = (u64)((pmcnt_high & 0xff) << 32) + (u64)pmcnt_low;
+ load_count = ((u64)((pmcnt_high & 0xff)) << 32)
+ + (u64)pmcnt_low;
break;
}
edata->load_count = load_count;
static int devfreq_simple_ondemand_func(struct devfreq *df,
unsigned long *freq)
{
- struct devfreq_dev_status stat;
- int err = df->profile->get_dev_status(df->dev.parent, &stat);
+ int err;
+ struct devfreq_dev_status *stat;
unsigned long long a, b;
unsigned int dfso_upthreshold = DFSO_UPTHRESHOLD;
unsigned int dfso_downdifferential = DFSO_DOWNDIFFERENCTIAL;
struct devfreq_simple_ondemand_data *data = df->data;
unsigned long max = (df->max_freq) ? df->max_freq : UINT_MAX;
+ err = devfreq_update_stats(df);
if (err)
return err;
+ stat = &df->last_status;
+
if (data) {
if (data->upthreshold)
dfso_upthreshold = data->upthreshold;
return -EINVAL;
/* Assume MAX if it is going to be divided by zero */
- if (stat.total_time == 0) {
+ if (stat->total_time == 0) {
*freq = max;
return 0;
}
/* Prevent overflow */
- if (stat.busy_time >= (1 << 24) || stat.total_time >= (1 << 24)) {
- stat.busy_time >>= 7;
- stat.total_time >>= 7;
+ if (stat->busy_time >= (1 << 24) || stat->total_time >= (1 << 24)) {
+ stat->busy_time >>= 7;
+ stat->total_time >>= 7;
}
/* Set MAX if it's busy enough */
- if (stat.busy_time * 100 >
- stat.total_time * dfso_upthreshold) {
+ if (stat->busy_time * 100 >
+ stat->total_time * dfso_upthreshold) {
*freq = max;
return 0;
}
/* Set MAX if we do not know the initial frequency */
- if (stat.current_frequency == 0) {
+ if (stat->current_frequency == 0) {
*freq = max;
return 0;
}
/* Keep the current frequency */
- if (stat.busy_time * 100 >
- stat.total_time * (dfso_upthreshold - dfso_downdifferential)) {
- *freq = stat.current_frequency;
+ if (stat->busy_time * 100 >
+ stat->total_time * (dfso_upthreshold - dfso_downdifferential)) {
+ *freq = stat->current_frequency;
return 0;
}
/* Set the desired frequency based on the load */
- a = stat.busy_time;
- a *= stat.current_frequency;
- b = div_u64(a, stat.total_time);
+ a = stat->busy_time;
+ a *= stat->current_frequency;
+ b = div_u64(a, stat->total_time);
b *= 100;
b = div_u64(b, (dfso_upthreshold - dfso_downdifferential / 2));
*freq = (unsigned long) b;
static int tegra_governor_get_target(struct devfreq *devfreq,
unsigned long *freq)
{
- struct devfreq_dev_status stat;
+ struct devfreq_dev_status *stat;
struct tegra_devfreq *tegra;
struct tegra_devfreq_device *dev;
unsigned long target_freq = 0;
unsigned int i;
int err;
- err = devfreq->profile->get_dev_status(devfreq->dev.parent, &stat);
+ err = devfreq_update_stats(devfreq);
if (err)
return err;
- tegra = stat.private_data;
+ stat = &devfreq->last_status;
+
+ tegra = stat->private_data;
for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
dev = &tegra->devices[i];
}
EXPORT_SYMBOL(fence_default_wait);
+static bool
+fence_test_signaled_any(struct fence **fences, uint32_t count)
+{
+ int i;
+
+ for (i = 0; i < count; ++i) {
+ struct fence *fence = fences[i];
+ if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+ return true;
+ }
+ return false;
+}
+
+/**
+ * fence_wait_any_timeout - sleep until any fence gets signaled
+ * or until timeout elapses
+ * @fences: [in] array of fences to wait on
+ * @count: [in] number of fences to wait on
+ * @intr: [in] if true, do an interruptible wait
+ * @timeout: [in] timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
+ *
+ * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
+ * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
+ * on success.
+ *
+ * Synchronous waits for the first fence in the array to be signaled. The
+ * caller needs to hold a reference to all fences in the array, otherwise a
+ * fence might be freed before return, resulting in undefined behavior.
+ */
+signed long
+fence_wait_any_timeout(struct fence **fences, uint32_t count,
+ bool intr, signed long timeout)
+{
+ struct default_wait_cb *cb;
+ signed long ret = timeout;
+ unsigned i;
+
+ if (WARN_ON(!fences || !count || timeout < 0))
+ return -EINVAL;
+
+ if (timeout == 0) {
+ for (i = 0; i < count; ++i)
+ if (fence_is_signaled(fences[i]))
+ return 1;
+
+ return 0;
+ }
+
+ cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
+ if (cb == NULL) {
+ ret = -ENOMEM;
+ goto err_free_cb;
+ }
+
+ for (i = 0; i < count; ++i) {
+ struct fence *fence = fences[i];
+
+ if (fence->ops->wait != fence_default_wait) {
+ ret = -EINVAL;
+ goto fence_rm_cb;
+ }
+
+ cb[i].task = current;
+ if (fence_add_callback(fence, &cb[i].base,
+ fence_default_wait_cb)) {
+ /* This fence is already signaled */
+ goto fence_rm_cb;
+ }
+ }
+
+ while (ret > 0) {
+ if (intr)
+ set_current_state(TASK_INTERRUPTIBLE);
+ else
+ set_current_state(TASK_UNINTERRUPTIBLE);
+
+ if (fence_test_signaled_any(fences, count))
+ break;
+
+ ret = schedule_timeout(ret);
+
+ if (ret > 0 && intr && signal_pending(current))
+ ret = -ERESTARTSYS;
+ }
+
+ __set_current_state(TASK_RUNNING);
+
+fence_rm_cb:
+ while (i-- > 0)
+ fence_remove_callback(fences[i], &cb[i].base);
+
+err_free_cb:
+ kfree(cb);
+
+ return ret;
+}
+EXPORT_SYMBOL(fence_wait_any_timeout);
+
/**
* fence_init - Initialize a custom fence.
* @fence: [in] the fence to initialize
return desc;
}
+void at_xdmac_init_used_desc(struct at_xdmac_desc *desc)
+{
+ memset(&desc->lld, 0, sizeof(desc->lld));
+ INIT_LIST_HEAD(&desc->descs_list);
+ desc->direction = DMA_TRANS_NONE;
+ desc->xfer_size = 0;
+ desc->active_xfer = false;
+}
+
/* Call must be protected by lock. */
static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)
{
desc = list_first_entry(&atchan->free_descs_list,
struct at_xdmac_desc, desc_node);
list_del(&desc->desc_node);
- desc->active_xfer = false;
+ at_xdmac_init_used_desc(desc);
}
return desc;
if (xt->src_inc) {
if (xt->src_sgl)
- chan_cc |= AT_XDMAC_CC_SAM_UBS_DS_AM;
+ chan_cc |= AT_XDMAC_CC_SAM_UBS_AM;
else
chan_cc |= AT_XDMAC_CC_SAM_INCREMENTED_AM;
}
if (xt->dst_inc) {
if (xt->dst_sgl)
- chan_cc |= AT_XDMAC_CC_DAM_UBS_DS_AM;
+ chan_cc |= AT_XDMAC_CC_DAM_UBS_AM;
else
chan_cc |= AT_XDMAC_CC_DAM_INCREMENTED_AM;
}
mutex_lock(&dma_list_mutex);
if (chan->client_count == 0) {
+ struct dma_device *device = chan->device;
+
+ dma_cap_set(DMA_PRIVATE, device->cap_mask);
+ device->privatecnt++;
err = dma_chan_get(chan);
- if (err)
+ if (err) {
pr_debug("%s: failed to get %s: (%d)\n",
__func__, dma_chan_name(chan), err);
+ chan = NULL;
+ if (--device->privatecnt == 0)
+ dma_cap_clear(DMA_PRIVATE, device->cap_mask);
+ }
} else
chan = NULL;
INIT_LIST_HEAD(&dw->dma.channels);
for (i = 0; i < nr_channels; i++) {
struct dw_dma_chan *dwc = &dw->chan[i];
- int r = nr_channels - i - 1;
dwc->chan.device = &dw->dma;
dma_cookie_init(&dwc->chan);
/* 7 is highest priority & 0 is lowest. */
if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
- dwc->priority = r;
+ dwc->priority = nr_channels - i - 1;
else
dwc->priority = i;
/* Hardware configuration */
if (autocfg) {
unsigned int dwc_params;
+ unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1;
void __iomem *addr = chip->regs + r * sizeof(u32);
dwc_params = dma_read_byaddr(addr, DWC_PARAMS);
struct idma64_desc *desc = idma64c->desc;
struct idma64_hw_desc *hw;
size_t bytes = desc->length;
- u64 llp;
- u32 ctlhi;
+ u64 llp = channel_readq(idma64c, LLP);
+ u32 ctlhi = channel_readl(idma64c, CTL_HI);
unsigned int i = 0;
- llp = channel_readq(idma64c, LLP);
do {
hw = &desc->hw[i];
- } while ((hw->llp != llp) && (++i < desc->ndesc));
+ if (hw->llp == llp)
+ break;
+ bytes -= hw->len;
+ } while (++i < desc->ndesc);
if (!i)
return bytes;
- do {
- bytes -= desc->hw[--i].len;
- } while (i);
+ /* The current chunk is not fully transfered yet */
+ bytes += desc->hw[--i].len;
- ctlhi = channel_readl(idma64c, CTL_HI);
return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi);
}
}
/* Chained IRQ handler for IPU function and error interrupt */
-static void ipu_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void ipu_irq_handler(struct irq_desc *desc)
{
struct ipu *ipu = irq_desc_get_handler_data(desc);
u32 status;
return;
/* clear the channel mapping in DRCMR */
- reg = pxad_drcmr(chan->drcmr);
- writel_relaxed(0, chan->phy->base + reg);
+ if (chan->drcmr <= DRCMR_CHLNUM) {
+ reg = pxad_drcmr(chan->drcmr);
+ writel_relaxed(0, chan->phy->base + reg);
+ }
spin_lock_irqsave(&pdev->phy_lock, flags);
for (i = 0; i < 32; i++)
"%s(); phy=%p(%d) misaligned=%d\n", __func__,
phy, phy->idx, misaligned);
- reg = pxad_drcmr(phy->vchan->drcmr);
- writel_relaxed(DRCMR_MAPVLD | phy->idx, phy->base + reg);
+ if (phy->vchan->drcmr <= DRCMR_CHLNUM) {
+ reg = pxad_drcmr(phy->vchan->drcmr);
+ writel_relaxed(DRCMR_MAPVLD | phy->idx, phy->base + reg);
+ }
dalgn = phy_readl_relaxed(phy, DALGN);
if (misaligned)
struct dma_async_tx_descriptor *tx;
struct pxad_chan *chan = container_of(vc, struct pxad_chan, vc);
+ INIT_LIST_HEAD(&vd->node);
tx = vchan_tx_prep(vc, vd, tx_flags);
tx->tx_submit = pxad_tx_submit;
dev_dbg(&chan->vc.chan.dev->device,
width = chan->cfg.src_addr_width;
dev_addr = chan->cfg.src_addr;
*dev_src = dev_addr;
- *dcmd |= PXA_DCMD_INCTRGADDR | PXA_DCMD_FLOWSRC;
+ *dcmd |= PXA_DCMD_INCTRGADDR;
+ if (chan->drcmr <= DRCMR_CHLNUM)
+ *dcmd |= PXA_DCMD_FLOWSRC;
}
if (dir == DMA_MEM_TO_DEV) {
maxburst = chan->cfg.dst_maxburst;
width = chan->cfg.dst_addr_width;
dev_addr = chan->cfg.dst_addr;
*dev_dst = dev_addr;
- *dcmd |= PXA_DCMD_INCSRCADDR | PXA_DCMD_FLOWTRG;
+ *dcmd |= PXA_DCMD_INCSRCADDR;
+ if (chan->drcmr <= DRCMR_CHLNUM)
+ *dcmd |= PXA_DCMD_FLOWTRG;
}
if (dir == DMA_MEM_TO_MEM)
*dcmd |= PXA_DCMD_BURST32 | PXA_DCMD_INCTRGADDR |
else
curr = phy_readl_relaxed(chan->phy, DTADR);
+ /*
+ * curr has to be actually read before checking descriptor
+ * completion, so that a curr inside a status updater
+ * descriptor implies the following test returns true, and
+ * preventing reordering of curr load and the test.
+ */
+ rmb();
+ if (is_desc_completed(vd))
+ goto out;
+
for (i = 0; i < sw_desc->nb_desc - 1; i++) {
hw_desc = sw_desc->hw_desc[i];
if (sw_desc->hw_desc[0]->dcmd & PXA_DCMD_INCSRCADDR)
static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
{
struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
- struct sun4i_dma_promise *promise;
+ struct sun4i_dma_promise *promise, *tmp;
/* Free all the demands and completed demands */
- list_for_each_entry(promise, &contract->demands, list)
+ list_for_each_entry_safe(promise, tmp, &contract->demands, list)
kfree(promise);
- list_for_each_entry(promise, &contract->completed_demands, list)
+ list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
kfree(promise);
kfree(contract);
#define XGENE_DMA_RING_MEM_RAM_SHUTDOWN 0xD070
#define XGENE_DMA_RING_BLK_MEM_RDY 0xD074
#define XGENE_DMA_RING_BLK_MEM_RDY_VAL 0xFFFFFFFF
-#define XGENE_DMA_RING_DESC_CNT(v) (((v) & 0x0001FFFE) >> 1)
#define XGENE_DMA_RING_ID_GET(owner, num) (((owner) << 6) | (num))
#define XGENE_DMA_RING_DST_ID(v) ((1 << 10) | (v))
#define XGENE_DMA_RING_CMD_OFFSET 0x2C
return flyby_type[src_cnt];
}
-static u32 xgene_dma_ring_desc_cnt(struct xgene_dma_ring *ring)
-{
- u32 __iomem *cmd_base = ring->cmd_base;
- u32 ring_state = ioread32(&cmd_base[1]);
-
- return XGENE_DMA_RING_DESC_CNT(ring_state);
-}
-
static void xgene_dma_set_src_buffer(__le64 *ext8, size_t *len,
dma_addr_t *paddr)
{
dma_pool_free(chan->desc_pool, desc, desc->tx.phys);
}
-static int xgene_chan_xfer_request(struct xgene_dma_ring *ring,
- struct xgene_dma_desc_sw *desc_sw)
+static void xgene_chan_xfer_request(struct xgene_dma_chan *chan,
+ struct xgene_dma_desc_sw *desc_sw)
{
+ struct xgene_dma_ring *ring = &chan->tx_ring;
struct xgene_dma_desc_hw *desc_hw;
- /* Check if can push more descriptor to hw for execution */
- if (xgene_dma_ring_desc_cnt(ring) > (ring->slots - 2))
- return -EBUSY;
-
/* Get hw descriptor from DMA tx ring */
desc_hw = &ring->desc_hw[ring->head];
memcpy(desc_hw, &desc_sw->desc2, sizeof(*desc_hw));
}
+ /* Increment the pending transaction count */
+ chan->pending += ((desc_sw->flags &
+ XGENE_DMA_FLAG_64B_DESC) ? 2 : 1);
+
/* Notify the hw that we have descriptor ready for execution */
iowrite32((desc_sw->flags & XGENE_DMA_FLAG_64B_DESC) ?
2 : 1, ring->cmd);
-
- return 0;
}
/**
static void xgene_chan_xfer_ld_pending(struct xgene_dma_chan *chan)
{
struct xgene_dma_desc_sw *desc_sw, *_desc_sw;
- int ret;
/*
* If the list of pending descriptors is empty, then we
if (chan->pending >= chan->max_outstanding)
return;
- ret = xgene_chan_xfer_request(&chan->tx_ring, desc_sw);
- if (ret)
- return;
+ xgene_chan_xfer_request(chan, desc_sw);
/*
* Delete this element from ld pending queue and append it to
* ld running queue
*/
list_move_tail(&desc_sw->node, &chan->ld_running);
-
- /* Increment the pending transaction count */
- chan->pending++;
}
}
* Decrement the pending transaction count
* as we have processed one
*/
- chan->pending--;
+ chan->pending -= ((desc_sw->flags &
+ XGENE_DMA_FLAG_64B_DESC) ? 2 : 1);
/*
* Delete this node from ld running queue and append it to
struct xgene_dma_ring *ring,
enum xgene_dma_ring_cfgsize cfgsize)
{
+ int ret;
+
/* Setup DMA ring descriptor variables */
ring->pdma = chan->pdma;
ring->cfgsize = cfgsize;
ring->num = chan->pdma->ring_num++;
ring->id = XGENE_DMA_RING_ID_GET(ring->owner, ring->buf_num);
- ring->size = xgene_dma_get_ring_size(chan, cfgsize);
- if (ring->size <= 0)
- return ring->size;
+ ret = xgene_dma_get_ring_size(chan, cfgsize);
+ if (ret <= 0)
+ return ret;
+ ring->size = ret;
/* Allocate memory for DMA ring descriptor */
ring->desc_vaddr = dma_zalloc_coherent(chan->dev, ring->size,
tx_ring->id, tx_ring->num, tx_ring->desc_vaddr);
/* Set the max outstanding request possible to this channel */
- chan->max_outstanding = rx_ring->slots;
+ chan->max_outstanding = tx_ring->slots;
return ret;
}
struct dma_chan *chan;
struct zx_dma_chan *c;
- if (request > d->dma_requests)
+ if (request >= d->dma_requests)
return NULL;
chan = dma_get_any_slave_channel(&d->slave);
static bool is_extcon_changed(u32 prev, u32 new, int idx, bool *attached)
{
if (((prev >> idx) & 0x1) != ((new >> idx) & 0x1)) {
- *attached = new ? true : false;
+ *attached = ((new >> idx) & 0x1) ? true : false;
return true;
}
bool
depends on ARM || ARM64
+config QCOM_SCM_32
+ def_bool y
+ depends on QCOM_SCM && ARM
+
+config QCOM_SCM_64
+ def_bool y
+ depends on QCOM_SCM && ARM64
+
source "drivers/firmware/broadcom/Kconfig"
source "drivers/firmware/google/Kconfig"
source "drivers/firmware/efi/Kconfig"
obj-$(CONFIG_ISCSI_IBFT) += iscsi_ibft.o
obj-$(CONFIG_FIRMWARE_MEMMAP) += memmap.o
obj-$(CONFIG_QCOM_SCM) += qcom_scm.o
-obj-$(CONFIG_QCOM_SCM) += qcom_scm-32.o
+obj-$(CONFIG_QCOM_SCM_64) += qcom_scm-64.o
+obj-$(CONFIG_QCOM_SCM_32) += qcom_scm-32.o
CFLAGS_qcom_scm-32.o :=$(call as-instr,.arch_extension sec,-DREQUIRES_SEC=1)
obj-y += broadcom/
*/
#include <linux/efi.h>
+#include <linux/sort.h>
#include <asm/efi.h>
#include "efistub.h"
*/
#define EFI_RT_VIRTUAL_BASE 0x40000000
+static int cmp_mem_desc(const void *l, const void *r)
+{
+ const efi_memory_desc_t *left = l, *right = r;
+
+ return (left->phys_addr > right->phys_addr) ? 1 : -1;
+}
+
+/*
+ * Returns whether region @left ends exactly where region @right starts,
+ * or false if either argument is NULL.
+ */
+static bool regions_are_adjacent(efi_memory_desc_t *left,
+ efi_memory_desc_t *right)
+{
+ u64 left_end;
+
+ if (left == NULL || right == NULL)
+ return false;
+
+ left_end = left->phys_addr + left->num_pages * EFI_PAGE_SIZE;
+
+ return left_end == right->phys_addr;
+}
+
+/*
+ * Returns whether region @left and region @right have compatible memory type
+ * mapping attributes, and are both EFI_MEMORY_RUNTIME regions.
+ */
+static bool regions_have_compatible_memory_type_attrs(efi_memory_desc_t *left,
+ efi_memory_desc_t *right)
+{
+ static const u64 mem_type_mask = EFI_MEMORY_WB | EFI_MEMORY_WT |
+ EFI_MEMORY_WC | EFI_MEMORY_UC |
+ EFI_MEMORY_RUNTIME;
+
+ return ((left->attribute ^ right->attribute) & mem_type_mask) == 0;
+}
+
/*
* efi_get_virtmap() - create a virtual mapping for the EFI memory map
*
int *count)
{
u64 efi_virt_base = EFI_RT_VIRTUAL_BASE;
- efi_memory_desc_t *out = runtime_map;
+ efi_memory_desc_t *in, *prev = NULL, *out = runtime_map;
int l;
- for (l = 0; l < map_size; l += desc_size) {
- efi_memory_desc_t *in = (void *)memory_map + l;
+ /*
+ * To work around potential issues with the Properties Table feature
+ * introduced in UEFI 2.5, which may split PE/COFF executable images
+ * in memory into several RuntimeServicesCode and RuntimeServicesData
+ * regions, we need to preserve the relative offsets between adjacent
+ * EFI_MEMORY_RUNTIME regions with the same memory type attributes.
+ * The easiest way to find adjacent regions is to sort the memory map
+ * before traversing it.
+ */
+ sort(memory_map, map_size / desc_size, desc_size, cmp_mem_desc, NULL);
+
+ for (l = 0; l < map_size; l += desc_size, prev = in) {
u64 paddr, size;
+ in = (void *)memory_map + l;
if (!(in->attribute & EFI_MEMORY_RUNTIME))
continue;
+ paddr = in->phys_addr;
+ size = in->num_pages * EFI_PAGE_SIZE;
+
/*
* Make the mapping compatible with 64k pages: this allows
* a 4k page size kernel to kexec a 64k page size kernel and
* vice versa.
*/
- paddr = round_down(in->phys_addr, SZ_64K);
- size = round_up(in->num_pages * EFI_PAGE_SIZE +
- in->phys_addr - paddr, SZ_64K);
-
- /*
- * Avoid wasting memory on PTEs by choosing a virtual base that
- * is compatible with section mappings if this region has the
- * appropriate size and physical alignment. (Sections are 2 MB
- * on 4k granule kernels)
- */
- if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M)
- efi_virt_base = round_up(efi_virt_base, SZ_2M);
+ if (!regions_are_adjacent(prev, in) ||
+ !regions_have_compatible_memory_type_attrs(prev, in)) {
+
+ paddr = round_down(in->phys_addr, SZ_64K);
+ size += in->phys_addr - paddr;
+
+ /*
+ * Avoid wasting memory on PTEs by choosing a virtual
+ * base that is compatible with section mappings if this
+ * region has the appropriate size and physical
+ * alignment. (Sections are 2 MB on 4k granule kernels)
+ */
+ if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M)
+ efi_virt_base = round_up(efi_virt_base, SZ_2M);
+ else
+ efi_virt_base = round_up(efi_virt_base, SZ_64K);
+ }
in->virt_addr = efi_virt_base + in->phys_addr - paddr;
efi_virt_base += size;
/* error code which can't be mistaken for valid address */
#define EFI_ERROR (~0UL)
-#undef memcpy
-#undef memset
-#undef memmove
-
void efi_char16_printk(efi_system_table_t *, efi_char16_t *);
efi_status_t efi_open_volume(efi_system_table_t *sys_table_arg, void *__image,
--- /dev/null
+/* Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/io.h>
+#include <linux/errno.h>
+#include <linux/qcom_scm.h>
+
+/**
+ * qcom_scm_set_cold_boot_addr() - Set the cold boot address for cpus
+ * @entry: Entry point function for the cpus
+ * @cpus: The cpumask of cpus that will use the entry point
+ *
+ * Set the cold boot address of the cpus. Any cpu outside the supported
+ * range would be removed from the cpu present mask.
+ */
+int __qcom_scm_set_cold_boot_addr(void *entry, const cpumask_t *cpus)
+{
+ return -ENOTSUPP;
+}
+
+/**
+ * qcom_scm_set_warm_boot_addr() - Set the warm boot address for cpus
+ * @entry: Entry point function for the cpus
+ * @cpus: The cpumask of cpus that will use the entry point
+ *
+ * Set the Linux entry point for the SCM to transfer control to when coming
+ * out of a power down. CPU power down may be executed on cpuidle or hotplug.
+ */
+int __qcom_scm_set_warm_boot_addr(void *entry, const cpumask_t *cpus)
+{
+ return -ENOTSUPP;
+}
+
+/**
+ * qcom_scm_cpu_power_down() - Power down the cpu
+ * @flags - Flags to flush cache
+ *
+ * This is an end point to power down cpu. If there was a pending interrupt,
+ * the control would return from this function, otherwise, the cpu jumps to the
+ * warm boot entry point set for this cpu upon reset.
+ */
+void __qcom_scm_cpu_power_down(u32 flags)
+{
+}
+
+int __qcom_scm_is_call_available(u32 svc_id, u32 cmd_id)
+{
+ return -ENOTSUPP;
+}
+
+int __qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt, u32 *resp)
+{
+ return -ENOTSUPP;
+}
config GPIO_RCAR
tristate "Renesas R-Car GPIO"
- depends on ARM && (ARCH_SHMOBILE || COMPILE_TEST)
+ depends on ARCH_SHMOBILE || COMPILE_TEST
select GPIOLIB_IRQCHIP
help
Say yes here to support GPIO on Renesas R-Car SoCs.
return 0;
}
-static void altera_gpio_irq_edge_handler(unsigned int irq,
- struct irq_desc *desc)
+static void altera_gpio_irq_edge_handler(struct irq_desc *desc)
{
struct altera_gpio_chip *altera_gc;
struct irq_chip *chip;
}
-static void altera_gpio_irq_leveL_high_handler(unsigned int irq,
- struct irq_desc *desc)
+static void altera_gpio_irq_leveL_high_handler(struct irq_desc *desc)
{
struct altera_gpio_chip *altera_gc;
struct irq_chip *chip;
return 0;
}
-static void bcm_kona_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void bcm_kona_gpio_irq_handler(struct irq_desc *desc)
{
void __iomem *reg_base;
int bit, bank_id;
}
/* Each UPG GIO block has one IRQ for all banks */
-static void brcmstb_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void brcmstb_gpio_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct brcmstb_gpio_priv *priv = brcmstb_gpio_gc_to_priv(gc);
.flags = IRQCHIP_SET_TYPE_MASKED,
};
-static void
-gpio_irq_handler(unsigned __irq, struct irq_desc *desc)
+static void gpio_irq_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct davinci_gpio_regs __iomem *g;
return ret;
}
-static void dwapb_irq_handler(u32 irq, struct irq_desc *desc)
+static void dwapb_irq_handler(struct irq_desc *desc)
{
struct dwapb_gpio *gpio = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
EP93XX_GPIO_REG(int_debounce_register_offset[port]));
}
-static void ep93xx_gpio_ab_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ep93xx_gpio_ab_irq_handler(struct irq_desc *desc)
{
unsigned char status;
int i;
}
}
-static void ep93xx_gpio_f_irq_handler(unsigned int __irq,
- struct irq_desc *desc)
+static void ep93xx_gpio_f_irq_handler(struct irq_desc *desc)
{
/*
* map discontiguous hw irq range to continuous sw irq range:
};
MODULE_DEVICE_TABLE(pci, intel_gpio_ids);
-static void intel_mid_irq_handler(unsigned irq, struct irq_desc *desc)
+static void intel_mid_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct intel_mid_gpio *priv = to_intel_gpio_priv(gc);
return 0;
}
-static void lp_gpio_irq_handler(unsigned hwirq, struct irq_desc *desc)
+static void lp_gpio_irq_handler(struct irq_desc *desc)
{
struct irq_data *data = irq_desc_get_irq_data(desc);
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
return -ENXIO;
}
-static void mpc8xxx_gpio_irq_cascade(unsigned int irq, struct irq_desc *desc)
+static void mpc8xxx_gpio_irq_cascade(struct irq_desc *desc)
{
struct mpc8xxx_gpio_chip *mpc8xxx_gc = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
.irq_bus_sync_unlock = msic_bus_sync_unlock,
};
-static void msic_gpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void msic_gpio_irq_handler(struct irq_desc *desc)
{
struct irq_data *data = irq_desc_get_irq_data(desc);
struct msic_gpio *mg = irq_data_get_irq_handler_data(data);
* which have been set as summary IRQ lines and which are triggered,
* and to call their interrupt handlers.
*/
-static void msm_summary_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void msm_summary_irq_handler(struct irq_desc *desc)
{
unsigned long i;
struct irq_chip *chip = irq_desc_get_chip(desc);
return 0;
}
-static void mvebu_gpio_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void mvebu_gpio_irq_handler(struct irq_desc *desc)
{
struct mvebu_gpio_chip *mvchip = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
}
/* MX1 and MX3 has one interrupt *per* gpio port */
-static void mx3_gpio_irq_handler(u32 irq, struct irq_desc *desc)
+static void mx3_gpio_irq_handler(struct irq_desc *desc)
{
u32 irq_stat;
struct mxc_gpio_port *port = irq_desc_get_handler_data(desc);
}
/* MX2 has one interrupt *for all* gpio ports */
-static void mx2_gpio_irq_handler(u32 irq, struct irq_desc *desc)
+static void mx2_gpio_irq_handler(struct irq_desc *desc)
{
u32 irq_msk, irq_stat;
struct mxc_gpio_port *port;
return 0;
}
-static void mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base)
+static int mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base)
{
struct irq_chip_generic *gc;
struct irq_chip_type *ct;
gc = irq_alloc_generic_chip("gpio-mxc", 1, irq_base,
port->base, handle_level_irq);
+ if (!gc)
+ return -ENOMEM;
gc->private = port;
ct = gc->chip_types;
irq_setup_generic_chip(gc, IRQ_MSK(32), IRQ_GC_INIT_NESTED_LOCK,
IRQ_NOREQUEST, 0);
+
+ return 0;
}
static void mxc_gpio_get_hw(struct platform_device *pdev)
}
/* gpio-mxc can be a generic irq chip */
- mxc_gpio_init_gc(port, irq_base);
+ err = mxc_gpio_init_gc(port, irq_base);
+ if (err < 0)
+ goto out_irqdomain_remove;
list_add_tail(&port->node, &mxc_gpio_ports);
return 0;
+out_irqdomain_remove:
+ irq_domain_remove(port->domain);
out_irqdesc_free:
irq_free_descs(irq_base, 32);
out_gpiochip_remove:
}
/* MXS has one interrupt *per* gpio port */
-static void mxs_gpio_irq_handler(u32 irq, struct irq_desc *desc)
+static void mxs_gpio_irq_handler(struct irq_desc *desc)
{
u32 irq_stat;
struct mxs_gpio_port *port = irq_desc_get_handler_data(desc);
return 0;
}
-static void __init mxs_gpio_init_gc(struct mxs_gpio_port *port, int irq_base)
+static int __init mxs_gpio_init_gc(struct mxs_gpio_port *port, int irq_base)
{
struct irq_chip_generic *gc;
struct irq_chip_type *ct;
gc = irq_alloc_generic_chip("gpio-mxs", 1, irq_base,
port->base, handle_level_irq);
+ if (!gc)
+ return -ENOMEM;
+
gc->private = port;
ct = gc->chip_types;
irq_setup_generic_chip(gc, IRQ_MSK(32), IRQ_GC_INIT_NESTED_LOCK,
IRQ_NOREQUEST, 0);
+
+ return 0;
}
static int mxs_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
}
/* gpio-mxs can be a generic irq chip */
- mxs_gpio_init_gc(port, irq_base);
+ err = mxs_gpio_init_gc(port, irq_base);
+ if (err < 0)
+ goto out_irqdomain_remove;
/* setup one handler for each entry */
irq_set_chained_handler_and_data(port->irq, mxs_gpio_irq_handler,
out_bgpio_remove:
bgpio_remove(&port->bgc);
+out_irqdomain_remove:
+ irq_domain_remove(port->domain);
out_irqdesc_free:
irq_free_descs(irq_base, 32);
return err;
* line's interrupt handler has been run, we may miss some nested
* interrupts.
*/
-static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void omap_gpio_irq_handler(struct irq_desc *desc)
{
void __iomem *isr_reg = NULL;
u32 isr;
} else {
bank->chip.label = "gpio";
bank->chip.base = gpio;
- gpio += bank->width;
}
bank->chip.ngpio = bank->width;
return ret;
}
+ if (!bank->is_mpuio)
+ gpio += bank->width;
+
#ifdef CONFIG_ARCH_OMAP1
/*
* REVISIT: Once we have OMAP1 supporting SPARSE_IRQ, we can drop
omap_gpio_mod_init(bank);
ret = omap_gpio_chip_init(bank, irqc);
- if (ret)
+ if (ret) {
+ pm_runtime_put_sync(bank->dev);
+ pm_runtime_disable(bank->dev);
return ret;
+ }
omap_gpio_show_rev(bank);
return 0;
}
-static void pl061_irq_handler(unsigned irq, struct irq_desc *desc)
+static void pl061_irq_handler(struct irq_desc *desc)
{
unsigned long pending;
int offset;
return 0;
}
-static void pxa_gpio_demux_handler(unsigned int irq, struct irq_desc *desc)
+static void pxa_gpio_demux_handler(struct irq_desc *desc)
{
struct pxa_gpio_chip *c;
int loop, gpio, gpio_base, n;
* irq_controller_lock held, and IRQs disabled. Decode the IRQ
* and call the handler.
*/
-static void
-sa1100_gpio_handler(unsigned int __irq, struct irq_desc *desc)
+static void sa1100_gpio_handler(struct irq_desc *desc)
{
unsigned int irq, mask;
MODULE_AUTHOR("Gregory Bean <gbean@codeaurora.org>");
MODULE_DESCRIPTION("Driver for Semtech SX150X I2C GPIO Expanders");
MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("i2c:sx150x");
gpiochip_unlock_as_irq(&tegra_gpio_chip, gpio);
}
-static void tegra_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void tegra_gpio_irq_handler(struct irq_desc *desc)
{
int port;
int pin;
return ret;
}
-static void timbgpio_irq(unsigned int irq, struct irq_desc *desc)
+static void timbgpio_irq(struct irq_desc *desc)
{
struct timbgpio *tgpio = irq_desc_get_handler_data(desc);
struct irq_data *data = irq_desc_get_irq_data(desc);
#define gpio_set_irq_wake NULL
#endif
-static void tz1090_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void tz1090_gpio_irq_handler(struct irq_desc *desc)
{
irq_hw_number_t hw;
unsigned int irq_stat, irq_no;
== IRQ_TYPE_EDGE_BOTH)
tz1090_gpio_irq_next_edge(bank, hw);
- generic_handle_irq_desc(irq_no, child_desc);
+ generic_handle_irq_desc(child_desc);
}
}
return pinctrl_gpio_direction_output(chip->base + gpio);
}
-static void vf610_gpio_irq_handler(u32 irq, struct irq_desc *desc)
+static void vf610_gpio_irq_handler(struct irq_desc *desc)
{
struct vf610_gpio_port *port = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
port->irqc[d->hwirq] = irqc;
if (type & IRQ_TYPE_LEVEL_MASK)
- __irq_set_handler_locked(d->irq, handle_level_irq);
+ irq_set_handler_locked(d, handle_level_irq);
else
- __irq_set_handler_locked(d->irq, handle_edge_irq);
+ irq_set_handler_locked(d, handle_edge_irq);
return 0;
}
return 0;
}
-static void zx_irq_handler(unsigned irq, struct irq_desc *desc)
+static void zx_irq_handler(struct irq_desc *desc)
{
unsigned long pending;
int offset;
* application for that pin.
* Note: A bug is reported if no handler is set for the gpio pin.
*/
-static void zynq_gpio_irqhandler(unsigned int irq, struct irq_desc *desc)
+static void zynq_gpio_irqhandler(struct irq_desc *desc)
{
u32 int_sts, int_enb;
unsigned int bank_num;
* that the GPIO was actually requested.
*/
-static bool _gpiod_get_raw_value(const struct gpio_desc *desc)
+static int _gpiod_get_raw_value(const struct gpio_desc *desc)
{
struct gpio_chip *chip;
- bool value;
int offset;
+ int value;
chip = desc->chip;
offset = gpio_chip_hwgpio(desc);
- value = chip->get ? chip->get(chip, offset) : false;
+ value = chip->get ? chip->get(chip, offset) : -EIO;
+ value = value < 0 ? value : !!value;
trace_gpio_value(desc_to_gpio(desc), 1, value);
return value;
}
* @desc: gpio whose value will be returned
*
* Return the GPIO's raw value, i.e. the value of the physical line disregarding
- * its ACTIVE_LOW status.
+ * its ACTIVE_LOW status, or negative errno on failure.
*
* This function should be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
* @desc: gpio whose value will be returned
*
* Return the GPIO's logical value, i.e. taking the ACTIVE_LOW status into
- * account.
+ * account, or negative errno on failure.
*
* This function should be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
WARN_ON(desc->chip->can_sleep);
value = _gpiod_get_raw_value(desc);
+ if (value < 0)
+ return value;
+
if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
value = !value;
* @desc: gpio whose value will be returned
*
* Return the GPIO's raw value, i.e. the value of the physical line disregarding
- * its ACTIVE_LOW status.
+ * its ACTIVE_LOW status, or negative errno on failure.
*
* This function is to be called from contexts that can sleep.
*/
* @desc: gpio whose value will be returned
*
* Return the GPIO's logical value, i.e. taking the ACTIVE_LOW status into
- * account.
+ * account, or negative errno on failure.
*
* This function is to be called from contexts that can sleep.
*/
return 0;
value = _gpiod_get_raw_value(desc);
+ if (value < 0)
+ return value;
+
if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
value = !value;
source "drivers/gpu/drm/amd/amdkfd/Kconfig"
source "drivers/gpu/drm/imx/Kconfig"
+
+source "drivers/gpu/drm/vc4/Kconfig"
drm_context.o drm_dma.o \
drm_fops.o drm_gem.o drm_ioctl.o drm_irq.o \
drm_lock.o drm_memory.o drm_drv.o drm_vm.o \
- drm_agpsupport.o drm_scatter.o drm_pci.o \
+ drm_scatter.o drm_pci.o \
drm_platform.o drm_sysfs.o drm_hashtab.o drm_mm.o \
drm_crtc.o drm_modes.o drm_edid.o \
drm_info.o drm_debugfs.o drm_encoder_slave.o \
drm-$(CONFIG_PCI) += ati_pcigart.o
drm-$(CONFIG_DRM_PANEL) += drm_panel.o
drm-$(CONFIG_OF) += drm_of.o
+drm-$(CONFIG_AGP) += drm_agpsupport.o
+
+drm-y += $(drm-m)
drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \
drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o
obj-$(CONFIG_DRM_I810) += i810/
obj-$(CONFIG_DRM_I915) += i915/
obj-$(CONFIG_DRM_MGAG200) += mgag200/
+obj-$(CONFIG_DRM_VC4) += vc4/
obj-$(CONFIG_DRM_CIRRUS_QEMU) += cirrus/
obj-$(CONFIG_DRM_SIS) += sis/
obj-$(CONFIG_DRM_SAVAGE)+= savage/
extern int amdgpu_deep_color;
extern int amdgpu_vm_size;
extern int amdgpu_vm_block_size;
+extern int amdgpu_vm_fault_stop;
+extern int amdgpu_vm_debug;
extern int amdgpu_enable_scheduler;
extern int amdgpu_sched_jobs;
extern int amdgpu_sched_hw_submission;
+extern int amdgpu_enable_semaphores;
#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000
#define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */
/* testing functions */
int (*test_ring)(struct amdgpu_ring *ring);
int (*test_ib)(struct amdgpu_ring *ring);
- bool (*is_lockup)(struct amdgpu_ring *ring);
/* insert NOP packets */
void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count);
};
/* some special values for the owner field */
#define AMDGPU_FENCE_OWNER_UNDEFINED ((void*)0ul)
#define AMDGPU_FENCE_OWNER_VM ((void*)1ul)
-#define AMDGPU_FENCE_OWNER_MOVE ((void*)2ul)
#define AMDGPU_FENCE_FLAG_64BIT (1 << 0)
#define AMDGPU_FENCE_FLAG_INT (1 << 1)
void amdgpu_fence_driver_fini(struct amdgpu_device *adev);
void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev);
-void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring);
+int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring);
int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
struct amdgpu_irq_src *irq_src,
unsigned irq_type);
int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
-signed long amdgpu_fence_wait_any(struct amdgpu_device *adev,
- struct fence **array,
- uint32_t count,
- bool intr,
- signed long t);
-struct amdgpu_fence *amdgpu_fence_ref(struct amdgpu_fence *fence);
-void amdgpu_fence_unref(struct amdgpu_fence **fence);
-
bool amdgpu_fence_need_sync(struct amdgpu_fence *fence,
struct amdgpu_ring *ring);
void amdgpu_fence_note_sync(struct amdgpu_fence *fence,
struct amdgpu_ring *ring);
-static inline struct amdgpu_fence *amdgpu_fence_later(struct amdgpu_fence *a,
- struct amdgpu_fence *b)
-{
- if (!a) {
- return b;
- }
-
- if (!b) {
- return a;
- }
-
- BUG_ON(a->ring != b->ring);
-
- if (a->seq > b->seq) {
- return a;
- } else {
- return b;
- }
-}
-
-static inline bool amdgpu_fence_is_earlier(struct amdgpu_fence *a,
- struct amdgpu_fence *b)
-{
- if (!a) {
- return false;
- }
-
- if (!b) {
- return true;
- }
-
- BUG_ON(a->ring != b->ring);
-
- return a->seq < b->seq;
-}
-
-int amdgpu_user_fence_emit(struct amdgpu_ring *ring, struct amdgpu_user_fence *user,
- void *owner, struct amdgpu_fence **fence);
-
/*
* TTM.
*/
*/
struct amdgpu_sync {
struct amdgpu_semaphore *semaphores[AMDGPU_NUM_SYNCS];
- struct amdgpu_fence *sync_to[AMDGPU_MAX_RINGS];
+ struct fence *sync_to[AMDGPU_MAX_RINGS];
DECLARE_HASHTABLE(fences, 4);
struct fence *last_vm_update;
};
struct amdgpu_device *adev;
const struct amdgpu_ring_funcs *funcs;
struct amdgpu_fence_driver fence_drv;
- struct amd_gpu_scheduler *scheduler;
+ struct amd_gpu_scheduler sched;
spinlock_t fence_lock;
struct mutex *ring_lock;
unsigned ring_size;
unsigned ring_free_dw;
int count_dw;
- atomic_t last_rptr;
- atomic64_t last_activity;
uint64_t gpu_addr;
uint32_t align_mask;
uint32_t ptr_mask;
#define AMDGPU_PTE_FRAG_64KB (4 << 7)
#define AMDGPU_LOG2_PAGES_PER_FRAG 4
+/* How to programm VM fault handling */
+#define AMDGPU_VM_FAULT_STOP_NEVER 0
+#define AMDGPU_VM_FAULT_STOP_FIRST 1
+#define AMDGPU_VM_FAULT_STOP_ALWAYS 2
+
struct amdgpu_vm_pt {
struct amdgpu_bo *bo;
uint64_t addr;
/* last flushed PD/PT update */
struct fence *flushed_updates;
/* last use of vmid */
- struct amdgpu_fence *last_id_use;
+ struct fence *last_id_use;
};
struct amdgpu_vm {
};
struct amdgpu_vm_manager {
- struct amdgpu_fence *active[AMDGPU_NUM_VM];
+ struct fence *active[AMDGPU_NUM_VM];
uint32_t max_pfn;
/* number of VMIDs */
unsigned nvm;
struct amdgpu_irq_src priv_inst_irq;
/* gfx status */
uint32_t gfx_current_status;
- /* sync signal for const engine */
- unsigned ce_sync_offs;
/* ce ram size*/
unsigned ce_ram_size;
};
void amdgpu_ring_unlock_commit(struct amdgpu_ring *ring);
void amdgpu_ring_undo(struct amdgpu_ring *ring);
void amdgpu_ring_unlock_undo(struct amdgpu_ring *ring);
-void amdgpu_ring_lockup_update(struct amdgpu_ring *ring);
-bool amdgpu_ring_test_lockup(struct amdgpu_ring *ring);
unsigned amdgpu_ring_backup(struct amdgpu_ring *ring,
uint32_t **data);
int amdgpu_ring_restore(struct amdgpu_ring *ring,
struct amdgpu_irq_src *irq_src, unsigned irq_type,
enum amdgpu_ring_type ring_type);
void amdgpu_ring_fini(struct amdgpu_ring *ring);
+struct amdgpu_ring *amdgpu_ring_from_fence(struct fence *f);
/*
* CS.
uint32_t num_ibs;
struct mutex job_lock;
struct amdgpu_user_fence uf;
- int (*free_job)(struct amdgpu_job *sched_job);
+ int (*free_job)(struct amdgpu_job *job);
};
+#define to_amdgpu_job(sched_job) \
+ container_of((sched_job), struct amdgpu_job, base)
static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p, uint32_t ib_idx, int idx)
{
/*
* SDMA
*/
-struct amdgpu_sdma {
+struct amdgpu_sdma_instance {
/* SDMA firmware */
const struct firmware *fw;
uint32_t fw_version;
bool burst_nop;
};
+struct amdgpu_sdma {
+ struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES];
+ struct amdgpu_irq_src trap_irq;
+ struct amdgpu_irq_src illegal_inst_irq;
+ int num_instances;
+};
+
/*
* Firmware
*/
int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
void amdgpu_mn_unregister(struct amdgpu_bo *bo);
#else
-static int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
+static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
{
return -ENODEV;
}
-static void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
+static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
#endif
/*
struct device *dev;
struct drm_device *ddev;
struct pci_dev *pdev;
- struct rw_semaphore exclusive_lock;
/* ASIC */
enum amd_asic_type asic_type;
bool suspend;
bool need_dma32;
bool accel_working;
- bool needs_reset;
struct work_struct reset_work;
struct notifier_block acpi_nb;
struct amdgpu_i2c_chan *i2c_bus[AMDGPU_MAX_I2C_BUS];
struct amdgpu_gfx gfx;
/* sdma */
- struct amdgpu_sdma sdma[AMDGPU_MAX_SDMA_INSTANCES];
- struct amdgpu_irq_src sdma_trap_irq;
- struct amdgpu_irq_src sdma_illegal_inst_irq;
+ struct amdgpu_sdma sdma;
/* uvd */
bool has_uvd;
ring->ring_free_dw--;
}
-static inline struct amdgpu_sdma * amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
+static inline struct amdgpu_sdma_instance *
+amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
int i;
- for (i = 0; i < AMDGPU_MAX_SDMA_INSTANCES; i++)
- if (&adev->sdma[i].ring == ring)
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ if (&adev->sdma.instance[i].ring == ring)
break;
if (i < AMDGPU_MAX_SDMA_INSTANCES)
- return &adev->sdma[i];
+ return &adev->sdma.instance[i];
else
return NULL;
}
#define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))
#define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))
#define amdgpu_ring_test_ib(r) (r)->funcs->test_ib((r))
-#define amdgpu_ring_is_lockup(r) (r)->funcs->is_lockup((r))
#define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r))
#define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r))
#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
struct drm_file *file_priv);
int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon);
int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon);
-u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, int crtc);
-int amdgpu_enable_vblank_kms(struct drm_device *dev, int crtc);
-void amdgpu_disable_vblank_kms(struct drm_device *dev, int crtc);
-int amdgpu_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
+u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe);
+int amdgpu_enable_vblank_kms(struct drm_device *dev, unsigned int pipe);
+void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe);
+int amdgpu_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int pipe,
int *max_error,
struct timeval *vblank_time,
unsigned flags);
#include <linux/acpi.h>
#include <linux/slab.h>
#include <linux/power_supply.h>
-#include <linux/vga_switcheroo.h>
#include <acpi/video.h>
#include <drm/drmP.h>
#include <drm/drm_crtc_helper.h>
return -ENOMEM;
r = amdgpu_bo_create(rdev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT,
- AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, &(*mem)->bo);
+ AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, &(*mem)->bo);
if (r) {
dev_err(rdev->dev,
"failed to allocate BO for amdkfd (%d)\n", r);
case KGD_ENGINE_SDMA1:
hdr = (const union amdgpu_firmware_header *)
- adev->sdma[0].fw->data;
+ adev->sdma.instance[0].fw->data;
break;
case KGD_ENGINE_SDMA2:
hdr = (const union amdgpu_firmware_header *)
- adev->sdma[1].fw->data;
+ adev->sdma.instance[1].fw->data;
break;
default:
case KGD_ENGINE_SDMA1:
hdr = (const union amdgpu_firmware_header *)
- adev->sdma[0].fw->data;
+ adev->sdma.instance[0].fw->data;
break;
case KGD_ENGINE_SDMA2:
hdr = (const union amdgpu_firmware_header *)
- adev->sdma[1].fw->data;
+ adev->sdma.instance[1].fw->data;
break;
default:
return VGA_SWITCHEROO_DIS;
}
-static struct vga_switcheroo_handler amdgpu_atpx_handler = {
+static const struct vga_switcheroo_handler amdgpu_atpx_handler = {
.switchto = amdgpu_atpx_switchto,
.power_state = amdgpu_atpx_power_state,
.init = amdgpu_atpx_init,
if (has_atpx && vga_count == 2) {
acpi_get_name(amdgpu_atpx_priv.atpx.handle, ACPI_FULL_PATHNAME, &buffer);
- printk(KERN_INFO "VGA switcheroo: detected switching method %s handle\n",
+ printk(KERN_INFO "vga_switcheroo: detected switching method %s handle\n",
acpi_method_name);
amdgpu_atpx_priv.atpx_detected = true;
return true;
int time;
n = AMDGPU_BENCHMARK_ITERATIONS;
- r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, sdomain, 0, NULL, &sobj);
+ r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, sdomain, 0, NULL,
+ NULL, &sobj);
if (r) {
goto out_cleanup;
}
if (r) {
goto out_cleanup;
}
- r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, ddomain, 0, NULL, &dobj);
+ r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, ddomain, 0, NULL,
+ NULL, &dobj);
if (r) {
goto out_cleanup;
}
#include "amdgpu.h"
#include "atom.h"
-#include <linux/vga_switcheroo.h>
#include <linux/slab.h>
#include <linux/acpi.h>
/*
struct sg_table *sg = drm_prime_pages_to_sg(&kmem_page, npages);
ret = amdgpu_bo_create(adev, size, PAGE_SIZE, false,
- AMDGPU_GEM_DOMAIN_GTT, 0, sg, &bo);
+ AMDGPU_GEM_DOMAIN_GTT, 0, sg, NULL, &bo);
if (ret)
return ret;
ret = amdgpu_bo_reserve(bo, false);
ret = amdgpu_bo_create_restricted(adev, size, PAGE_SIZE,
true, domain, flags,
- NULL, &placement, &obj);
+ NULL, &placement, NULL,
+ &obj);
if (ret) {
DRM_ERROR("(%d) bo create failed\n", ret);
return ret;
return ret;
}
-static int amdgpu_cgs_import_gpu_mem(void *cgs_device, int dmabuf_fd,
- cgs_handle_t *handle)
-{
- CGS_FUNC_ADEV;
- int r;
- uint32_t dma_handle;
- struct drm_gem_object *obj;
- struct amdgpu_bo *bo;
- struct drm_device *dev = adev->ddev;
- struct drm_file *file_priv = NULL, *priv;
-
- mutex_lock(&dev->struct_mutex);
- list_for_each_entry(priv, &dev->filelist, lhead) {
- rcu_read_lock();
- if (priv->pid == get_pid(task_pid(current)))
- file_priv = priv;
- rcu_read_unlock();
- if (file_priv)
- break;
- }
- mutex_unlock(&dev->struct_mutex);
- r = dev->driver->prime_fd_to_handle(dev,
- file_priv, dmabuf_fd,
- &dma_handle);
- spin_lock(&file_priv->table_lock);
-
- /* Check if we currently have a reference on the object */
- obj = idr_find(&file_priv->object_idr, dma_handle);
- if (obj == NULL) {
- spin_unlock(&file_priv->table_lock);
- return -EINVAL;
- }
- spin_unlock(&file_priv->table_lock);
- bo = gem_to_amdgpu_bo(obj);
- *handle = (cgs_handle_t)bo;
- return 0;
-}
-
static int amdgpu_cgs_free_gpu_mem(void *cgs_device, cgs_handle_t handle)
{
struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
};
static const struct cgs_os_ops amdgpu_cgs_os_ops = {
- amdgpu_cgs_import_gpu_mem,
amdgpu_cgs_add_irq_source,
amdgpu_cgs_irq_get,
amdgpu_cgs_irq_put
}
break;
case AMDGPU_HW_IP_DMA:
- if (ring < 2) {
- *out_ring = &adev->sdma[ring].ring;
+ if (ring < adev->sdma.num_instances) {
+ *out_ring = &adev->sdma.instance[ring].ring;
} else {
- DRM_ERROR("only two SDMA rings are supported\n");
+ DRM_ERROR("only %d SDMA rings are supported\n",
+ adev->sdma.num_instances);
return -EINVAL;
}
break;
{
union drm_amdgpu_cs *cs = data;
uint64_t *chunk_array_user;
- uint64_t *chunk_array = NULL;
+ uint64_t *chunk_array;
struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
- unsigned size, i;
- int r = 0;
+ unsigned size;
+ int i;
+ int ret;
- if (!cs->in.num_chunks)
- goto out;
+ if (cs->in.num_chunks == 0)
+ return 0;
+
+ chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
+ if (!chunk_array)
+ return -ENOMEM;
p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
if (!p->ctx) {
- r = -EINVAL;
- goto out;
+ ret = -EINVAL;
+ goto free_chunk;
}
+
p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
/* get chunks */
INIT_LIST_HEAD(&p->validated);
- chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
- if (chunk_array == NULL) {
- r = -ENOMEM;
- goto out;
- }
-
- chunk_array_user = (uint64_t __user *)(cs->in.chunks);
+ chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks);
if (copy_from_user(chunk_array, chunk_array_user,
sizeof(uint64_t)*cs->in.num_chunks)) {
- r = -EFAULT;
- goto out;
+ ret = -EFAULT;
+ goto put_bo_list;
}
p->nchunks = cs->in.num_chunks;
p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
GFP_KERNEL);
- if (p->chunks == NULL) {
- r = -ENOMEM;
- goto out;
+ if (!p->chunks) {
+ ret = -ENOMEM;
+ goto put_bo_list;
}
for (i = 0; i < p->nchunks; i++) {
struct drm_amdgpu_cs_chunk user_chunk;
uint32_t __user *cdata;
- chunk_ptr = (void __user *)chunk_array[i];
+ chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
if (copy_from_user(&user_chunk, chunk_ptr,
sizeof(struct drm_amdgpu_cs_chunk))) {
- r = -EFAULT;
- goto out;
+ ret = -EFAULT;
+ i--;
+ goto free_partial_kdata;
}
p->chunks[i].chunk_id = user_chunk.chunk_id;
p->chunks[i].length_dw = user_chunk.length_dw;
size = p->chunks[i].length_dw;
- cdata = (void __user *)user_chunk.chunk_data;
+ cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
p->chunks[i].user_ptr = cdata;
p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
if (p->chunks[i].kdata == NULL) {
- r = -ENOMEM;
- goto out;
+ ret = -ENOMEM;
+ i--;
+ goto free_partial_kdata;
}
size *= sizeof(uint32_t);
if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
- r = -EFAULT;
- goto out;
+ ret = -EFAULT;
+ goto free_partial_kdata;
}
switch (p->chunks[i].chunk_id) {
gobj = drm_gem_object_lookup(p->adev->ddev,
p->filp, handle);
if (gobj == NULL) {
- r = -EINVAL;
- goto out;
+ ret = -EINVAL;
+ goto free_partial_kdata;
}
p->uf.bo = gem_to_amdgpu_bo(gobj);
p->uf.offset = fence_data->offset;
} else {
- r = -EINVAL;
- goto out;
+ ret = -EINVAL;
+ goto free_partial_kdata;
}
break;
break;
default:
- r = -EINVAL;
- goto out;
+ ret = -EINVAL;
+ goto free_partial_kdata;
}
}
p->ibs = kcalloc(p->num_ibs, sizeof(struct amdgpu_ib), GFP_KERNEL);
- if (!p->ibs)
- r = -ENOMEM;
+ if (!p->ibs) {
+ ret = -ENOMEM;
+ goto free_all_kdata;
+ }
-out:
kfree(chunk_array);
- return r;
+ return 0;
+
+free_all_kdata:
+ i = p->nchunks - 1;
+free_partial_kdata:
+ for (; i >= 0; i--)
+ drm_free_large(p->chunks[i].kdata);
+ kfree(p->chunks);
+put_bo_list:
+ if (p->bo_list)
+ amdgpu_bo_list_put(p->bo_list);
+ amdgpu_ctx_put(p->ctx);
+free_chunk:
+ kfree(chunk_array);
+
+ return ret;
}
/* Returns how many bytes TTM can move per IB.
return max(bytes_moved_threshold, 1024*1024ull);
}
-int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p)
+int amdgpu_cs_list_validate(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm,
+ struct list_head *validated)
{
- struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
- struct amdgpu_vm *vm = &fpriv->vm;
- struct amdgpu_device *adev = p->adev;
struct amdgpu_bo_list_entry *lobj;
- struct list_head duplicates;
struct amdgpu_bo *bo;
u64 bytes_moved = 0, initial_bytes_moved;
u64 bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(adev);
int r;
- INIT_LIST_HEAD(&duplicates);
- r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates);
- if (unlikely(r != 0)) {
- return r;
- }
-
- list_for_each_entry(lobj, &p->validated, tv.head) {
+ list_for_each_entry(lobj, validated, tv.head) {
bo = lobj->robj;
if (!bo->pin_count) {
u32 domain = lobj->prefered_domains;
domain = lobj->allowed_domains;
goto retry;
}
- ttm_eu_backoff_reservation(&p->ticket, &p->validated);
return r;
}
}
{
struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
struct amdgpu_cs_buckets buckets;
+ struct list_head duplicates;
bool need_mmap_lock = false;
int i, r;
if (need_mmap_lock)
down_read(¤t->mm->mmap_sem);
- r = amdgpu_cs_list_validate(p);
+ INIT_LIST_HEAD(&duplicates);
+ r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates);
+ if (unlikely(r != 0))
+ goto error_reserve;
+
+ r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &p->validated);
+ if (r)
+ goto error_validate;
+
+ r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &duplicates);
+error_validate:
+ if (r)
+ ttm_eu_backoff_reservation(&p->ticket, &p->validated);
+
+error_reserve:
if (need_mmap_lock)
up_read(¤t->mm->mmap_sem);
if (r)
return r;
}
+
+ }
+
+ r = amdgpu_vm_clear_invalids(adev, vm, &p->ibs[0].sync);
+
+ if (amdgpu_vm_debug && p->bo_list) {
+ /* Invalidate all BOs to test for userspace bugs */
+ for (i = 0; i < p->bo_list->num_entries; i++) {
+ /* ignore duplicates */
+ bo = p->bo_list->array[i].robj;
+ if (!bo)
+ continue;
+
+ amdgpu_vm_bo_invalidate(adev, bo);
+ }
}
- return amdgpu_vm_clear_invalids(adev, vm, &p->ibs[0].sync);
+ return r;
}
static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
}
}
- mutex_lock(&vm->mutex);
r = amdgpu_bo_vm_update_pte(parser, vm);
if (r) {
goto out;
parser->filp);
out:
- mutex_unlock(&vm->mutex);
return r;
}
return 0;
}
-static int amdgpu_cs_free_job(struct amdgpu_job *sched_job)
+static int amdgpu_cs_free_job(struct amdgpu_job *job)
{
int i;
- if (sched_job->ibs)
- for (i = 0; i < sched_job->num_ibs; i++)
- amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]);
- kfree(sched_job->ibs);
- if (sched_job->uf.bo)
- drm_gem_object_unreference_unlocked(&sched_job->uf.bo->gem_base);
+ if (job->ibs)
+ for (i = 0; i < job->num_ibs; i++)
+ amdgpu_ib_free(job->adev, &job->ibs[i]);
+ kfree(job->ibs);
+ if (job->uf.bo)
+ drm_gem_object_unreference_unlocked(&job->uf.bo->gem_base);
return 0;
}
{
struct amdgpu_device *adev = dev->dev_private;
union drm_amdgpu_cs *cs = data;
+ struct amdgpu_fpriv *fpriv = filp->driver_priv;
+ struct amdgpu_vm *vm = &fpriv->vm;
struct amdgpu_cs_parser *parser;
bool reserved_buffers = false;
int i, r;
- down_read(&adev->exclusive_lock);
- if (!adev->accel_working) {
- up_read(&adev->exclusive_lock);
+ if (!adev->accel_working)
return -EBUSY;
- }
parser = amdgpu_cs_parser_create(adev, filp, NULL, NULL, 0);
if (!parser)
if (r) {
DRM_ERROR("Failed to initialize parser !\n");
amdgpu_cs_parser_fini(parser, r, false);
- up_read(&adev->exclusive_lock);
r = amdgpu_cs_handle_lockup(adev, r);
return r;
}
-
+ mutex_lock(&vm->mutex);
r = amdgpu_cs_parser_relocs(parser);
if (r == -ENOMEM)
DRM_ERROR("Not enough memory for command submission!\n");
struct amdgpu_job *job;
struct amdgpu_ring * ring = parser->ibs->ring;
job = kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL);
- if (!job)
- return -ENOMEM;
- job->base.sched = ring->scheduler;
+ if (!job) {
+ r = -ENOMEM;
+ goto out;
+ }
+ job->base.sched = &ring->sched;
job->base.s_entity = &parser->ctx->rings[ring->idx].entity;
job->adev = parser->adev;
job->ibs = parser->ibs;
job->free_job = amdgpu_cs_free_job;
mutex_lock(&job->job_lock);
- r = amd_sched_entity_push_job((struct amd_sched_job *)job);
+ r = amd_sched_entity_push_job(&job->base);
if (r) {
mutex_unlock(&job->job_lock);
amdgpu_cs_free_job(job);
mutex_unlock(&job->job_lock);
amdgpu_cs_parser_fini_late(parser);
- up_read(&adev->exclusive_lock);
+ mutex_unlock(&vm->mutex);
return 0;
}
cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence;
out:
amdgpu_cs_parser_fini(parser, r, reserved_buffers);
- up_read(&adev->exclusive_lock);
+ mutex_unlock(&vm->mutex);
r = amdgpu_cs_handle_lockup(adev, r);
return r;
}
for (i = 0; i < adev->num_rings; i++) {
struct amd_sched_rq *rq;
if (kernel)
- rq = &adev->rings[i]->scheduler->kernel_rq;
+ rq = &adev->rings[i]->sched.kernel_rq;
else
- rq = &adev->rings[i]->scheduler->sched_rq;
- r = amd_sched_entity_init(adev->rings[i]->scheduler,
+ rq = &adev->rings[i]->sched.sched_rq;
+ r = amd_sched_entity_init(&adev->rings[i]->sched,
&ctx->rings[i].entity,
rq, amdgpu_sched_jobs);
if (r)
if (i < adev->num_rings) {
for (j = 0; j < i; j++)
- amd_sched_entity_fini(adev->rings[j]->scheduler,
+ amd_sched_entity_fini(&adev->rings[j]->sched,
&ctx->rings[j].entity);
kfree(ctx);
return r;
struct amdgpu_device *adev = ctx->adev;
unsigned i, j;
+ if (!adev)
+ return;
+
for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
for (j = 0; j < AMDGPU_CTX_MAX_CS_PENDING; ++j)
fence_put(ctx->rings[i].fences[j]);
if (amdgpu_enable_scheduler) {
for (i = 0; i < adev->num_rings; i++)
- amd_sched_entity_fini(adev->rings[i]->scheduler,
+ amd_sched_entity_fini(&adev->rings[i]->sched,
&ctx->rings[i].entity);
}
}
"TONGA",
"FIJI",
"CARRIZO",
+ "STONEY",
"LAST",
};
r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->vram_scratch.robj);
+ NULL, NULL, &adev->vram_scratch.robj);
if (r) {
return r;
}
if (adev->wb.wb_obj == NULL) {
r = amdgpu_bo_create(adev, AMDGPU_MAX_WB * 4, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GTT, 0, NULL, &adev->wb.wb_obj);
+ AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
+ &adev->wb.wb_obj);
if (r) {
dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
return r;
* amdgpu_switcheroo_set_state - set switcheroo state
*
* @pdev: pci dev pointer
- * @state: vga switcheroo state
+ * @state: vga_switcheroo state
*
* Callback for the switcheroo driver. Suspends or resumes the
* the asics before or after it is powered up using ACPI methods.
case CHIP_TONGA:
case CHIP_FIJI:
case CHIP_CARRIZO:
- if (adev->asic_type == CHIP_CARRIZO)
+ case CHIP_STONEY:
+ if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
adev->family = AMDGPU_FAMILY_CZ;
else
adev->family = AMDGPU_FAMILY_VI;
mutex_init(&adev->gfx.gpu_clock_mutex);
mutex_init(&adev->srbm_mutex);
mutex_init(&adev->grbm_idx_mutex);
- init_rwsem(&adev->exclusive_lock);
mutex_init(&adev->mn_lock);
hash_init(adev->mn_hash);
drm_kms_helper_poll_disable(dev);
/* turn off display hw */
+ drm_modeset_lock_all(dev);
list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
}
+ drm_modeset_unlock_all(dev);
- /* unpin the front buffers */
+ /* unpin the front buffers and cursors */
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+ struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
struct amdgpu_bo *robj;
+ if (amdgpu_crtc->cursor_bo) {
+ struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+ r = amdgpu_bo_reserve(aobj, false);
+ if (r == 0) {
+ amdgpu_bo_unpin(aobj);
+ amdgpu_bo_unreserve(aobj);
+ }
+ }
+
if (rfb == NULL || rfb->obj == NULL) {
continue;
}
{
struct drm_connector *connector;
struct amdgpu_device *adev = dev->dev_private;
+ struct drm_crtc *crtc;
int r;
if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
if (r)
return r;
+ /* pin cursors */
+ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+ struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+ if (amdgpu_crtc->cursor_bo) {
+ struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+ r = amdgpu_bo_reserve(aobj, false);
+ if (r == 0) {
+ r = amdgpu_bo_pin(aobj,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &amdgpu_crtc->cursor_addr);
+ if (r != 0)
+ DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
+ amdgpu_bo_unreserve(aobj);
+ }
+ }
+ }
+
/* blat the mode back in */
if (fbcon) {
drm_helper_resume_force_mode(dev);
/* turn on display hw */
+ drm_modeset_lock_all(dev);
list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
}
+ drm_modeset_unlock_all(dev);
}
drm_kms_helper_poll_enable(dev);
int i, r;
int resched;
- down_write(&adev->exclusive_lock);
-
- if (!adev->needs_reset) {
- up_write(&adev->exclusive_lock);
- return 0;
- }
-
- adev->needs_reset = false;
atomic_inc(&adev->gpu_reset_counter);
/* block TTM */
dev_info(adev->dev, "GPU reset failed\n");
}
- up_write(&adev->exclusive_lock);
return r;
}
fence = to_amdgpu_fence(*f);
if (fence) {
r = fence_wait(&fence->base, false);
- if (r == -EDEADLK) {
- up_read(&adev->exclusive_lock);
+ if (r == -EDEADLK)
r = amdgpu_gpu_reset(adev);
- down_read(&adev->exclusive_lock);
- }
} else
r = fence_wait(*f, false);
unsigned long flags;
unsigned i;
- down_read(&adev->exclusive_lock);
amdgpu_flip_wait_fence(adev, &work->excl);
for (i = 0; i < work->shared_count; ++i)
amdgpu_flip_wait_fence(adev, &work->shared[i]);
amdgpuCrtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
- up_read(&adev->exclusive_lock);
}
/*
* an optional accurate timestamp of when query happened.
*
* \param dev Device to query.
- * \param crtc Crtc to query.
+ * \param pipe Crtc to query.
* \param flags Flags from caller (DRM_CALLED_FROM_VBLIRQ or 0).
* \param *vpos Location where vertical scanout position should be stored.
* \param *hpos Location where horizontal scanout position should go.
* unknown small number of scanlines wrt. real scanout position.
*
*/
-int amdgpu_get_crtc_scanoutpos(struct drm_device *dev, int crtc, unsigned int flags,
- int *vpos, int *hpos, ktime_t *stime, ktime_t *etime)
+int amdgpu_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
+ unsigned int flags, int *vpos, int *hpos,
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode)
{
u32 vbl = 0, position = 0;
int vbl_start, vbl_end, vtotal, ret = 0;
if (stime)
*stime = ktime_get();
- if (amdgpu_display_page_flip_get_scanoutpos(adev, crtc, &vbl, &position) == 0)
+ if (amdgpu_display_page_flip_get_scanoutpos(adev, pipe, &vbl, &position) == 0)
ret |= DRM_SCANOUTPOS_VALID;
/* Get optional system timestamp after query. */
}
else {
/* No: Fake something reasonable which gives at least ok results. */
- vbl_start = adev->mode_info.crtcs[crtc]->base.hwmode.crtc_vdisplay;
+ vbl_start = mode->crtc_vdisplay;
vbl_end = 0;
}
/* Inside "upper part" of vblank area? Apply corrective offset if so: */
if (in_vbl && (*vpos >= vbl_start)) {
- vtotal = adev->mode_info.crtcs[crtc]->base.hwmode.crtc_vtotal;
+ vtotal = mode->crtc_vtotal;
*vpos = *vpos - vtotal;
}
* We only do this if DRM_CALLED_FROM_VBLIRQ.
*/
if ((flags & DRM_CALLED_FROM_VBLIRQ) && !in_vbl) {
- vbl_start = adev->mode_info.crtcs[crtc]->base.hwmode.crtc_vdisplay;
- vtotal = adev->mode_info.crtcs[crtc]->base.hwmode.crtc_vtotal;
+ vbl_start = mode->crtc_vdisplay;
+ vtotal = mode->crtc_vtotal;
if (vbl_start - *vpos < vtotal / 100) {
*vpos -= vtotal;
unsigned amdgpu_ip_block_mask = 0xffffffff;
int amdgpu_bapm = -1;
int amdgpu_deep_color = 0;
-int amdgpu_vm_size = 8;
+int amdgpu_vm_size = 64;
int amdgpu_vm_block_size = -1;
+int amdgpu_vm_fault_stop = 0;
+int amdgpu_vm_debug = 0;
int amdgpu_exp_hw_support = 0;
-int amdgpu_enable_scheduler = 0;
+int amdgpu_enable_scheduler = 1;
int amdgpu_sched_jobs = 16;
int amdgpu_sched_hw_submission = 2;
+int amdgpu_enable_semaphores = 0;
MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
MODULE_PARM_DESC(deep_color, "Deep Color support (1 = enable, 0 = disable (default))");
module_param_named(deep_color, amdgpu_deep_color, int, 0444);
-MODULE_PARM_DESC(vm_size, "VM address space size in gigabytes (default 8GB)");
+MODULE_PARM_DESC(vm_size, "VM address space size in gigabytes (default 64GB)");
module_param_named(vm_size, amdgpu_vm_size, int, 0444);
MODULE_PARM_DESC(vm_block_size, "VM page table size in bits (default depending on vm_size)");
module_param_named(vm_block_size, amdgpu_vm_block_size, int, 0444);
+MODULE_PARM_DESC(vm_fault_stop, "Stop on VM fault (0 = never (default), 1 = print first, 2 = always)");
+module_param_named(vm_fault_stop, amdgpu_vm_fault_stop, int, 0444);
+
+MODULE_PARM_DESC(vm_debug, "Debug VM handling (0 = disabled (default), 1 = enabled)");
+module_param_named(vm_debug, amdgpu_vm_debug, int, 0644);
+
MODULE_PARM_DESC(exp_hw_support, "experimental hw support (1 = enable, 0 = disable (default))");
module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
-MODULE_PARM_DESC(enable_scheduler, "enable SW GPU scheduler (1 = enable, 0 = disable ((default))");
+MODULE_PARM_DESC(enable_scheduler, "enable SW GPU scheduler (1 = enable (default), 0 = disable)");
module_param_named(enable_scheduler, amdgpu_enable_scheduler, int, 0444);
MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 16)");
MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)");
module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
+MODULE_PARM_DESC(enable_semaphores, "Enable semaphores (1 = enable, 0 = disable (default))");
+module_param_named(enable_semaphores, amdgpu_enable_semaphores, int, 0644);
+
static struct pci_device_id pciidlist[] = {
#ifdef CONFIG_DRM_AMDGPU_CIK
/* Kaveri */
{0x1002, 0x985F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
#endif
/* topaz */
- {0x1002, 0x6900, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ},
- {0x1002, 0x6901, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ},
- {0x1002, 0x6902, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ},
- {0x1002, 0x6903, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ},
- {0x1002, 0x6907, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ},
+ {0x1002, 0x6900, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ|AMD_EXP_HW_SUPPORT},
+ {0x1002, 0x6901, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ|AMD_EXP_HW_SUPPORT},
+ {0x1002, 0x6902, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ|AMD_EXP_HW_SUPPORT},
+ {0x1002, 0x6903, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ|AMD_EXP_HW_SUPPORT},
+ {0x1002, 0x6907, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ|AMD_EXP_HW_SUPPORT},
/* tonga */
{0x1002, 0x6920, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
{0x1002, 0x6921, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
{0x1002, 0x9875, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
{0x1002, 0x9876, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
{0x1002, 0x9877, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
+ /* stoney */
+ {0x1002, 0x98E4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_STONEY|AMD_IS_APU},
{0, 0, 0}
};
}
info->par = rfbdev;
+ info->skip_vt_switch = true;
ret = amdgpu_framebuffer_init(adev->ddev, &rfbdev->rfb, &mode_cmd, gobj);
if (ret) {
return true;
return false;
}
+
+void amdgpu_fbdev_restore_mode(struct amdgpu_device *adev)
+{
+ struct amdgpu_fbdev *afbdev = adev->mode_info.rfbdev;
+ struct drm_fb_helper *fb_helper;
+ int ret;
+
+ if (!afbdev)
+ return;
+
+ fb_helper = &afbdev->helper;
+
+ ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper);
+ if (ret)
+ DRM_DEBUG("failed to restore crtc mode\n");
+}
return 0;
}
-/**
- * amdgpu_fence_check_signaled - callback from fence_queue
- *
- * this function is called with fence_queue lock held, which is also used
- * for the fence locking itself, so unlocked variants are used for
- * fence_signal, and remove_wait_queue.
- */
-static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key)
-{
- struct amdgpu_fence *fence;
- struct amdgpu_device *adev;
- u64 seq;
- int ret;
-
- fence = container_of(wait, struct amdgpu_fence, fence_wake);
- adev = fence->ring->adev;
-
- /*
- * We cannot use amdgpu_fence_process here because we're already
- * in the waitqueue, in a call from wake_up_all.
- */
- seq = atomic64_read(&fence->ring->fence_drv.last_seq);
- if (seq >= fence->seq) {
- ret = fence_signal_locked(&fence->base);
- if (!ret)
- FENCE_TRACE(&fence->base, "signaled from irq context\n");
- else
- FENCE_TRACE(&fence->base, "was already signaled\n");
-
- __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake);
- fence_put(&fence->base);
- } else
- FENCE_TRACE(&fence->base, "pending\n");
- return 0;
-}
-
/**
* amdgpu_fence_activity - check for fence activity
*
lockup_work.work);
ring = fence_drv->ring;
- if (!down_read_trylock(&ring->adev->exclusive_lock)) {
- /* just reschedule the check if a reset is going on */
- amdgpu_fence_schedule_check(ring);
- return;
- }
-
- if (amdgpu_fence_activity(ring)) {
- wake_up_all(&ring->fence_drv.fence_queue);
- }
- else if (amdgpu_ring_is_lockup(ring)) {
- /* good news we believe it's a lockup */
- dev_warn(ring->adev->dev, "GPU lockup (current fence id "
- "0x%016llx last fence id 0x%016llx on ring %d)\n",
- (uint64_t)atomic64_read(&fence_drv->last_seq),
- fence_drv->sync_seq[ring->idx], ring->idx);
-
- /* remember that we need an reset */
- ring->adev->needs_reset = true;
+ if (amdgpu_fence_activity(ring))
wake_up_all(&ring->fence_drv.fence_queue);
- }
- up_read(&ring->adev->exclusive_lock);
}
/**
return false;
}
-static bool amdgpu_fence_is_signaled(struct fence *f)
-{
- struct amdgpu_fence *fence = to_amdgpu_fence(f);
- struct amdgpu_ring *ring = fence->ring;
- struct amdgpu_device *adev = ring->adev;
-
- if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
- return true;
-
- if (down_read_trylock(&adev->exclusive_lock)) {
- amdgpu_fence_process(ring);
- up_read(&adev->exclusive_lock);
-
- if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
- return true;
- }
- return false;
-}
-
-/**
- * amdgpu_fence_enable_signaling - enable signalling on fence
- * @fence: fence
- *
- * This function is called with fence_queue lock held, and adds a callback
- * to fence_queue that checks if this fence is signaled, and if so it
- * signals the fence and removes itself.
- */
-static bool amdgpu_fence_enable_signaling(struct fence *f)
-{
- struct amdgpu_fence *fence = to_amdgpu_fence(f);
- struct amdgpu_ring *ring = fence->ring;
-
- if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
- return false;
-
- fence->fence_wake.flags = 0;
- fence->fence_wake.private = NULL;
- fence->fence_wake.func = amdgpu_fence_check_signaled;
- __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake);
- fence_get(f);
- FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
- return true;
-}
-
/*
* amdgpu_ring_wait_seq_timeout - wait for seq of the specific ring to signal
* @ring: ring to wait on for the seq number
*/
static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq)
{
- struct amdgpu_device *adev = ring->adev;
bool signaled = false;
BUG_ON(!ring);
if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
return 0;
+ amdgpu_fence_schedule_check(ring);
wait_event(ring->fence_drv.fence_queue, (
- (signaled = amdgpu_fence_seq_signaled(ring, seq))
- || adev->needs_reset));
+ (signaled = amdgpu_fence_seq_signaled(ring, seq))));
if (signaled)
return 0;
return amdgpu_fence_ring_wait_seq(ring, seq);
}
-/**
- * amdgpu_fence_ref - take a ref on a fence
- *
- * @fence: amdgpu fence object
- *
- * Take a reference on a fence (all asics).
- * Returns the fence.
- */
-struct amdgpu_fence *amdgpu_fence_ref(struct amdgpu_fence *fence)
-{
- fence_get(&fence->base);
- return fence;
-}
-
-/**
- * amdgpu_fence_unref - remove a ref on a fence
- *
- * @fence: amdgpu fence object
- *
- * Remove a reference on a fence (all asics).
- */
-void amdgpu_fence_unref(struct amdgpu_fence **fence)
-{
- struct amdgpu_fence *tmp = *fence;
-
- *fence = NULL;
- if (tmp)
- fence_put(&tmp->base);
-}
-
/**
* amdgpu_fence_count_emitted - get the count of emitted fences
*
* Init the fence driver for the requested ring (all asics).
* Helper function for amdgpu_fence_driver_init().
*/
-void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
+int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
{
- int i;
+ int i, r;
ring->fence_drv.cpu_addr = NULL;
ring->fence_drv.gpu_addr = 0;
amdgpu_fence_check_lockup);
ring->fence_drv.ring = ring;
+ init_waitqueue_head(&ring->fence_drv.fence_queue);
+
if (amdgpu_enable_scheduler) {
- ring->scheduler = amd_sched_create(&amdgpu_sched_ops,
- ring->idx,
- amdgpu_sched_hw_submission,
- (void *)ring->adev);
- if (!ring->scheduler)
- DRM_ERROR("Failed to create scheduler on ring %d.\n",
- ring->idx);
+ long timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
+ if (timeout == 0) {
+ /*
+ * FIXME:
+ * Delayed workqueue cannot use it directly,
+ * so the scheduler will not use delayed workqueue if
+ * MAX_SCHEDULE_TIMEOUT is set.
+ * Currently keep it simple and silly.
+ */
+ timeout = MAX_SCHEDULE_TIMEOUT;
+ }
+ r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
+ amdgpu_sched_hw_submission,
+ timeout, ring->name);
+ if (r) {
+ DRM_ERROR("Failed to create scheduler on ring %s.\n",
+ ring->name);
+ return r;
+ }
}
+
+ return 0;
}
/**
wake_up_all(&ring->fence_drv.fence_queue);
amdgpu_irq_put(adev, ring->fence_drv.irq_src,
ring->fence_drv.irq_type);
- if (ring->scheduler)
- amd_sched_destroy(ring->scheduler);
+ amd_sched_fini(&ring->sched);
ring->fence_drv.initialized = false;
}
mutex_unlock(&adev->ring_lock);
}
}
+/*
+ * Common fence implementation
+ */
+
+static const char *amdgpu_fence_get_driver_name(struct fence *fence)
+{
+ return "amdgpu";
+}
+
+static const char *amdgpu_fence_get_timeline_name(struct fence *f)
+{
+ struct amdgpu_fence *fence = to_amdgpu_fence(f);
+ return (const char *)fence->ring->name;
+}
+
+/**
+ * amdgpu_fence_is_signaled - test if fence is signaled
+ *
+ * @f: fence to test
+ *
+ * Test the fence sequence number if it is already signaled. If it isn't
+ * signaled start fence processing. Returns True if the fence is signaled.
+ */
+static bool amdgpu_fence_is_signaled(struct fence *f)
+{
+ struct amdgpu_fence *fence = to_amdgpu_fence(f);
+ struct amdgpu_ring *ring = fence->ring;
+
+ if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
+ return true;
+
+ amdgpu_fence_process(ring);
+
+ if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
+ return true;
+
+ return false;
+}
+
+/**
+ * amdgpu_fence_check_signaled - callback from fence_queue
+ *
+ * this function is called with fence_queue lock held, which is also used
+ * for the fence locking itself, so unlocked variants are used for
+ * fence_signal, and remove_wait_queue.
+ */
+static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key)
+{
+ struct amdgpu_fence *fence;
+ struct amdgpu_device *adev;
+ u64 seq;
+ int ret;
+
+ fence = container_of(wait, struct amdgpu_fence, fence_wake);
+ adev = fence->ring->adev;
+
+ /*
+ * We cannot use amdgpu_fence_process here because we're already
+ * in the waitqueue, in a call from wake_up_all.
+ */
+ seq = atomic64_read(&fence->ring->fence_drv.last_seq);
+ if (seq >= fence->seq) {
+ ret = fence_signal_locked(&fence->base);
+ if (!ret)
+ FENCE_TRACE(&fence->base, "signaled from irq context\n");
+ else
+ FENCE_TRACE(&fence->base, "was already signaled\n");
+
+ __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake);
+ fence_put(&fence->base);
+ } else
+ FENCE_TRACE(&fence->base, "pending\n");
+ return 0;
+}
+
+/**
+ * amdgpu_fence_enable_signaling - enable signalling on fence
+ * @fence: fence
+ *
+ * This function is called with fence_queue lock held, and adds a callback
+ * to fence_queue that checks if this fence is signaled, and if so it
+ * signals the fence and removes itself.
+ */
+static bool amdgpu_fence_enable_signaling(struct fence *f)
+{
+ struct amdgpu_fence *fence = to_amdgpu_fence(f);
+ struct amdgpu_ring *ring = fence->ring;
+
+ if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
+ return false;
+
+ fence->fence_wake.flags = 0;
+ fence->fence_wake.private = NULL;
+ fence->fence_wake.func = amdgpu_fence_check_signaled;
+ __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake);
+ fence_get(f);
+ amdgpu_fence_schedule_check(ring);
+ FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
+ return true;
+}
+
+const struct fence_ops amdgpu_fence_ops = {
+ .get_driver_name = amdgpu_fence_get_driver_name,
+ .get_timeline_name = amdgpu_fence_get_timeline_name,
+ .enable_signaling = amdgpu_fence_enable_signaling,
+ .signaled = amdgpu_fence_is_signaled,
+ .wait = fence_default_wait,
+ .release = NULL,
+};
/*
* Fence debugfs
#endif
}
-static const char *amdgpu_fence_get_driver_name(struct fence *fence)
-{
- return "amdgpu";
-}
-
-static const char *amdgpu_fence_get_timeline_name(struct fence *f)
-{
- struct amdgpu_fence *fence = to_amdgpu_fence(f);
- return (const char *)fence->ring->name;
-}
-
-static inline bool amdgpu_test_signaled(struct amdgpu_fence *fence)
-{
- return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags);
-}
-
-static bool amdgpu_test_signaled_any(struct fence **fences, uint32_t count)
-{
- int idx;
- struct fence *fence;
-
- for (idx = 0; idx < count; ++idx) {
- fence = fences[idx];
- if (fence) {
- if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
- return true;
- }
- }
- return false;
-}
-
-struct amdgpu_wait_cb {
- struct fence_cb base;
- struct task_struct *task;
-};
-
-static void amdgpu_fence_wait_cb(struct fence *fence, struct fence_cb *cb)
-{
- struct amdgpu_wait_cb *wait =
- container_of(cb, struct amdgpu_wait_cb, base);
- wake_up_process(wait->task);
-}
-
-static signed long amdgpu_fence_default_wait(struct fence *f, bool intr,
- signed long t)
-{
- struct amdgpu_fence *fence = to_amdgpu_fence(f);
- struct amdgpu_device *adev = fence->ring->adev;
-
- return amdgpu_fence_wait_any(adev, &f, 1, intr, t);
-}
-
-/**
- * Wait the fence array with timeout
- *
- * @adev: amdgpu device
- * @array: the fence array with amdgpu fence pointer
- * @count: the number of the fence array
- * @intr: when sleep, set the current task interruptable or not
- * @t: timeout to wait
- *
- * It will return when any fence is signaled or timeout.
- */
-signed long amdgpu_fence_wait_any(struct amdgpu_device *adev,
- struct fence **array, uint32_t count,
- bool intr, signed long t)
-{
- struct amdgpu_wait_cb *cb;
- struct fence *fence;
- unsigned idx;
-
- BUG_ON(!array);
-
- cb = kcalloc(count, sizeof(struct amdgpu_wait_cb), GFP_KERNEL);
- if (cb == NULL) {
- t = -ENOMEM;
- goto err_free_cb;
- }
-
- for (idx = 0; idx < count; ++idx) {
- fence = array[idx];
- if (fence) {
- cb[idx].task = current;
- if (fence_add_callback(fence,
- &cb[idx].base, amdgpu_fence_wait_cb)) {
- /* The fence is already signaled */
- goto fence_rm_cb;
- }
- }
- }
-
- while (t > 0) {
- if (intr)
- set_current_state(TASK_INTERRUPTIBLE);
- else
- set_current_state(TASK_UNINTERRUPTIBLE);
-
- /*
- * amdgpu_test_signaled_any must be called after
- * set_current_state to prevent a race with wake_up_process
- */
- if (amdgpu_test_signaled_any(array, count))
- break;
-
- if (adev->needs_reset) {
- t = -EDEADLK;
- break;
- }
-
- t = schedule_timeout(t);
-
- if (t > 0 && intr && signal_pending(current))
- t = -ERESTARTSYS;
- }
-
- __set_current_state(TASK_RUNNING);
-
-fence_rm_cb:
- for (idx = 0; idx < count; ++idx) {
- fence = array[idx];
- if (fence && cb[idx].base.func)
- fence_remove_callback(fence, &cb[idx].base);
- }
-
-err_free_cb:
- kfree(cb);
-
- return t;
-}
-
-const struct fence_ops amdgpu_fence_ops = {
- .get_driver_name = amdgpu_fence_get_driver_name,
- .get_timeline_name = amdgpu_fence_get_timeline_name,
- .enable_signaling = amdgpu_fence_enable_signaling,
- .signaled = amdgpu_fence_is_signaled,
- .wait = amdgpu_fence_default_wait,
- .release = NULL,
-};
r = amdgpu_bo_create(adev, adev->gart.table_size,
PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->gart.robj);
+ NULL, NULL, &adev->gart.robj);
if (r) {
return r;
}
}
}
retry:
- r = amdgpu_bo_create(adev, size, alignment, kernel, initial_domain, flags, NULL, &robj);
+ r = amdgpu_bo_create(adev, size, alignment, kernel, initial_domain,
+ flags, NULL, NULL, &robj);
if (r) {
if (r != -ERESTARTSYS) {
if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
struct amdgpu_vm *vm = &fpriv->vm;
struct amdgpu_bo_va *bo_va;
int r;
-
+ mutex_lock(&vm->mutex);
r = amdgpu_bo_reserve(rbo, false);
if (r) {
+ mutex_unlock(&vm->mutex);
return r;
}
++bo_va->ref_count;
}
amdgpu_bo_unreserve(rbo);
-
+ mutex_unlock(&vm->mutex);
return 0;
}
struct amdgpu_vm *vm = &fpriv->vm;
struct amdgpu_bo_va *bo_va;
int r;
-
+ mutex_lock(&vm->mutex);
r = amdgpu_bo_reserve(rbo, true);
if (r) {
+ mutex_unlock(&vm->mutex);
dev_err(adev->dev, "leaking bo va because "
"we fail to reserve bo (%d)\n", r);
return;
}
}
amdgpu_bo_unreserve(rbo);
+ mutex_unlock(&vm->mutex);
}
static int amdgpu_gem_handle_lockup(struct amdgpu_device *adev, int r)
bool kernel = false;
int r;
- down_read(&adev->exclusive_lock);
/* create a gem object to contain this object in */
if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
memset(args, 0, sizeof(*args));
args->out.handle = handle;
- up_read(&adev->exclusive_lock);
return 0;
error_unlock:
- up_read(&adev->exclusive_lock);
r = amdgpu_gem_handle_lockup(adev, r);
return r;
}
return -EACCES;
}
- down_read(&adev->exclusive_lock);
-
/* create a gem object to contain this object in */
r = amdgpu_gem_object_create(adev, args->size, 0,
AMDGPU_GEM_DOMAIN_CPU, 0,
goto handle_lockup;
args->handle = handle;
- up_read(&adev->exclusive_lock);
return 0;
release_object:
drm_gem_object_unreference_unlocked(gobj);
handle_lockup:
- up_read(&adev->exclusive_lock);
r = amdgpu_gem_handle_lockup(adev, r);
return r;
&args->data.data_size_bytes,
&args->data.flags);
} else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
+ if (args->data.data_size_bytes > sizeof(args->data.data)) {
+ r = -EINVAL;
+ goto unreserve;
+ }
r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
if (!r)
r = amdgpu_bo_set_metadata(robj, args->data.data,
args->data.flags);
}
+unreserve:
amdgpu_bo_unreserve(robj);
out:
drm_gem_object_unreference_unlocked(gobj);
struct ttm_validate_buffer tv, *entry;
struct amdgpu_bo_list_entry *vm_bos;
struct ww_acquire_ctx ticket;
- struct list_head list;
+ struct list_head list, duplicates;
unsigned domain;
int r;
INIT_LIST_HEAD(&list);
+ INIT_LIST_HEAD(&duplicates);
tv.bo = &bo_va->bo->tbo;
tv.shared = true;
if (!vm_bos)
return;
- r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL);
+ /* Provide duplicates to avoid -EALREADY */
+ r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
if (r)
goto error_free;
goto error_unreserve;
}
- mutex_lock(&bo_va->vm->mutex);
r = amdgpu_vm_clear_freed(adev, bo_va->vm);
if (r)
- goto error_unlock;
-
+ goto error_unreserve;
if (operation == AMDGPU_VA_OP_MAP)
r = amdgpu_vm_bo_update(adev, bo_va, &bo_va->bo->tbo.mem);
-error_unlock:
- mutex_unlock(&bo_va->vm->mutex);
-
error_unreserve:
ttm_eu_backoff_reservation(&ticket, &list);
gobj = drm_gem_object_lookup(dev, filp, args->handle);
if (gobj == NULL)
return -ENOENT;
-
+ mutex_lock(&fpriv->vm.mutex);
rbo = gem_to_amdgpu_bo(gobj);
r = amdgpu_bo_reserve(rbo, false);
if (r) {
+ mutex_unlock(&fpriv->vm.mutex);
drm_gem_object_unreference_unlocked(gobj);
return r;
}
bo_va = amdgpu_vm_bo_find(&fpriv->vm, rbo);
if (!bo_va) {
amdgpu_bo_unreserve(rbo);
+ mutex_unlock(&fpriv->vm.mutex);
return -ENOENT;
}
if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE))
amdgpu_gem_va_update_vm(adev, bo_va, args->operation);
-
+ mutex_unlock(&fpriv->vm.mutex);
drm_gem_object_unreference_unlocked(gobj);
return r;
}
int r;
args->pitch = amdgpu_align_pitch(adev, args->width, args->bpp, 0) * ((args->bpp + 1) / 8);
- args->size = args->pitch * args->height;
+ args->size = (u64)args->pitch * args->height;
args->size = ALIGN(args->size, PAGE_SIZE);
r = amdgpu_gem_object_create(adev, args->size, 0,
{
amdgpu_sync_free(adev, &ib->sync, &ib->fence->base);
amdgpu_sa_bo_free(adev, &ib->sa_bo, &ib->fence->base);
- amdgpu_fence_unref(&ib->fence);
+ if (ib->fence)
+ fence_put(&ib->fence->base);
}
/**
r = amdgpu_ring_test_ib(ring);
if (r) {
ring->ready = false;
- adev->needs_reset = false;
if (ring == &adev->gfx.gfx_ring[0]) {
/* oh, oh, that's really bad */
r = amdgpu_bo_create(adev, adev->irq.ih.ring_size,
PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_GTT, 0,
- NULL, &adev->irq.ih.ring_obj);
+ NULL, NULL, &adev->irq.ih.ring_obj);
if (r) {
DRM_ERROR("amdgpu: failed to create ih ring buffer (%d).\n", r);
return r;
*/
int amdgpu_irq_postinstall(struct drm_device *dev)
{
- dev->max_vblank_count = 0x001fffff;
+ dev->max_vblank_count = 0x00ffffff;
return 0;
}
break;
case AMDGPU_HW_IP_DMA:
type = AMD_IP_BLOCK_TYPE_SDMA;
- ring_mask = adev->sdma[0].ring.ready ? 1 : 0;
- ring_mask |= ((adev->sdma[1].ring.ready ? 1 : 0) << 1);
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ ring_mask |= ((adev->sdma.instance[i].ring.ready ? 1 : 0) << i);
ib_start_alignment = AMDGPU_GPU_PAGE_SIZE;
ib_size_alignment = 1;
break;
fw_info.feature = 0;
break;
case AMDGPU_INFO_FW_SDMA:
- if (info->query_fw.index >= 2)
+ if (info->query_fw.index >= adev->sdma.num_instances)
return -EINVAL;
- fw_info.ver = adev->sdma[info->query_fw.index].fw_version;
- fw_info.feature = adev->sdma[info->query_fw.index].feature_version;
+ fw_info.ver = adev->sdma.instance[info->query_fw.index].fw_version;
+ fw_info.feature = adev->sdma.instance[info->query_fw.index].feature_version;
break;
default:
return -EINVAL;
min((size_t)size, sizeof(vram_gtt))) ? -EFAULT : 0;
}
case AMDGPU_INFO_READ_MMR_REG: {
- unsigned n, alloc_size = info->read_mmr_reg.count * 4;
+ unsigned n, alloc_size;
uint32_t *regs;
unsigned se_num = (info->read_mmr_reg.instance >>
AMDGPU_INFO_MMR_SE_INDEX_SHIFT) &
if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK)
sh_num = 0xffffffff;
- regs = kmalloc(alloc_size, GFP_KERNEL);
+ regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), GFP_KERNEL);
if (!regs)
return -ENOMEM;
+ alloc_size = info->read_mmr_reg.count * sizeof(*regs);
for (i = 0; i < info->read_mmr_reg.count; i++)
if (amdgpu_asic_read_register(adev, se_num, sh_num,
* Outdated mess for old drm with Xorg being in charge (void function now).
*/
/**
- * amdgpu_driver_firstopen_kms - drm callback for last close
+ * amdgpu_driver_lastclose_kms - drm callback for last close
*
* @dev: drm dev pointer
*
- * Switch vga switcheroo state after last close (all asics).
+ * Switch vga_switcheroo state after last close (all asics).
*/
void amdgpu_driver_lastclose_kms(struct drm_device *dev)
{
+ struct amdgpu_device *adev = dev->dev_private;
+
+ amdgpu_fbdev_restore_mode(adev);
vga_switcheroo_process_delayed_switch();
}
* amdgpu_get_vblank_counter_kms - get frame count
*
* @dev: drm dev pointer
- * @crtc: crtc to get the frame count from
+ * @pipe: crtc to get the frame count from
*
* Gets the frame count on the requested crtc (all asics).
* Returns frame count on success, -EINVAL on failure.
*/
-u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, int crtc)
+u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe)
{
struct amdgpu_device *adev = dev->dev_private;
- if (crtc < 0 || crtc >= adev->mode_info.num_crtc) {
- DRM_ERROR("Invalid crtc %d\n", crtc);
+ if (pipe >= adev->mode_info.num_crtc) {
+ DRM_ERROR("Invalid crtc %u\n", pipe);
return -EINVAL;
}
- return amdgpu_display_vblank_get_counter(adev, crtc);
+ return amdgpu_display_vblank_get_counter(adev, pipe);
}
/**
* amdgpu_enable_vblank_kms - enable vblank interrupt
*
* @dev: drm dev pointer
- * @crtc: crtc to enable vblank interrupt for
+ * @pipe: crtc to enable vblank interrupt for
*
* Enable the interrupt on the requested crtc (all asics).
* Returns 0 on success, -EINVAL on failure.
*/
-int amdgpu_enable_vblank_kms(struct drm_device *dev, int crtc)
+int amdgpu_enable_vblank_kms(struct drm_device *dev, unsigned int pipe)
{
struct amdgpu_device *adev = dev->dev_private;
- int idx = amdgpu_crtc_idx_to_irq_type(adev, crtc);
+ int idx = amdgpu_crtc_idx_to_irq_type(adev, pipe);
return amdgpu_irq_get(adev, &adev->crtc_irq, idx);
}
* amdgpu_disable_vblank_kms - disable vblank interrupt
*
* @dev: drm dev pointer
- * @crtc: crtc to disable vblank interrupt for
+ * @pipe: crtc to disable vblank interrupt for
*
* Disable the interrupt on the requested crtc (all asics).
*/
-void amdgpu_disable_vblank_kms(struct drm_device *dev, int crtc)
+void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe)
{
struct amdgpu_device *adev = dev->dev_private;
- int idx = amdgpu_crtc_idx_to_irq_type(adev, crtc);
+ int idx = amdgpu_crtc_idx_to_irq_type(adev, pipe);
amdgpu_irq_put(adev, &adev->crtc_irq, idx);
}
* scanout position. (all asics).
* Returns postive status flags on success, negative error on failure.
*/
-int amdgpu_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
+int amdgpu_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int pipe,
int *max_error,
struct timeval *vblank_time,
unsigned flags)
{
- struct drm_crtc *drmcrtc;
+ struct drm_crtc *crtc;
struct amdgpu_device *adev = dev->dev_private;
- if (crtc < 0 || crtc >= dev->num_crtcs) {
- DRM_ERROR("Invalid crtc %d\n", crtc);
+ if (pipe >= dev->num_crtcs) {
+ DRM_ERROR("Invalid crtc %u\n", pipe);
return -EINVAL;
}
/* Get associated drm_crtc: */
- drmcrtc = &adev->mode_info.crtcs[crtc]->base;
+ crtc = &adev->mode_info.crtcs[pipe]->base;
/* Helper routine in DRM core does all the work: */
- return drm_calc_vbltimestamp_from_scanoutpos(dev, crtc, max_error,
+ return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
vblank_time, flags,
- drmcrtc, &drmcrtc->hwmode);
+ &crtc->hwmode);
}
const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
- DRM_IOCTL_DEF_DRV(AMDGPU_GEM_CREATE, amdgpu_gem_create_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_CREATE, amdgpu_gem_create_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
/* KMS */
- DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_CS, amdgpu_cs_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_INFO, amdgpu_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_CS, amdgpu_cs_wait_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_GEM_METADATA, amdgpu_gem_metadata_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_GEM_VA, amdgpu_gem_va_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_GEM_OP, amdgpu_gem_op_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_CS, amdgpu_cs_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_INFO, amdgpu_info_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_CS, amdgpu_cs_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_METADATA, amdgpu_gem_metadata_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_VA, amdgpu_gem_va_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_OP, amdgpu_gem_op_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
};
int amdgpu_max_kms_ioctl = ARRAY_SIZE(amdgpu_ioctls_kms);
uint32_t crtc_offset;
struct drm_gem_object *cursor_bo;
uint64_t cursor_addr;
+ int cursor_x;
+ int cursor_y;
+ int cursor_hot_x;
+ int cursor_hot_y;
int cursor_width;
int cursor_height;
int max_cursor_width;
void amdgpu_encoder_set_active_device(struct drm_encoder *encoder);
-int amdgpu_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
- unsigned int flags,
- int *vpos, int *hpos, ktime_t *stime,
- ktime_t *etime);
+int amdgpu_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
+ unsigned int flags, int *vpos, int *hpos,
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode);
int amdgpu_framebuffer_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state);
int amdgpu_fbdev_total_size(struct amdgpu_device *adev);
bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj);
+void amdgpu_fbdev_restore_mode(struct amdgpu_device *adev);
void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev);
placements[c].fpfn = 0;
placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
TTM_PL_FLAG_VRAM;
+ if (!(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED))
+ placements[c - 1].flags |= TTM_PL_FLAG_TOPDOWN;
}
if (domain & AMDGPU_GEM_DOMAIN_GTT) {
bool kernel, u32 domain, u64 flags,
struct sg_table *sg,
struct ttm_placement *placement,
+ struct reservation_object *resv,
struct amdgpu_bo **bo_ptr)
{
struct amdgpu_bo *bo;
/* Kernel allocation are uninterruptible */
r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type,
&bo->placement, page_align, !kernel, NULL,
- acc_size, sg, NULL, &amdgpu_ttm_bo_destroy);
+ acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
if (unlikely(r != 0)) {
return r;
}
int amdgpu_bo_create(struct amdgpu_device *adev,
unsigned long size, int byte_align,
bool kernel, u32 domain, u64 flags,
- struct sg_table *sg, struct amdgpu_bo **bo_ptr)
+ struct sg_table *sg,
+ struct reservation_object *resv,
+ struct amdgpu_bo **bo_ptr)
{
struct ttm_placement placement = {0};
struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
amdgpu_ttm_placement_init(adev, &placement,
placements, domain, flags);
- return amdgpu_bo_create_restricted(adev, size, byte_align,
- kernel, domain, flags,
- sg,
- &placement,
- bo_ptr);
+ return amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
+ domain, flags, sg, &placement,
+ resv, bo_ptr);
}
int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
if (metadata == NULL)
return -EINVAL;
- buffer = kzalloc(metadata_size, GFP_KERNEL);
+ buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
if (buffer == NULL)
return -ENOMEM;
- memcpy(buffer, metadata, metadata_size);
-
kfree(bo->metadata);
bo->metadata_flags = flags;
bo->metadata = buffer;
unsigned long size, int byte_align,
bool kernel, u32 domain, u64 flags,
struct sg_table *sg,
+ struct reservation_object *resv,
struct amdgpu_bo **bo_ptr);
int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
unsigned long size, int byte_align,
bool kernel, u32 domain, u64 flags,
struct sg_table *sg,
struct ttm_placement *placement,
+ struct reservation_object *resv,
struct amdgpu_bo **bo_ptr);
int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);
void amdgpu_bo_kunmap(struct amdgpu_bo *bo);
struct dma_buf_attachment *attach,
struct sg_table *sg)
{
+ struct reservation_object *resv = attach->dmabuf->resv;
struct amdgpu_device *adev = dev->dev_private;
struct amdgpu_bo *bo;
int ret;
+ ww_mutex_lock(&resv->lock, NULL);
ret = amdgpu_bo_create(adev, attach->dmabuf->size, PAGE_SIZE, false,
- AMDGPU_GEM_DOMAIN_GTT, 0, sg, &bo);
+ AMDGPU_GEM_DOMAIN_GTT, 0, sg, resv, &bo);
+ ww_mutex_unlock(&resv->lock);
if (ret)
return ERR_PTR(ret);
if (!ring->ring_free_dw) {
/* this is an empty ring */
ring->ring_free_dw = ring->ring_size / 4;
- /* update lockup info to avoid false positive */
- amdgpu_ring_lockup_update(ring);
}
}
mutex_unlock(ring->ring_lock);
}
-/**
- * amdgpu_ring_lockup_update - update lockup variables
- *
- * @ring: amdgpu_ring structure holding ring information
- *
- * Update the last rptr value and timestamp (all asics).
- */
-void amdgpu_ring_lockup_update(struct amdgpu_ring *ring)
-{
- atomic_set(&ring->last_rptr, amdgpu_ring_get_rptr(ring));
- atomic64_set(&ring->last_activity, jiffies_64);
-}
-
-/**
- * amdgpu_ring_test_lockup() - check if ring is lockedup by recording information
- * @ring: amdgpu_ring structure holding ring information
- *
- */
-bool amdgpu_ring_test_lockup(struct amdgpu_ring *ring)
-{
- uint32_t rptr = amdgpu_ring_get_rptr(ring);
- uint64_t last = atomic64_read(&ring->last_activity);
- uint64_t elapsed;
-
- if (rptr != atomic_read(&ring->last_rptr)) {
- /* ring is still working, no lockup */
- amdgpu_ring_lockup_update(ring);
- return false;
- }
-
- elapsed = jiffies_to_msecs(jiffies_64 - last);
- if (amdgpu_lockup_timeout && elapsed >= amdgpu_lockup_timeout) {
- dev_err(ring->adev->dev, "ring %d stalled for more than %llumsec\n",
- ring->idx, elapsed);
- return true;
- }
- /* give a chance to the GPU ... */
- return false;
-}
-
/**
* amdgpu_ring_backup - Back up the content of a ring
*
ring->adev = adev;
ring->idx = adev->num_rings++;
adev->rings[ring->idx] = ring;
- amdgpu_fence_driver_init_ring(ring);
+ r = amdgpu_fence_driver_init_ring(ring);
+ if (r)
+ return r;
}
- init_waitqueue_head(&ring->fence_drv.fence_queue);
-
r = amdgpu_wb_get(adev, &ring->rptr_offs);
if (r) {
dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
if (ring->ring_obj == NULL) {
r = amdgpu_bo_create(adev, ring->ring_size, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_GTT, 0,
- NULL, &ring->ring_obj);
+ NULL, NULL, &ring->ring_obj);
if (r) {
dev_err(adev->dev, "(%d) ring create failed\n", r);
return r;
if (amdgpu_debugfs_ring_init(adev, ring)) {
DRM_ERROR("Failed to register debugfs file for rings !\n");
}
- amdgpu_ring_lockup_update(ring);
return 0;
}
}
}
+/**
+ * amdgpu_ring_from_fence - get ring from fence
+ *
+ * @f: fence structure
+ *
+ * Extract the ring a fence belongs to. Handles both scheduler as
+ * well as hardware fences.
+ */
+struct amdgpu_ring *amdgpu_ring_from_fence(struct fence *f)
+{
+ struct amdgpu_fence *a_fence;
+ struct amd_sched_fence *s_fence;
+
+ s_fence = to_amd_sched_fence(f);
+ if (s_fence)
+ return container_of(s_fence->sched, struct amdgpu_ring, sched);
+
+ a_fence = to_amdgpu_fence(f);
+ if (a_fence)
+ return a_fence->ring;
+
+ return NULL;
+}
+
/*
* Debugfs info
*/
static int amdgpu_gfx_index = offsetof(struct amdgpu_device, gfx.gfx_ring[0]);
static int cayman_cp1_index = offsetof(struct amdgpu_device, gfx.compute_ring[0]);
static int cayman_cp2_index = offsetof(struct amdgpu_device, gfx.compute_ring[1]);
-static int amdgpu_dma1_index = offsetof(struct amdgpu_device, sdma[0].ring);
-static int amdgpu_dma2_index = offsetof(struct amdgpu_device, sdma[1].ring);
+static int amdgpu_dma1_index = offsetof(struct amdgpu_device, sdma.instance[0].ring);
+static int amdgpu_dma2_index = offsetof(struct amdgpu_device, sdma.instance[1].ring);
static int r600_uvd_index = offsetof(struct amdgpu_device, uvd.ring);
static int si_vce1_index = offsetof(struct amdgpu_device, vce.ring[0]);
static int si_vce2_index = offsetof(struct amdgpu_device, vce.ring[1]);
INIT_LIST_HEAD(&sa_manager->flist[i]);
}
- r = amdgpu_bo_create(adev, size, align, true,
- domain, 0, NULL, &sa_manager->bo);
+ r = amdgpu_bo_create(adev, size, align, true, domain,
+ 0, NULL, NULL, &sa_manager->bo);
if (r) {
dev_err(adev->dev, "(%d) failed to allocate bo for manager\n", r);
return r;
return r;
}
-static uint32_t amdgpu_sa_get_ring_from_fence(struct fence *f)
-{
- struct amdgpu_fence *a_fence;
- struct amd_sched_fence *s_fence;
-
- s_fence = to_amd_sched_fence(f);
- if (s_fence)
- return s_fence->scheduler->ring_id;
- a_fence = to_amdgpu_fence(f);
- if (a_fence)
- return a_fence->ring->idx;
- return 0;
-}
-
static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo)
{
struct amdgpu_sa_manager *sa_manager = sa_bo->manager;
}
if (best_bo) {
- uint32_t idx = amdgpu_sa_get_ring_from_fence(best_bo->fence);
+ uint32_t idx = amdgpu_ring_from_fence(best_bo->fence)->idx;
++tries[idx];
sa_manager->hole = best_bo->olist.prev;
{
struct fence *fences[AMDGPU_MAX_RINGS];
unsigned tries[AMDGPU_MAX_RINGS];
+ unsigned count;
int i, r;
signed long t;
/* see if we can skip over some allocations */
} while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries));
- spin_unlock(&sa_manager->wq.lock);
- t = amdgpu_fence_wait_any(adev, fences, AMDGPU_MAX_RINGS,
- false, MAX_SCHEDULE_TIMEOUT);
- r = (t > 0) ? 0 : t;
- spin_lock(&sa_manager->wq.lock);
- /* if we have nothing to wait for block */
- if (r == -ENOENT) {
+ for (i = 0, count = 0; i < AMDGPU_MAX_RINGS; ++i)
+ if (fences[i])
+ fences[count++] = fences[i];
+
+ if (count) {
+ spin_unlock(&sa_manager->wq.lock);
+ t = fence_wait_any_timeout(fences, count, false,
+ MAX_SCHEDULE_TIMEOUT);
+ r = (t > 0) ? 0 : t;
+ spin_lock(&sa_manager->wq.lock);
+ } else {
+ /* if we have nothing to wait for block */
r = wait_event_interruptible_locked(
sa_manager->wq,
amdgpu_sa_event(sa_manager, size, align)
if (fence && !fence_is_signaled(fence)) {
uint32_t idx;
(*sa_bo)->fence = fence_get(fence);
- idx = amdgpu_sa_get_ring_from_fence(fence);
+ idx = amdgpu_ring_from_fence(fence)->idx;
list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]);
} else {
amdgpu_sa_bo_remove_locked(*sa_bo);
}
#if defined(CONFIG_DEBUG_FS)
+
+static void amdgpu_sa_bo_dump_fence(struct fence *fence, struct seq_file *m)
+{
+ struct amdgpu_fence *a_fence = to_amdgpu_fence(fence);
+ struct amd_sched_fence *s_fence = to_amd_sched_fence(fence);
+
+ if (a_fence)
+ seq_printf(m, " protected by 0x%016llx on ring %d",
+ a_fence->seq, a_fence->ring->idx);
+
+ if (s_fence) {
+ struct amdgpu_ring *ring;
+
+
+ ring = container_of(s_fence->sched, struct amdgpu_ring, sched);
+ seq_printf(m, " protected by 0x%016x on ring %d",
+ s_fence->base.seqno, ring->idx);
+ }
+}
+
void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
struct seq_file *m)
{
}
seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
soffset, eoffset, eoffset - soffset);
- if (i->fence) {
- struct amdgpu_fence *a_fence = to_amdgpu_fence(i->fence);
- struct amd_sched_fence *s_fence = to_amd_sched_fence(i->fence);
- if (a_fence)
- seq_printf(m, " protected by 0x%016llx on ring %d",
- a_fence->seq, a_fence->ring->idx);
- if (s_fence)
- seq_printf(m, " protected by 0x%016x on ring %d",
- s_fence->base.seqno,
- s_fence->scheduler->ring_id);
-
- }
+ if (i->fence)
+ amdgpu_sa_bo_dump_fence(i->fence, m);
seq_printf(m, "\n");
}
spin_unlock(&sa_manager->wq.lock);
#include <drm/drmP.h>
#include "amdgpu.h"
-static struct fence *amdgpu_sched_dependency(struct amd_sched_job *job)
+static struct fence *amdgpu_sched_dependency(struct amd_sched_job *sched_job)
{
- struct amdgpu_job *sched_job = (struct amdgpu_job *)job;
- return amdgpu_sync_get_fence(&sched_job->ibs->sync);
+ struct amdgpu_job *job = to_amdgpu_job(sched_job);
+ return amdgpu_sync_get_fence(&job->ibs->sync);
}
-static struct fence *amdgpu_sched_run_job(struct amd_sched_job *job)
+static struct fence *amdgpu_sched_run_job(struct amd_sched_job *sched_job)
{
- struct amdgpu_job *sched_job;
- struct amdgpu_fence *fence;
+ struct amdgpu_fence *fence = NULL;
+ struct amdgpu_job *job;
int r;
- if (!job) {
+ if (!sched_job) {
DRM_ERROR("job is null\n");
return NULL;
}
- sched_job = (struct amdgpu_job *)job;
- mutex_lock(&sched_job->job_lock);
- r = amdgpu_ib_schedule(sched_job->adev,
- sched_job->num_ibs,
- sched_job->ibs,
- sched_job->base.owner);
- if (r)
+ job = to_amdgpu_job(sched_job);
+ mutex_lock(&job->job_lock);
+ r = amdgpu_ib_schedule(job->adev,
+ job->num_ibs,
+ job->ibs,
+ job->base.owner);
+ if (r) {
+ DRM_ERROR("Error scheduling IBs (%d)\n", r);
goto err;
- fence = amdgpu_fence_ref(sched_job->ibs[sched_job->num_ibs - 1].fence);
-
- if (sched_job->free_job)
- sched_job->free_job(sched_job);
+ }
- mutex_unlock(&sched_job->job_lock);
- return &fence->base;
+ fence = job->ibs[job->num_ibs - 1].fence;
+ fence_get(&fence->base);
err:
- DRM_ERROR("Run job error\n");
- mutex_unlock(&sched_job->job_lock);
- job->sched->ops->process_job(job);
- return NULL;
-}
+ if (job->free_job)
+ job->free_job(job);
-static void amdgpu_sched_process_job(struct amd_sched_job *job)
-{
- struct amdgpu_job *sched_job;
-
- if (!job) {
- DRM_ERROR("job is null\n");
- return;
- }
- sched_job = (struct amdgpu_job *)job;
- /* after processing job, free memory */
- fence_put(&sched_job->base.s_fence->base);
- kfree(sched_job);
+ mutex_unlock(&job->job_lock);
+ fence_put(&job->base.s_fence->base);
+ kfree(job);
+ return fence ? &fence->base : NULL;
}
struct amd_sched_backend_ops amdgpu_sched_ops = {
.dependency = amdgpu_sched_dependency,
.run_job = amdgpu_sched_run_job,
- .process_job = amdgpu_sched_process_job
};
int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev,
kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL);
if (!job)
return -ENOMEM;
- job->base.sched = ring->scheduler;
+ job->base.sched = &ring->sched;
job->base.s_entity = &adev->kernel_ctx.rings[ring->idx].entity;
job->adev = adev;
job->ibs = ibs;
mutex_init(&job->job_lock);
job->free_job = free_job;
mutex_lock(&job->job_lock);
- r = amd_sched_entity_push_job((struct amd_sched_job *)job);
+ r = amd_sched_entity_push_job(&job->base);
if (r) {
mutex_unlock(&job->job_lock);
kfree(job);
if (a_fence)
return a_fence->ring->adev == adev;
- if (s_fence)
- return (struct amdgpu_device *)s_fence->scheduler->priv == adev;
+
+ if (s_fence) {
+ struct amdgpu_ring *ring;
+
+ ring = container_of(s_fence->sched, struct amdgpu_ring, sched);
+ return ring->adev == adev;
+ }
+
return false;
}
return false;
}
+static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence)
+{
+ if (*keep && fence_is_later(*keep, fence))
+ return;
+
+ fence_put(*keep);
+ *keep = fence_get(fence);
+}
+
/**
* amdgpu_sync_fence - remember to sync to this fence
*
{
struct amdgpu_sync_entry *e;
struct amdgpu_fence *fence;
- struct amdgpu_fence *other;
- struct fence *tmp, *later;
if (!f)
return 0;
if (amdgpu_sync_same_dev(adev, f) &&
- amdgpu_sync_test_owner(f, AMDGPU_FENCE_OWNER_VM)) {
- if (sync->last_vm_update) {
- tmp = sync->last_vm_update;
- BUG_ON(f->context != tmp->context);
- later = (f->seqno - tmp->seqno <= INT_MAX) ? f : tmp;
- sync->last_vm_update = fence_get(later);
- fence_put(tmp);
- } else
- sync->last_vm_update = fence_get(f);
- }
+ amdgpu_sync_test_owner(f, AMDGPU_FENCE_OWNER_VM))
+ amdgpu_sync_keep_later(&sync->last_vm_update, f);
fence = to_amdgpu_fence(f);
if (!fence || fence->ring->adev != adev) {
hash_for_each_possible(sync->fences, e, node, f->context) {
- struct fence *new;
if (unlikely(e->fence->context != f->context))
continue;
- new = fence_get(fence_later(e->fence, f));
- if (new) {
- fence_put(e->fence);
- e->fence = new;
- }
+
+ amdgpu_sync_keep_later(&e->fence, f);
return 0;
}
return 0;
}
- other = sync->sync_to[fence->ring->idx];
- sync->sync_to[fence->ring->idx] = amdgpu_fence_ref(
- amdgpu_fence_later(fence, other));
- amdgpu_fence_unref(&other);
+ amdgpu_sync_keep_later(&sync->sync_to[fence->ring->idx], f);
return 0;
}
* for other VM updates and moves.
*/
fence_owner = amdgpu_sync_get_owner(f);
- if ((owner != AMDGPU_FENCE_OWNER_MOVE) &&
- (fence_owner != AMDGPU_FENCE_OWNER_MOVE) &&
+ if ((owner != AMDGPU_FENCE_OWNER_UNDEFINED) &&
+ (fence_owner != AMDGPU_FENCE_OWNER_UNDEFINED) &&
((owner == AMDGPU_FENCE_OWNER_VM) !=
(fence_owner == AMDGPU_FENCE_OWNER_VM)))
continue;
fence_put(e->fence);
kfree(e);
}
+
+ if (amdgpu_enable_semaphores)
+ return 0;
+
+ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+ struct fence *fence = sync->sync_to[i];
+ if (!fence)
+ continue;
+
+ r = fence_wait(fence, false);
+ if (r)
+ return r;
+ }
+
return 0;
}
int i, r;
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- struct amdgpu_fence *fence = sync->sync_to[i];
- struct amdgpu_semaphore *semaphore;
struct amdgpu_ring *other = adev->rings[i];
+ struct amdgpu_semaphore *semaphore;
+ struct amdgpu_fence *fence;
+
+ if (!sync->sync_to[i])
+ continue;
+
+ fence = to_amdgpu_fence(sync->sync_to[i]);
/* check if we really need to sync */
if (!amdgpu_fence_need_sync(fence, ring))
return -EINVAL;
}
- if (amdgpu_enable_scheduler || (count >= AMDGPU_NUM_SYNCS)) {
+ if (amdgpu_enable_scheduler || !amdgpu_enable_semaphores ||
+ (count >= AMDGPU_NUM_SYNCS)) {
/* not enough room, wait manually */
r = fence_wait(&fence->base, false);
if (r)
amdgpu_semaphore_free(adev, &sync->semaphores[i], fence);
for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
- amdgpu_fence_unref(&sync->sync_to[i]);
+ fence_put(sync->sync_to[i]);
fence_put(sync->last_vm_update);
}
goto out_cleanup;
}
- r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, 0,
- NULL, &vram_obj);
+ r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
+ AMDGPU_GEM_DOMAIN_VRAM, 0,
+ NULL, NULL, &vram_obj);
if (r) {
DRM_ERROR("Failed to create VRAM object\n");
goto out_cleanup;
struct fence *fence = NULL;
r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GTT, 0, NULL, gtt_obj + i);
+ AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
+ NULL, gtt_obj + i);
if (r) {
DRM_ERROR("Failed to create GTT object %d\n", i);
goto out_lclean;
__entry->offset, __entry->flags)
);
-TRACE_EVENT(amdgpu_vm_bo_update,
+DECLARE_EVENT_CLASS(amdgpu_vm_mapping,
TP_PROTO(struct amdgpu_bo_va_mapping *mapping),
TP_ARGS(mapping),
TP_STRUCT__entry(
__entry->soffset, __entry->eoffset, __entry->flags)
);
+DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_update,
+ TP_PROTO(struct amdgpu_bo_va_mapping *mapping),
+ TP_ARGS(mapping)
+);
+
+DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_mapping,
+ TP_PROTO(struct amdgpu_bo_va_mapping *mapping),
+ TP_ARGS(mapping)
+);
+
TRACE_EVENT(amdgpu_vm_set_page,
TP_PROTO(uint64_t pe, uint64_t addr, unsigned count,
uint32_t incr, uint32_t flags),
r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->stollen_vga_memory);
+ NULL, NULL, &adev->stollen_vga_memory);
if (r) {
return r;
}
WARN_ON(ib->length_dw > num_dw);
r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
&amdgpu_vm_free_job,
- AMDGPU_FENCE_OWNER_MOVE,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
fence);
if (r)
goto error_free;
spin_lock(&glob->lru_lock);
ret = drm_mm_dump_table(m, mm);
spin_unlock(&glob->lru_lock);
+ if (ttm_pl == TTM_PL_VRAM)
+ seq_printf(m, "man size:%llu pages, ram usage:%luMB, vis usage:%luMB\n",
+ adev->mman.bdev.man[ttm_pl].size,
+ atomic64_read(&adev->vram_usage) >> 20,
+ atomic64_read(&adev->vram_vis_usage) >> 20);
return ret;
}
const struct common_firmware_header *header = NULL;
err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GTT, 0, NULL, bo);
+ AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL, bo);
if (err) {
dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
err = -ENOMEM;
#define FIRMWARE_TONGA "amdgpu/tonga_uvd.bin"
#define FIRMWARE_CARRIZO "amdgpu/carrizo_uvd.bin"
#define FIRMWARE_FIJI "amdgpu/fiji_uvd.bin"
+#define FIRMWARE_STONEY "amdgpu/stoney_uvd.bin"
/**
* amdgpu_uvd_cs_ctx - Command submission parser context
MODULE_FIRMWARE(FIRMWARE_TONGA);
MODULE_FIRMWARE(FIRMWARE_CARRIZO);
MODULE_FIRMWARE(FIRMWARE_FIJI);
+MODULE_FIRMWARE(FIRMWARE_STONEY);
static void amdgpu_uvd_note_usage(struct amdgpu_device *adev);
static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
case CHIP_CARRIZO:
fw_name = FIRMWARE_CARRIZO;
break;
+ case CHIP_STONEY:
+ fw_name = FIRMWARE_STONEY;
+ break;
default:
return -EINVAL;
}
r = amdgpu_bo_create(adev, bo_size, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->uvd.vcpu_bo);
+ NULL, NULL, &adev->uvd.vcpu_bo);
if (r) {
dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r);
return r;
return -EINVAL;
}
- if (msg_type == 1) {
+ switch (msg_type) {
+ case 0:
+ /* it's a create msg, calc image size (width * height) */
+ amdgpu_bo_kunmap(bo);
+
+ /* try to alloc a new handle */
+ for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
+ if (atomic_read(&adev->uvd.handles[i]) == handle) {
+ DRM_ERROR("Handle 0x%x already in use!\n", handle);
+ return -EINVAL;
+ }
+
+ if (!atomic_cmpxchg(&adev->uvd.handles[i], 0, handle)) {
+ adev->uvd.filp[i] = ctx->parser->filp;
+ return 0;
+ }
+ }
+
+ DRM_ERROR("No more free UVD handles!\n");
+ return -EINVAL;
+
+ case 1:
/* it's a decode msg, calc buffer sizes */
r = amdgpu_uvd_cs_msg_decode(msg, ctx->buf_sizes);
amdgpu_bo_kunmap(bo);
if (r)
return r;
- } else if (msg_type == 2) {
+ /* validate the handle */
+ for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
+ if (atomic_read(&adev->uvd.handles[i]) == handle) {
+ if (adev->uvd.filp[i] != ctx->parser->filp) {
+ DRM_ERROR("UVD handle collision detected!\n");
+ return -EINVAL;
+ }
+ return 0;
+ }
+ }
+
+ DRM_ERROR("Invalid UVD handle 0x%x!\n", handle);
+ return -ENOENT;
+
+ case 2:
/* it's a destroy msg, free the handle */
for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i)
atomic_cmpxchg(&adev->uvd.handles[i], handle, 0);
amdgpu_bo_kunmap(bo);
return 0;
- } else {
- /* it's a create msg */
- amdgpu_bo_kunmap(bo);
-
- if (msg_type != 0) {
- DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type);
- return -EINVAL;
- }
-
- /* it's a create msg, no special handling needed */
- }
-
- /* create or decode, validate the handle */
- for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
- if (atomic_read(&adev->uvd.handles[i]) == handle)
- return 0;
- }
- /* handle not found try to alloc a new one */
- for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
- if (!atomic_cmpxchg(&adev->uvd.handles[i], 0, handle)) {
- adev->uvd.filp[i] = ctx->parser->filp;
- return 0;
- }
+ default:
+ DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type);
+ return -EINVAL;
}
-
- DRM_ERROR("No more free UVD handles!\n");
+ BUG();
return -EINVAL;
}
}
static int amdgpu_uvd_free_job(
- struct amdgpu_job *sched_job)
+ struct amdgpu_job *job)
{
- amdgpu_ib_free(sched_job->adev, sched_job->ibs);
- kfree(sched_job->ibs);
+ amdgpu_ib_free(job->adev, job->ibs);
+ kfree(job->ibs);
return 0;
}
r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &bo);
+ NULL, NULL, &bo);
if (r)
return r;
r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &bo);
+ NULL, NULL, &bo);
if (r)
return r;
#define FIRMWARE_TONGA "amdgpu/tonga_vce.bin"
#define FIRMWARE_CARRIZO "amdgpu/carrizo_vce.bin"
#define FIRMWARE_FIJI "amdgpu/fiji_vce.bin"
+#define FIRMWARE_STONEY "amdgpu/stoney_vce.bin"
#ifdef CONFIG_DRM_AMDGPU_CIK
MODULE_FIRMWARE(FIRMWARE_BONAIRE);
MODULE_FIRMWARE(FIRMWARE_TONGA);
MODULE_FIRMWARE(FIRMWARE_CARRIZO);
MODULE_FIRMWARE(FIRMWARE_FIJI);
+MODULE_FIRMWARE(FIRMWARE_STONEY);
static void amdgpu_vce_idle_work_handler(struct work_struct *work);
case CHIP_FIJI:
fw_name = FIRMWARE_FIJI;
break;
+ case CHIP_STONEY:
+ fw_name = FIRMWARE_STONEY;
+ break;
default:
return -EINVAL;
r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->vce.vcpu_bo);
+ NULL, NULL, &adev->vce.vcpu_bo);
if (r) {
dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);
return r;
}
static int amdgpu_vce_free_job(
- struct amdgpu_job *sched_job)
+ struct amdgpu_job *job)
{
- amdgpu_ib_free(sched_job->adev, sched_job->ibs);
- kfree(sched_job->ibs);
+ amdgpu_ib_free(job->adev, job->ibs);
+ kfree(job->ibs);
return 0;
}
struct amdgpu_bo_list_entry *list;
unsigned i, idx;
- mutex_lock(&vm->mutex);
list = drm_malloc_ab(vm->max_pde_used + 2,
sizeof(struct amdgpu_bo_list_entry));
if (!list) {
- mutex_unlock(&vm->mutex);
return NULL;
}
list[idx].tv.shared = true;
list_add(&list[idx++].tv.head, head);
}
- mutex_unlock(&vm->mutex);
return list;
}
int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
struct amdgpu_sync *sync)
{
- struct amdgpu_fence *best[AMDGPU_MAX_RINGS] = {};
+ struct fence *best[AMDGPU_MAX_RINGS] = {};
struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx];
struct amdgpu_device *adev = ring->adev;
/* check if the id is still valid */
if (vm_id->id && vm_id->last_id_use &&
- vm_id->last_id_use == adev->vm_manager.active[vm_id->id])
+ vm_id->last_id_use == adev->vm_manager.active[vm_id->id]) {
+ trace_amdgpu_vm_grab_id(vm_id->id, ring->idx);
return 0;
+ }
/* we definately need to flush */
vm_id->pd_gpu_addr = ~0ll;
/* skip over VMID 0, since it is the system VM */
for (i = 1; i < adev->vm_manager.nvm; ++i) {
- struct amdgpu_fence *fence = adev->vm_manager.active[i];
+ struct fence *fence = adev->vm_manager.active[i];
+ struct amdgpu_ring *fring;
if (fence == NULL) {
/* found a free one */
return 0;
}
- if (amdgpu_fence_is_earlier(fence, best[fence->ring->idx])) {
- best[fence->ring->idx] = fence;
- choices[fence->ring == ring ? 0 : 1] = i;
+ fring = amdgpu_ring_from_fence(fence);
+ if (best[fring->idx] == NULL ||
+ fence_is_later(best[fring->idx], fence)) {
+ best[fring->idx] = fence;
+ choices[fring == ring ? 0 : 1] = i;
}
}
for (i = 0; i < 2; ++i) {
if (choices[i]) {
- struct amdgpu_fence *fence;
+ struct fence *fence;
fence = adev->vm_manager.active[choices[i]];
vm_id->id = choices[i];
trace_amdgpu_vm_grab_id(choices[i], ring->idx);
- return amdgpu_sync_fence(ring->adev, sync, &fence->base);
+ return amdgpu_sync_fence(ring->adev, sync, fence);
}
}
unsigned ridx = fence->ring->idx;
unsigned vm_id = vm->ids[ridx].id;
- amdgpu_fence_unref(&adev->vm_manager.active[vm_id]);
- adev->vm_manager.active[vm_id] = amdgpu_fence_ref(fence);
+ fence_put(adev->vm_manager.active[vm_id]);
+ adev->vm_manager.active[vm_id] = fence_get(&fence->base);
- amdgpu_fence_unref(&vm->ids[ridx].last_id_use);
- vm->ids[ridx].last_id_use = amdgpu_fence_ref(fence);
+ fence_put(vm->ids[ridx].last_id_use);
+ vm->ids[ridx].last_id_use = fence_get(&fence->base);
}
/**
}
}
-int amdgpu_vm_free_job(struct amdgpu_job *sched_job)
+int amdgpu_vm_free_job(struct amdgpu_job *job)
{
int i;
- for (i = 0; i < sched_job->num_ibs; i++)
- amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]);
- kfree(sched_job->ibs);
+ for (i = 0; i < job->num_ibs; i++)
+ amdgpu_ib_free(job->adev, &job->ibs[i]);
+ kfree(job->ibs);
return 0;
}
return -ENOMEM;
r = amdgpu_ib_get(ring, NULL, ndw * 4, ib);
- if (r)
+ if (r) {
+ kfree(ib);
return r;
+ }
ib->length_dw = 0;
/* walk over the address space and update the page directory */
return 0;
}
-/**
- * amdgpu_vm_fence_pts - fence page tables after an update
- *
- * @vm: requested vm
- * @start: start of GPU address range
- * @end: end of GPU address range
- * @fence: fence to use
- *
- * Fence the page tables in the range @start - @end (cayman+).
- *
- * Global and local mutex must be locked!
- */
-static void amdgpu_vm_fence_pts(struct amdgpu_vm *vm,
- uint64_t start, uint64_t end,
- struct fence *fence)
-{
- unsigned i;
-
- start >>= amdgpu_vm_block_size;
- end >>= amdgpu_vm_block_size;
-
- for (i = start; i <= end; ++i)
- amdgpu_bo_fence(vm->page_tables[i].bo, fence, true);
-}
-
/**
* amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
*
if (r)
goto error_free;
- amdgpu_vm_fence_pts(vm, mapping->it.start,
- mapping->it.last + 1, f);
+ amdgpu_bo_fence(vm->page_directory, f, true);
if (fence) {
fence_put(*fence);
*fence = fence_get(f);
int r;
if (mem) {
- addr = mem->start << PAGE_SHIFT;
+ addr = (u64)mem->start << PAGE_SHIFT;
if (mem->mem_type != TTM_PL_TT)
addr += adev->vm_manager.vram_base_offset;
} else {
return r;
}
+ if (trace_amdgpu_vm_bo_mapping_enabled()) {
+ list_for_each_entry(mapping, &bo_va->valids, list)
+ trace_amdgpu_vm_bo_mapping(mapping);
+
+ list_for_each_entry(mapping, &bo_va->invalids, list)
+ trace_amdgpu_vm_bo_mapping(mapping);
+ }
+
spin_lock(&vm->status_lock);
list_splice_init(&bo_va->invalids, &bo_va->valids);
list_del_init(&bo_va->vm_status);
INIT_LIST_HEAD(&bo_va->invalids);
INIT_LIST_HEAD(&bo_va->vm_status);
- mutex_lock(&vm->mutex);
list_add_tail(&bo_va->bo_list, &bo->va);
- mutex_unlock(&vm->mutex);
return bo_va;
}
return -EINVAL;
}
- mutex_lock(&vm->mutex);
-
saddr /= AMDGPU_GPU_PAGE_SIZE;
eaddr /= AMDGPU_GPU_PAGE_SIZE;
tmp->it.start, tmp->it.last + 1);
amdgpu_bo_unreserve(bo_va->bo);
r = -EINVAL;
- goto error_unlock;
+ goto error;
}
mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
if (!mapping) {
amdgpu_bo_unreserve(bo_va->bo);
r = -ENOMEM;
- goto error_unlock;
+ goto error;
}
INIT_LIST_HEAD(&mapping->list);
/* walk over the address space and allocate the page tables */
for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
+ struct reservation_object *resv = vm->page_directory->tbo.resv;
struct amdgpu_bo *pt;
if (vm->page_tables[pt_idx].bo)
continue;
- /* drop mutex to allocate and clear page table */
- mutex_unlock(&vm->mutex);
-
+ ww_mutex_lock(&resv->lock, NULL);
r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
AMDGPU_GPU_PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
- NULL, &pt);
+ NULL, resv, &pt);
+ ww_mutex_unlock(&resv->lock);
if (r)
goto error_free;
goto error_free;
}
- /* aquire mutex again */
- mutex_lock(&vm->mutex);
- if (vm->page_tables[pt_idx].bo) {
- /* someone else allocated the pt in the meantime */
- mutex_unlock(&vm->mutex);
- amdgpu_bo_unref(&pt);
- mutex_lock(&vm->mutex);
- continue;
- }
-
vm->page_tables[pt_idx].addr = 0;
vm->page_tables[pt_idx].bo = pt;
}
- mutex_unlock(&vm->mutex);
return 0;
error_free:
- mutex_lock(&vm->mutex);
list_del(&mapping->list);
interval_tree_remove(&mapping->it, &vm->va);
trace_amdgpu_vm_bo_unmap(bo_va, mapping);
kfree(mapping);
-error_unlock:
- mutex_unlock(&vm->mutex);
+error:
return r;
}
}
}
- mutex_lock(&vm->mutex);
list_del(&mapping->list);
interval_tree_remove(&mapping->it, &vm->va);
trace_amdgpu_vm_bo_unmap(bo_va, mapping);
list_add(&mapping->list, &vm->freed);
else
kfree(mapping);
- mutex_unlock(&vm->mutex);
amdgpu_bo_unreserve(bo_va->bo);
return 0;
list_del(&bo_va->bo_list);
- mutex_lock(&vm->mutex);
-
spin_lock(&vm->status_lock);
list_del(&bo_va->vm_status);
spin_unlock(&vm->status_lock);
fence_put(bo_va->last_pt_update);
kfree(bo_va);
-
- mutex_unlock(&vm->mutex);
}
/**
r = amdgpu_bo_create(adev, pd_size, align, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
- NULL, &vm->page_directory);
+ NULL, NULL, &vm->page_directory);
if (r)
return r;
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
fence_put(vm->ids[i].flushed_updates);
- amdgpu_fence_unref(&vm->ids[i].last_id_use);
+ fence_put(vm->ids[i].last_id_use);
}
mutex_destroy(&vm->mutex);
}
}
+static void atom_op_div32(atom_exec_context *ctx, int *ptr, int arg)
+{
+ uint64_t val64;
+ uint8_t attr = U8((*ptr)++);
+ uint32_t dst, src;
+ SDEBUG(" src1: ");
+ dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1);
+ SDEBUG(" src2: ");
+ src = atom_get_src(ctx, attr, ptr);
+ if (src != 0) {
+ val64 = dst;
+ val64 |= ((uint64_t)ctx->ctx->divmul[1]) << 32;
+ do_div(val64, src);
+ ctx->ctx->divmul[0] = lower_32_bits(val64);
+ ctx->ctx->divmul[1] = upper_32_bits(val64);
+ } else {
+ ctx->ctx->divmul[0] = 0;
+ ctx->ctx->divmul[1] = 0;
+ }
+}
+
static void atom_op_eot(atom_exec_context *ctx, int *ptr, int arg)
{
/* functionally, a nop */
ctx->ctx->divmul[0] = dst * src;
}
+static void atom_op_mul32(atom_exec_context *ctx, int *ptr, int arg)
+{
+ uint64_t val64;
+ uint8_t attr = U8((*ptr)++);
+ uint32_t dst, src;
+ SDEBUG(" src1: ");
+ dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1);
+ SDEBUG(" src2: ");
+ src = atom_get_src(ctx, attr, ptr);
+ val64 = (uint64_t)dst * (uint64_t)src;
+ ctx->ctx->divmul[0] = lower_32_bits(val64);
+ ctx->ctx->divmul[1] = upper_32_bits(val64);
+}
+
static void atom_op_nop(atom_exec_context *ctx, int *ptr, int arg)
{
/* nothing */
static void atom_op_debug(atom_exec_context *ctx, int *ptr, int arg)
{
- printk(KERN_INFO "unimplemented!\n");
+ uint8_t val = U8((*ptr)++);
+ SDEBUG("DEBUG output: 0x%02X\n", val);
+}
+
+static void atom_op_processds(atom_exec_context *ctx, int *ptr, int arg)
+{
+ uint16_t val = U16(*ptr);
+ (*ptr) += val + 2;
+ SDEBUG("PROCESSDS output: 0x%02X\n", val);
}
static struct {
atom_op_shr, ATOM_ARG_FB}, {
atom_op_shr, ATOM_ARG_PLL}, {
atom_op_shr, ATOM_ARG_MC}, {
-atom_op_debug, 0},};
+ atom_op_debug, 0}, {
+ atom_op_processds, 0}, {
+ atom_op_mul32, ATOM_ARG_PS}, {
+ atom_op_mul32, ATOM_ARG_WS}, {
+ atom_op_div32, ATOM_ARG_PS}, {
+ atom_op_div32, ATOM_ARG_WS},
+};
static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, uint32_t * params)
{
#define ATOM_CT_PS_MASK 0x7F
#define ATOM_CT_CODE_PTR 6
-#define ATOM_OP_CNT 123
+#define ATOM_OP_CNT 127
#define ATOM_OP_EOT 91
#define ATOM_CASE_MAGIC 0x63
amdgpu_atombios_encoder_setup_dig_encoder(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0);
}
if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
- amdgpu_atombios_encoder_setup_dig_transmitter(encoder,
- ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0);
+ amdgpu_atombios_encoder_set_backlight_level(amdgpu_encoder, dig->backlight_level);
if (ext_encoder)
amdgpu_atombios_encoder_setup_external_encoder(encoder, ext_encoder, ATOM_ENABLE);
} else {
if (!amdgpu_dpm)
return 0;
+ /* init the sysfs and debugfs files late */
+ ret = amdgpu_pm_sysfs_init(adev);
+ if (ret)
+ return ret;
+
ret = ci_set_temperature_range(adev);
if (ret)
return ret;
adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
if (amdgpu_dpm == 1)
amdgpu_pm_print_power_states(adev);
- ret = amdgpu_pm_sysfs_init(adev);
- if (ret)
- goto dpm_failed;
mutex_unlock(&adev->pm.mutex);
DRM_INFO("amdgpu: dpm initialized\n");
int ret, i;
u16 tmp16;
+ if (pci_is_root_bus(adev->pdev->bus))
+ return;
+
if (amdgpu_pcie_gen2 == 0)
return;
{
const char *chip_name;
char fw_name[30];
- int err, i;
+ int err = 0, i;
DRM_DEBUG("\n");
default: BUG();
}
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
if (i == 0)
snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
else
snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma1.bin", chip_name);
- err = request_firmware(&adev->sdma[i].fw, fw_name, adev->dev);
+ err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);
if (err)
goto out;
- err = amdgpu_ucode_validate(adev->sdma[i].fw);
+ err = amdgpu_ucode_validate(adev->sdma.instance[i].fw);
}
out:
if (err) {
printk(KERN_ERR
"cik_sdma: Failed to load firmware \"%s\"\n",
fw_name);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- release_firmware(adev->sdma[i].fw);
- adev->sdma[i].fw = NULL;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ release_firmware(adev->sdma.instance[i].fw);
+ adev->sdma.instance[i].fw = NULL;
}
}
return err;
static uint32_t cik_sdma_ring_get_wptr(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
- u32 me = (ring == &adev->sdma[0].ring) ? 0 : 1;
+ u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
return (RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me]) & 0x3fffc) >> 2;
}
static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
- u32 me = (ring == &adev->sdma[0].ring) ? 0 : 1;
+ u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], (ring->wptr << 2) & 0x3fffc);
}
static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
{
- struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring);
+ struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring);
int i;
for (i = 0; i < count; i++)
SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
u32 ref_and_mask;
- if (ring == &ring->adev->sdma[0].ring)
+ if (ring == &ring->adev->sdma.instance[0].ring)
ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA0_MASK;
else
ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA1_MASK;
*/
static void cik_sdma_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma[1].ring;
+ struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
+ struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl;
int i;
(adev->mman.buffer_funcs_ring == sdma1))
amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);
rb_cntl &= ~SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK;
WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
cik_sdma_rlc_stop(adev);
}
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
me_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]);
if (enable)
me_cntl &= ~SDMA0_F32_CNTL__HALT_MASK;
u32 wb_offset;
int i, j, r;
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- ring = &adev->sdma[i].ring;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
wb_offset = (ring->rptr_offs * 4);
mutex_lock(&adev->srbm_mutex);
u32 fw_size;
int i, j;
- if (!adev->sdma[0].fw || !adev->sdma[1].fw)
- return -EINVAL;
-
/* halt the MEs */
cik_sdma_enable(adev, false);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (!adev->sdma.instance[i].fw)
+ return -EINVAL;
+ hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
amdgpu_ucode_print_sdma_hdr(&hdr->header);
fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
- adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
- adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
- if (adev->sdma[i].feature_version >= 20)
- adev->sdma[i].burst_nop = true;
+ adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
+ adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
+ if (adev->sdma.instance[i].feature_version >= 20)
+ adev->sdma.instance[i].burst_nop = true;
fw_data = (const __le32 *)
- (adev->sdma[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+ (adev->sdma.instance[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0);
for (j = 0; j < fw_size; j++)
WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++));
- WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma[i].fw_version);
+ WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version);
}
return 0;
*/
static void cik_sdma_vm_pad_ib(struct amdgpu_ib *ib)
{
- struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring);
+ struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ib->ring);
u32 pad_count;
int i;
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ adev->sdma.num_instances = SDMA_MAX_INSTANCE;
+
cik_sdma_set_ring_funcs(adev);
cik_sdma_set_irq_funcs(adev);
cik_sdma_set_buffer_funcs(adev);
{
struct amdgpu_ring *ring;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- int r;
+ int r, i;
r = cik_sdma_init_microcode(adev);
if (r) {
}
/* SDMA trap event */
- r = amdgpu_irq_add_id(adev, 224, &adev->sdma_trap_irq);
+ r = amdgpu_irq_add_id(adev, 224, &adev->sdma.trap_irq);
if (r)
return r;
/* SDMA Privileged inst */
- r = amdgpu_irq_add_id(adev, 241, &adev->sdma_illegal_inst_irq);
+ r = amdgpu_irq_add_id(adev, 241, &adev->sdma.illegal_inst_irq);
if (r)
return r;
/* SDMA Privileged inst */
- r = amdgpu_irq_add_id(adev, 247, &adev->sdma_illegal_inst_irq);
+ r = amdgpu_irq_add_id(adev, 247, &adev->sdma.illegal_inst_irq);
if (r)
return r;
- ring = &adev->sdma[0].ring;
- ring->ring_obj = NULL;
-
- ring = &adev->sdma[1].ring;
- ring->ring_obj = NULL;
-
- ring = &adev->sdma[0].ring;
- sprintf(ring->name, "sdma0");
- r = amdgpu_ring_init(adev, ring, 256 * 1024,
- SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0), 0xf,
- &adev->sdma_trap_irq, AMDGPU_SDMA_IRQ_TRAP0,
- AMDGPU_RING_TYPE_SDMA);
- if (r)
- return r;
-
- ring = &adev->sdma[1].ring;
- sprintf(ring->name, "sdma1");
- r = amdgpu_ring_init(adev, ring, 256 * 1024,
- SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0), 0xf,
- &adev->sdma_trap_irq, AMDGPU_SDMA_IRQ_TRAP1,
- AMDGPU_RING_TYPE_SDMA);
- if (r)
- return r;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
+ ring->ring_obj = NULL;
+ sprintf(ring->name, "sdma%d", i);
+ r = amdgpu_ring_init(adev, ring, 256 * 1024,
+ SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0), 0xf,
+ &adev->sdma.trap_irq,
+ (i == 0) ?
+ AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
+ AMDGPU_RING_TYPE_SDMA);
+ if (r)
+ return r;
+ }
return r;
}
static int cik_sdma_sw_fini(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int i;
- amdgpu_ring_fini(&adev->sdma[0].ring);
- amdgpu_ring_fini(&adev->sdma[1].ring);
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ amdgpu_ring_fini(&adev->sdma.instance[i].ring);
return 0;
}
dev_info(adev->dev, "CIK SDMA registers\n");
dev_info(adev->dev, " SRBM_STATUS2=0x%08X\n",
RREG32(mmSRBM_STATUS2));
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
dev_info(adev->dev, " SDMA%d_STATUS_REG=0x%08X\n",
i, RREG32(mmSDMA0_STATUS_REG + sdma_offsets[i]));
dev_info(adev->dev, " SDMA%d_ME_CNTL=0x%08X\n",
case 0:
switch (queue_id) {
case 0:
- amdgpu_fence_process(&adev->sdma[0].ring);
+ amdgpu_fence_process(&adev->sdma.instance[0].ring);
break;
case 1:
/* XXX compute */
case 1:
switch (queue_id) {
case 0:
- amdgpu_fence_process(&adev->sdma[1].ring);
+ amdgpu_fence_process(&adev->sdma.instance[1].ring);
break;
case 1:
/* XXX compute */
.set_powergating_state = cik_sdma_set_powergating_state,
};
-/**
- * cik_sdma_ring_is_lockup - Check if the DMA engine is locked up
- *
- * @ring: amdgpu_ring structure holding ring information
- *
- * Check if the async DMA engine is locked up (CIK).
- * Returns true if the engine appears to be locked up, false if not.
- */
-static bool cik_sdma_ring_is_lockup(struct amdgpu_ring *ring)
-{
-
- if (cik_sdma_is_idle(ring->adev)) {
- amdgpu_ring_lockup_update(ring);
- return false;
- }
- return amdgpu_ring_test_lockup(ring);
-}
-
static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
.get_rptr = cik_sdma_ring_get_rptr,
.get_wptr = cik_sdma_ring_get_wptr,
.emit_hdp_flush = cik_sdma_ring_emit_hdp_flush,
.test_ring = cik_sdma_ring_test_ring,
.test_ib = cik_sdma_ring_test_ib,
- .is_lockup = cik_sdma_ring_is_lockup,
.insert_nop = cik_sdma_ring_insert_nop,
};
static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)
{
- adev->sdma[0].ring.funcs = &cik_sdma_ring_funcs;
- adev->sdma[1].ring.funcs = &cik_sdma_ring_funcs;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ adev->sdma.instance[i].ring.funcs = &cik_sdma_ring_funcs;
}
static const struct amdgpu_irq_src_funcs cik_sdma_trap_irq_funcs = {
static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev)
{
- adev->sdma_trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
- adev->sdma_trap_irq.funcs = &cik_sdma_trap_irq_funcs;
- adev->sdma_illegal_inst_irq.funcs = &cik_sdma_illegal_inst_irq_funcs;
+ adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+ adev->sdma.trap_irq.funcs = &cik_sdma_trap_irq_funcs;
+ adev->sdma.illegal_inst_irq.funcs = &cik_sdma_illegal_inst_irq_funcs;
}
/**
{
if (adev->mman.buffer_funcs == NULL) {
adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;
- adev->mman.buffer_funcs_ring = &adev->sdma[0].ring;
+ adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
}
}
{
if (adev->vm_manager.vm_pte_funcs == NULL) {
adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;
- adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring;
+ adev->vm_manager.vm_pte_funcs_ring = &adev->sdma.instance[0].ring;
adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true;
}
}
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
if (amdgpu_dpm) {
+ int ret;
+ /* init the sysfs and debugfs files late */
+ ret = amdgpu_pm_sysfs_init(adev);
+ if (ret)
+ return ret;
+
/* powerdown unused blocks for now */
cz_dpm_powergate_uvd(adev, true);
cz_dpm_powergate_vce(adev, true);
if (amdgpu_dpm == 1)
amdgpu_pm_print_power_states(adev);
- ret = amdgpu_pm_sysfs_init(adev);
- if (ret)
- goto dpm_init_failed;
-
mutex_unlock(&adev->pm.mutex);
DRM_INFO("amdgpu: dpm initialized\n");
static int cz_dpm_enable(struct amdgpu_device *adev)
{
+ const char *chip_name;
int ret = 0;
/* renable will hang up SMU, so check first */
cz_program_voting_clients(adev);
+ switch (adev->asic_type) {
+ case CHIP_CARRIZO:
+ chip_name = "carrizo";
+ break;
+ case CHIP_STONEY:
+ chip_name = "stoney";
+ break;
+ default:
+ BUG();
+ }
+
+
ret = cz_start_dpm(adev);
if (ret) {
- DRM_ERROR("Carrizo DPM enable failed\n");
+ DRM_ERROR("%s DPM enable failed\n", chip_name);
return -EINVAL;
}
ret = cz_program_bootup_state(adev);
if (ret) {
- DRM_ERROR("Carrizo bootup state program failed\n");
+ DRM_ERROR("%s bootup state program failed\n", chip_name);
return -EINVAL;
}
ret = cz_enable_didt(adev, true);
if (ret) {
- DRM_ERROR("Carrizo enable di/dt failed\n");
+ DRM_ERROR("%s enable di/dt failed\n", chip_name);
return -EINVAL;
}
ret = cz_enable_didt(adev, false);
if (ret) {
- DRM_ERROR("Carrizo disable di/dt failed\n");
+ DRM_ERROR("disable di/dt failed\n");
return -EINVAL;
}
UCODE_ID_CP_MEC_JT1_MASK |
UCODE_ID_CP_MEC_JT2_MASK;
+ if (adev->asic_type == CHIP_STONEY)
+ fw_to_check &= ~(UCODE_ID_SDMA1_MASK | UCODE_ID_CP_MEC_JT2_MASK);
+
cz_smu_request_load_fw(adev);
ret = cz_smu_check_fw_load_finish(adev, fw_to_check);
if (ret)
return ret;
/* manually load MEC firmware for CZ */
- if (adev->asic_type == CHIP_CARRIZO) {
+ if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) {
ret = cz_load_mec_firmware(adev);
if (ret) {
dev_err(adev->dev, "(%d) Mec Firmware load failed\n", ret);
AMDGPU_CPMEC2_UCODE_LOADED |
AMDGPU_CPRLC_UCODE_LOADED;
+ if (adev->asic_type == CHIP_STONEY)
+ adev->smu.fw_flags &= ~(AMDGPU_SDMA1_UCODE_LOADED | AMDGPU_CPMEC2_UCODE_LOADED);
+
return ret;
}
CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME, false);
cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
- cz_smu_populate_single_ucode_load_task(adev,
+ if (adev->asic_type == CHIP_STONEY) {
+ cz_smu_populate_single_ucode_load_task(adev,
+ CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
+ } else {
+ cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2, false);
+ }
cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G, false);
}
if (adev->firmware.smu_load) {
cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
- cz_smu_populate_single_ucode_load_task(adev,
+ if (adev->asic_type == CHIP_STONEY) {
+ cz_smu_populate_single_ucode_load_task(adev,
+ CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
+ } else {
+ cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1, false);
+ }
cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE, false);
cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME, false);
cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
- cz_smu_populate_single_ucode_load_task(adev,
+ if (adev->asic_type == CHIP_STONEY) {
+ cz_smu_populate_single_ucode_load_task(adev,
+ CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
+ } else {
+ cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2, false);
+ }
cz_smu_populate_single_ucode_load_task(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G, true);
}
* 3. map kernel virtual address
*/
ret = amdgpu_bo_create(adev, priv->toc_buffer.data_size, PAGE_SIZE,
- true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, toc_buf);
+ true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
+ toc_buf);
if (ret) {
dev_err(adev->dev, "(%d) SMC TOC buffer allocation failed\n", ret);
}
ret = amdgpu_bo_create(adev, priv->smu_buffer.data_size, PAGE_SIZE,
- true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, smu_buf);
+ true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
+ smu_buf);
if (ret) {
dev_err(adev->dev, "(%d) SMC Internal buffer allocation failed\n", ret);
CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0,
&priv->driver_buffer[priv->driver_buffer_length++]))
goto smu_init_failed;
- if (cz_smu_populate_single_firmware_entry(adev,
- CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1,
- &priv->driver_buffer[priv->driver_buffer_length++]))
- goto smu_init_failed;
+
+ if (adev->asic_type == CHIP_STONEY) {
+ if (cz_smu_populate_single_firmware_entry(adev,
+ CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0,
+ &priv->driver_buffer[priv->driver_buffer_length++]))
+ goto smu_init_failed;
+ } else {
+ if (cz_smu_populate_single_firmware_entry(adev,
+ CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1,
+ &priv->driver_buffer[priv->driver_buffer_length++]))
+ goto smu_init_failed;
+ }
if (cz_smu_populate_single_firmware_entry(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE,
&priv->driver_buffer[priv->driver_buffer_length++]))
CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1,
&priv->driver_buffer[priv->driver_buffer_length++]))
goto smu_init_failed;
- if (cz_smu_populate_single_firmware_entry(adev,
- CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2,
- &priv->driver_buffer[priv->driver_buffer_length++]))
- goto smu_init_failed;
+ if (adev->asic_type == CHIP_STONEY) {
+ if (cz_smu_populate_single_firmware_entry(adev,
+ CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1,
+ &priv->driver_buffer[priv->driver_buffer_length++]))
+ goto smu_init_failed;
+ } else {
+ if (cz_smu_populate_single_firmware_entry(adev,
+ CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2,
+ &priv->driver_buffer[priv->driver_buffer_length++]))
+ goto smu_init_failed;
+ }
if (cz_smu_populate_single_firmware_entry(adev,
CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G,
&priv->driver_buffer[priv->driver_buffer_length++]))
* @crtc_id: crtc to cleanup pageflip on
* @crtc_base: new address of the crtc (GPU MC address)
*
- * Does the actual pageflip (evergreen+).
- * During vblank we take the crtc lock and wait for the update_pending
- * bit to go high, when it does, we release the lock, and allow the
- * double buffered update to take place.
- * Returns the current update pending status.
+ * Triggers the actual pageflip by updating the primary
+ * surface base address.
*/
static void dce_v10_0_page_flip(struct amdgpu_device *adev,
int crtc_id, u64 crtc_base)
{
struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
- u32 tmp = RREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset);
- int i;
-
- /* Lock the graphics update lock */
- tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 1);
- WREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset, tmp);
-
- /* update the scanout addresses */
- WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
- upper_32_bits(crtc_base));
- WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
- lower_32_bits(crtc_base));
+ /* update the primary scanout address */
WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
upper_32_bits(crtc_base));
+ /* writing to the low address triggers the update */
WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
lower_32_bits(crtc_base));
-
- /* Wait for update_pending to go high. */
- for (i = 0; i < adev->usec_timeout; i++) {
- if (RREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset) &
- GRPH_UPDATE__GRPH_SURFACE_UPDATE_PENDING_MASK)
- break;
- udelay(1);
- }
- DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
-
- /* Unlock the lock, so double-buffering can take place inside vblank */
- tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 0);
- WREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset, tmp);
+ /* post the write */
+ RREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset);
}
static int dce_v10_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
struct amdgpu_device *adev = crtc->dev->dev_private;
u32 tmp;
+ WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+ upper_32_bits(amdgpu_crtc->cursor_addr));
+ WREG32(mmCUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+ lower_32_bits(amdgpu_crtc->cursor_addr));
+
tmp = RREG32_IDX(mmCUR_CONTROL + amdgpu_crtc->crtc_offset);
tmp = REG_SET_FIELD(tmp, CUR_CONTROL, CURSOR_EN, 1);
tmp = REG_SET_FIELD(tmp, CUR_CONTROL, CURSOR_MODE, 2);
WREG32_IDX(mmCUR_CONTROL + amdgpu_crtc->crtc_offset, tmp);
}
-static void dce_v10_0_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj,
- uint64_t gpu_addr)
-{
- struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
- struct amdgpu_device *adev = crtc->dev->dev_private;
-
- WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
- upper_32_bits(gpu_addr));
- WREG32(mmCUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
- lower_32_bits(gpu_addr));
-}
-
-static int dce_v10_0_crtc_cursor_move(struct drm_crtc *crtc,
- int x, int y)
+static int dce_v10_0_cursor_move_locked(struct drm_crtc *crtc,
+ int x, int y)
{
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct amdgpu_device *adev = crtc->dev->dev_private;
y = 0;
}
- dce_v10_0_lock_cursor(crtc, true);
WREG32(mmCUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
WREG32(mmCUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
- dce_v10_0_lock_cursor(crtc, false);
+
+ amdgpu_crtc->cursor_x = x;
+ amdgpu_crtc->cursor_y = y;
return 0;
}
-static int dce_v10_0_crtc_cursor_set(struct drm_crtc *crtc,
- struct drm_file *file_priv,
- uint32_t handle,
- uint32_t width,
- uint32_t height)
+static int dce_v10_0_crtc_cursor_move(struct drm_crtc *crtc,
+ int x, int y)
+{
+ int ret;
+
+ dce_v10_0_lock_cursor(crtc, true);
+ ret = dce_v10_0_cursor_move_locked(crtc, x, y);
+ dce_v10_0_lock_cursor(crtc, false);
+
+ return ret;
+}
+
+static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
+ struct drm_file *file_priv,
+ uint32_t handle,
+ uint32_t width,
+ uint32_t height,
+ int32_t hot_x,
+ int32_t hot_y)
{
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct drm_gem_object *obj;
- struct amdgpu_bo *robj;
- uint64_t gpu_addr;
+ struct amdgpu_bo *aobj;
int ret;
if (!handle) {
return -ENOENT;
}
- robj = gem_to_amdgpu_bo(obj);
- ret = amdgpu_bo_reserve(robj, false);
- if (unlikely(ret != 0))
- goto fail;
- ret = amdgpu_bo_pin_restricted(robj, AMDGPU_GEM_DOMAIN_VRAM,
- 0, 0, &gpu_addr);
- amdgpu_bo_unreserve(robj);
- if (ret)
- goto fail;
+ aobj = gem_to_amdgpu_bo(obj);
+ ret = amdgpu_bo_reserve(aobj, false);
+ if (ret != 0) {
+ drm_gem_object_unreference_unlocked(obj);
+ return ret;
+ }
+
+ ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+ amdgpu_bo_unreserve(aobj);
+ if (ret) {
+ DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
+ drm_gem_object_unreference_unlocked(obj);
+ return ret;
+ }
amdgpu_crtc->cursor_width = width;
amdgpu_crtc->cursor_height = height;
dce_v10_0_lock_cursor(crtc, true);
- dce_v10_0_set_cursor(crtc, obj, gpu_addr);
+
+ if (hot_x != amdgpu_crtc->cursor_hot_x ||
+ hot_y != amdgpu_crtc->cursor_hot_y) {
+ int x, y;
+
+ x = amdgpu_crtc->cursor_x + amdgpu_crtc->cursor_hot_x - hot_x;
+ y = amdgpu_crtc->cursor_y + amdgpu_crtc->cursor_hot_y - hot_y;
+
+ dce_v10_0_cursor_move_locked(crtc, x, y);
+
+ amdgpu_crtc->cursor_hot_x = hot_x;
+ amdgpu_crtc->cursor_hot_y = hot_y;
+ }
+
dce_v10_0_show_cursor(crtc);
dce_v10_0_lock_cursor(crtc, false);
unpin:
if (amdgpu_crtc->cursor_bo) {
- robj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
- ret = amdgpu_bo_reserve(robj, false);
+ struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+ ret = amdgpu_bo_reserve(aobj, false);
if (likely(ret == 0)) {
- amdgpu_bo_unpin(robj);
- amdgpu_bo_unreserve(robj);
+ amdgpu_bo_unpin(aobj);
+ amdgpu_bo_unreserve(aobj);
}
drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
}
amdgpu_crtc->cursor_bo = obj;
return 0;
-fail:
- drm_gem_object_unreference_unlocked(obj);
+}
- return ret;
+static void dce_v10_0_cursor_reset(struct drm_crtc *crtc)
+{
+ struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+ if (amdgpu_crtc->cursor_bo) {
+ dce_v10_0_lock_cursor(crtc, true);
+
+ dce_v10_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
+ amdgpu_crtc->cursor_y);
+
+ dce_v10_0_show_cursor(crtc);
+
+ dce_v10_0_lock_cursor(crtc, false);
+ }
}
static void dce_v10_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
}
static const struct drm_crtc_funcs dce_v10_0_crtc_funcs = {
- .cursor_set = dce_v10_0_crtc_cursor_set,
+ .cursor_set2 = dce_v10_0_crtc_cursor_set2,
.cursor_move = dce_v10_0_crtc_cursor_move,
.gamma_set = dce_v10_0_crtc_gamma_set,
.set_config = amdgpu_crtc_set_config,
dce_v10_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
amdgpu_atombios_crtc_overscan_setup(crtc, mode, adjusted_mode);
amdgpu_atombios_crtc_scaler_setup(crtc);
+ dce_v10_0_cursor_reset(crtc);
/* update the hw version fpr dpm */
amdgpu_crtc->hw_mode = *adjusted_mode;
amdgpu_atombios_scratch_regs_save(adev);
- dce_v10_0_hpd_fini(adev);
-
- return 0;
+ return dce_v10_0_hw_fini(handle);
}
static int dce_v10_0_resume(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int ret;
- dce_v10_0_init_golden_registers(adev);
+ ret = dce_v10_0_hw_init(handle);
amdgpu_atombios_scratch_regs_restore(adev);
- /* init dig PHYs, disp eng pll */
- amdgpu_atombios_encoder_init_dig(adev);
- amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
/* turn on the BL */
if (adev->mode_info.bl_encoder) {
u8 bl_level = amdgpu_display_backlight_get_level(adev,
bl_level);
}
- /* initialize hpd */
- dce_v10_0_hpd_init(adev);
-
- return 0;
+ return ret;
}
static bool dce_v10_0_is_idle(void *handle)
unsigned type,
enum amdgpu_interrupt_state state)
{
- u32 reg, reg_block;
- /* now deal with page flip IRQ */
- switch (type) {
- case AMDGPU_PAGEFLIP_IRQ_D1:
- reg_block = CRTC0_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D2:
- reg_block = CRTC1_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D3:
- reg_block = CRTC2_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D4:
- reg_block = CRTC3_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D5:
- reg_block = CRTC4_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D6:
- reg_block = CRTC5_REGISTER_OFFSET;
- break;
- default:
- DRM_ERROR("invalid pageflip crtc %d\n", type);
- return -EINVAL;
+ u32 reg;
+
+ if (type >= adev->mode_info.num_crtc) {
+ DRM_ERROR("invalid pageflip crtc %d\n", type);
+ return -EINVAL;
}
- reg = RREG32(mmGRPH_INTERRUPT_CONTROL + reg_block);
+ reg = RREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type]);
if (state == AMDGPU_IRQ_STATE_DISABLE)
- WREG32(mmGRPH_INTERRUPT_CONTROL + reg_block, reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+ WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
+ reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
else
- WREG32(mmGRPH_INTERRUPT_CONTROL + reg_block, reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+ WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
+ reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
return 0;
}
struct amdgpu_irq_src *source,
struct amdgpu_iv_entry *entry)
{
- int reg_block;
unsigned long flags;
unsigned crtc_id;
struct amdgpu_crtc *amdgpu_crtc;
crtc_id = (entry->src_id - 8) >> 1;
amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
- /* ack the interrupt */
- switch(crtc_id){
- case AMDGPU_PAGEFLIP_IRQ_D1:
- reg_block = CRTC0_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D2:
- reg_block = CRTC1_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D3:
- reg_block = CRTC2_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D4:
- reg_block = CRTC3_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D5:
- reg_block = CRTC4_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D6:
- reg_block = CRTC5_REGISTER_OFFSET;
- break;
- default:
- DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
- return -EINVAL;
+ if (crtc_id >= adev->mode_info.num_crtc) {
+ DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
+ return -EINVAL;
}
- if (RREG32(mmGRPH_INTERRUPT_STATUS + reg_block) & GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
- WREG32(mmGRPH_INTERRUPT_STATUS + reg_block, GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
+ if (RREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id]) &
+ GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
+ WREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id],
+ GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
/* IRQ could occur when in initial stage */
if (amdgpu_crtc == NULL)
mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
};
+static const u32 stoney_golden_settings_a11[] =
+{
+ mmCRTC_DOUBLE_BUFFER_CONTROL, 0x00010101, 0x00010000,
+ mmFBC_MISC, 0x1f311fff, 0x14302000,
+};
+
+
static void dce_v11_0_init_golden_registers(struct amdgpu_device *adev)
{
switch (adev->asic_type) {
cz_golden_settings_a11,
(const u32)ARRAY_SIZE(cz_golden_settings_a11));
break;
+ case CHIP_STONEY:
+ amdgpu_program_register_sequence(adev,
+ stoney_golden_settings_a11,
+ (const u32)ARRAY_SIZE(stoney_golden_settings_a11));
+ break;
default:
break;
}
* @crtc_id: crtc to cleanup pageflip on
* @crtc_base: new address of the crtc (GPU MC address)
*
- * Does the actual pageflip (evergreen+).
- * During vblank we take the crtc lock and wait for the update_pending
- * bit to go high, when it does, we release the lock, and allow the
- * double buffered update to take place.
- * Returns the current update pending status.
+ * Triggers the actual pageflip by updating the primary
+ * surface base address.
*/
static void dce_v11_0_page_flip(struct amdgpu_device *adev,
int crtc_id, u64 crtc_base)
{
struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
- u32 tmp = RREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset);
- int i;
-
- /* Lock the graphics update lock */
- tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 1);
- WREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset, tmp);
/* update the scanout addresses */
- WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
- upper_32_bits(crtc_base));
- WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
- lower_32_bits(crtc_base));
-
WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
upper_32_bits(crtc_base));
+ /* writing to the low address triggers the update */
WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
lower_32_bits(crtc_base));
-
- /* Wait for update_pending to go high. */
- for (i = 0; i < adev->usec_timeout; i++) {
- if (RREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset) &
- GRPH_UPDATE__GRPH_SURFACE_UPDATE_PENDING_MASK)
- break;
- udelay(1);
- }
- DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
-
- /* Unlock the lock, so double-buffering can take place inside vblank */
- tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 0);
- WREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset, tmp);
+ /* post the write */
+ RREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset);
}
static int dce_v11_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
/* XXX need to determine what plls are available on each DCE11 part */
pll_in_use = amdgpu_pll_get_use_mask(crtc);
- if (adev->asic_type == CHIP_CARRIZO) {
+ if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) {
if (!(pll_in_use & (1 << ATOM_PPLL1)))
return ATOM_PPLL1;
if (!(pll_in_use & (1 << ATOM_PPLL0)))
struct amdgpu_device *adev = crtc->dev->dev_private;
u32 tmp;
+ WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+ upper_32_bits(amdgpu_crtc->cursor_addr));
+ WREG32(mmCUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+ lower_32_bits(amdgpu_crtc->cursor_addr));
+
tmp = RREG32_IDX(mmCUR_CONTROL + amdgpu_crtc->crtc_offset);
tmp = REG_SET_FIELD(tmp, CUR_CONTROL, CURSOR_EN, 1);
tmp = REG_SET_FIELD(tmp, CUR_CONTROL, CURSOR_MODE, 2);
WREG32_IDX(mmCUR_CONTROL + amdgpu_crtc->crtc_offset, tmp);
}
-static void dce_v11_0_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj,
- uint64_t gpu_addr)
-{
- struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
- struct amdgpu_device *adev = crtc->dev->dev_private;
-
- WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
- upper_32_bits(gpu_addr));
- WREG32(mmCUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
- lower_32_bits(gpu_addr));
-}
-
-static int dce_v11_0_crtc_cursor_move(struct drm_crtc *crtc,
- int x, int y)
+static int dce_v11_0_cursor_move_locked(struct drm_crtc *crtc,
+ int x, int y)
{
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct amdgpu_device *adev = crtc->dev->dev_private;
y = 0;
}
- dce_v11_0_lock_cursor(crtc, true);
WREG32(mmCUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
WREG32(mmCUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
- dce_v11_0_lock_cursor(crtc, false);
+
+ amdgpu_crtc->cursor_x = x;
+ amdgpu_crtc->cursor_y = y;
return 0;
}
-static int dce_v11_0_crtc_cursor_set(struct drm_crtc *crtc,
- struct drm_file *file_priv,
- uint32_t handle,
- uint32_t width,
- uint32_t height)
+static int dce_v11_0_crtc_cursor_move(struct drm_crtc *crtc,
+ int x, int y)
+{
+ int ret;
+
+ dce_v11_0_lock_cursor(crtc, true);
+ ret = dce_v11_0_cursor_move_locked(crtc, x, y);
+ dce_v11_0_lock_cursor(crtc, false);
+
+ return ret;
+}
+
+static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
+ struct drm_file *file_priv,
+ uint32_t handle,
+ uint32_t width,
+ uint32_t height,
+ int32_t hot_x,
+ int32_t hot_y)
{
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct drm_gem_object *obj;
- struct amdgpu_bo *robj;
- uint64_t gpu_addr;
+ struct amdgpu_bo *aobj;
int ret;
if (!handle) {
return -ENOENT;
}
- robj = gem_to_amdgpu_bo(obj);
- ret = amdgpu_bo_reserve(robj, false);
- if (unlikely(ret != 0))
- goto fail;
- ret = amdgpu_bo_pin_restricted(robj, AMDGPU_GEM_DOMAIN_VRAM,
- 0, 0, &gpu_addr);
- amdgpu_bo_unreserve(robj);
- if (ret)
- goto fail;
+ aobj = gem_to_amdgpu_bo(obj);
+ ret = amdgpu_bo_reserve(aobj, false);
+ if (ret != 0) {
+ drm_gem_object_unreference_unlocked(obj);
+ return ret;
+ }
+
+ ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+ amdgpu_bo_unreserve(aobj);
+ if (ret) {
+ DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
+ drm_gem_object_unreference_unlocked(obj);
+ return ret;
+ }
amdgpu_crtc->cursor_width = width;
amdgpu_crtc->cursor_height = height;
dce_v11_0_lock_cursor(crtc, true);
- dce_v11_0_set_cursor(crtc, obj, gpu_addr);
+
+ if (hot_x != amdgpu_crtc->cursor_hot_x ||
+ hot_y != amdgpu_crtc->cursor_hot_y) {
+ int x, y;
+
+ x = amdgpu_crtc->cursor_x + amdgpu_crtc->cursor_hot_x - hot_x;
+ y = amdgpu_crtc->cursor_y + amdgpu_crtc->cursor_hot_y - hot_y;
+
+ dce_v11_0_cursor_move_locked(crtc, x, y);
+
+ amdgpu_crtc->cursor_hot_x = hot_x;
+ amdgpu_crtc->cursor_hot_y = hot_y;
+ }
+
dce_v11_0_show_cursor(crtc);
dce_v11_0_lock_cursor(crtc, false);
unpin:
if (amdgpu_crtc->cursor_bo) {
- robj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
- ret = amdgpu_bo_reserve(robj, false);
+ struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+ ret = amdgpu_bo_reserve(aobj, false);
if (likely(ret == 0)) {
- amdgpu_bo_unpin(robj);
- amdgpu_bo_unreserve(robj);
+ amdgpu_bo_unpin(aobj);
+ amdgpu_bo_unreserve(aobj);
}
drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
}
amdgpu_crtc->cursor_bo = obj;
return 0;
-fail:
- drm_gem_object_unreference_unlocked(obj);
+}
- return ret;
+static void dce_v11_0_cursor_reset(struct drm_crtc *crtc)
+{
+ struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+ if (amdgpu_crtc->cursor_bo) {
+ dce_v11_0_lock_cursor(crtc, true);
+
+ dce_v11_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
+ amdgpu_crtc->cursor_y);
+
+ dce_v11_0_show_cursor(crtc);
+
+ dce_v11_0_lock_cursor(crtc, false);
+ }
}
static void dce_v11_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
}
static const struct drm_crtc_funcs dce_v11_0_crtc_funcs = {
- .cursor_set = dce_v11_0_crtc_cursor_set,
+ .cursor_set2 = dce_v11_0_crtc_cursor_set2,
.cursor_move = dce_v11_0_crtc_cursor_move,
.gamma_set = dce_v11_0_crtc_gamma_set,
.set_config = amdgpu_crtc_set_config,
dce_v11_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
amdgpu_atombios_crtc_overscan_setup(crtc, mode, adjusted_mode);
amdgpu_atombios_crtc_scaler_setup(crtc);
+ dce_v11_0_cursor_reset(crtc);
/* update the hw version fpr dpm */
amdgpu_crtc->hw_mode = *adjusted_mode;
switch (adev->asic_type) {
case CHIP_CARRIZO:
- adev->mode_info.num_crtc = 4;
+ adev->mode_info.num_crtc = 3;
+ adev->mode_info.num_hpd = 6;
+ adev->mode_info.num_dig = 9;
+ break;
+ case CHIP_STONEY:
+ adev->mode_info.num_crtc = 2;
adev->mode_info.num_hpd = 6;
adev->mode_info.num_dig = 9;
break;
dce_v11_0_init_golden_registers(adev);
/* init dig PHYs, disp eng pll */
+ amdgpu_atombios_crtc_powergate_init(adev);
amdgpu_atombios_encoder_init_dig(adev);
amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
amdgpu_atombios_scratch_regs_save(adev);
- dce_v11_0_hpd_fini(adev);
-
- return 0;
+ return dce_v11_0_hw_fini(handle);
}
static int dce_v11_0_resume(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int ret;
- dce_v11_0_init_golden_registers(adev);
+ ret = dce_v11_0_hw_init(handle);
amdgpu_atombios_scratch_regs_restore(adev);
- /* init dig PHYs, disp eng pll */
- amdgpu_atombios_crtc_powergate_init(adev);
- amdgpu_atombios_encoder_init_dig(adev);
- amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
/* turn on the BL */
if (adev->mode_info.bl_encoder) {
u8 bl_level = amdgpu_display_backlight_get_level(adev,
bl_level);
}
- /* initialize hpd */
- dce_v11_0_hpd_init(adev);
-
- return 0;
+ return ret;
}
static bool dce_v11_0_is_idle(void *handle)
unsigned type,
enum amdgpu_interrupt_state state)
{
- u32 reg, reg_block;
- /* now deal with page flip IRQ */
- switch (type) {
- case AMDGPU_PAGEFLIP_IRQ_D1:
- reg_block = CRTC0_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D2:
- reg_block = CRTC1_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D3:
- reg_block = CRTC2_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D4:
- reg_block = CRTC3_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D5:
- reg_block = CRTC4_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D6:
- reg_block = CRTC5_REGISTER_OFFSET;
- break;
- default:
- DRM_ERROR("invalid pageflip crtc %d\n", type);
- return -EINVAL;
+ u32 reg;
+
+ if (type >= adev->mode_info.num_crtc) {
+ DRM_ERROR("invalid pageflip crtc %d\n", type);
+ return -EINVAL;
}
- reg = RREG32(mmGRPH_INTERRUPT_CONTROL + reg_block);
+ reg = RREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type]);
if (state == AMDGPU_IRQ_STATE_DISABLE)
- WREG32(mmGRPH_INTERRUPT_CONTROL + reg_block, reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+ WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
+ reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
else
- WREG32(mmGRPH_INTERRUPT_CONTROL + reg_block, reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+ WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
+ reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
return 0;
}
struct amdgpu_irq_src *source,
struct amdgpu_iv_entry *entry)
{
- int reg_block;
unsigned long flags;
unsigned crtc_id;
struct amdgpu_crtc *amdgpu_crtc;
crtc_id = (entry->src_id - 8) >> 1;
amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
- /* ack the interrupt */
- switch(crtc_id){
- case AMDGPU_PAGEFLIP_IRQ_D1:
- reg_block = CRTC0_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D2:
- reg_block = CRTC1_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D3:
- reg_block = CRTC2_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D4:
- reg_block = CRTC3_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D5:
- reg_block = CRTC4_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D6:
- reg_block = CRTC5_REGISTER_OFFSET;
- break;
- default:
- DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
- return -EINVAL;
+ if (crtc_id >= adev->mode_info.num_crtc) {
+ DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
+ return -EINVAL;
}
- if (RREG32(mmGRPH_INTERRUPT_STATUS + reg_block) & GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
- WREG32(mmGRPH_INTERRUPT_STATUS + reg_block, GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
+ if (RREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id]) &
+ GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
+ WREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id],
+ GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
/* IRQ could occur when in initial stage */
if(amdgpu_crtc == NULL)
* @crtc_id: crtc to cleanup pageflip on
* @crtc_base: new address of the crtc (GPU MC address)
*
- * Does the actual pageflip (evergreen+).
- * During vblank we take the crtc lock and wait for the update_pending
- * bit to go high, when it does, we release the lock, and allow the
- * double buffered update to take place.
- * Returns the current update pending status.
+ * Triggers the actual pageflip by updating the primary
+ * surface base address.
*/
static void dce_v8_0_page_flip(struct amdgpu_device *adev,
int crtc_id, u64 crtc_base)
{
struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
- u32 tmp = RREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset);
- int i;
-
- /* Lock the graphics update lock */
- tmp |= GRPH_UPDATE__GRPH_UPDATE_LOCK_MASK;
- WREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset, tmp);
-
- /* update the scanout addresses */
- WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
- upper_32_bits(crtc_base));
- WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
- (u32)crtc_base);
+ /* update the primary scanout addresses */
WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
upper_32_bits(crtc_base));
+ /* writing to the low address triggers the update */
WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
- (u32)crtc_base);
-
- /* Wait for update_pending to go high. */
- for (i = 0; i < adev->usec_timeout; i++) {
- if (RREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset) &
- GRPH_UPDATE__GRPH_SURFACE_UPDATE_PENDING_MASK)
- break;
- udelay(1);
- }
- DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
-
- /* Unlock the lock, so double-buffering can take place inside vblank */
- tmp &= ~GRPH_UPDATE__GRPH_UPDATE_LOCK_MASK;
- WREG32(mmGRPH_UPDATE + amdgpu_crtc->crtc_offset, tmp);
+ lower_32_bits(crtc_base));
+ /* post the write */
+ RREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset);
}
static int dce_v8_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct amdgpu_device *adev = crtc->dev->dev_private;
+ WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+ upper_32_bits(amdgpu_crtc->cursor_addr));
+ WREG32(mmCUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+ lower_32_bits(amdgpu_crtc->cursor_addr));
+
WREG32_IDX(mmCUR_CONTROL + amdgpu_crtc->crtc_offset,
CUR_CONTROL__CURSOR_EN_MASK |
(CURSOR_24_8_PRE_MULT << CUR_CONTROL__CURSOR_MODE__SHIFT) |
(CURSOR_URGENT_1_2 << CUR_CONTROL__CURSOR_URGENT_CONTROL__SHIFT));
}
-static void dce_v8_0_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj,
- uint64_t gpu_addr)
-{
- struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
- struct amdgpu_device *adev = crtc->dev->dev_private;
-
- WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
- upper_32_bits(gpu_addr));
- WREG32(mmCUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
- gpu_addr & 0xffffffff);
-}
-
-static int dce_v8_0_crtc_cursor_move(struct drm_crtc *crtc,
- int x, int y)
+static int dce_v8_0_cursor_move_locked(struct drm_crtc *crtc,
+ int x, int y)
{
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct amdgpu_device *adev = crtc->dev->dev_private;
y = 0;
}
- dce_v8_0_lock_cursor(crtc, true);
WREG32(mmCUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
WREG32(mmCUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
- dce_v8_0_lock_cursor(crtc, false);
+
+ amdgpu_crtc->cursor_x = x;
+ amdgpu_crtc->cursor_y = y;
return 0;
}
-static int dce_v8_0_crtc_cursor_set(struct drm_crtc *crtc,
- struct drm_file *file_priv,
- uint32_t handle,
- uint32_t width,
- uint32_t height)
+static int dce_v8_0_crtc_cursor_move(struct drm_crtc *crtc,
+ int x, int y)
+{
+ int ret;
+
+ dce_v8_0_lock_cursor(crtc, true);
+ ret = dce_v8_0_cursor_move_locked(crtc, x, y);
+ dce_v8_0_lock_cursor(crtc, false);
+
+ return ret;
+}
+
+static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
+ struct drm_file *file_priv,
+ uint32_t handle,
+ uint32_t width,
+ uint32_t height,
+ int32_t hot_x,
+ int32_t hot_y)
{
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
struct drm_gem_object *obj;
- struct amdgpu_bo *robj;
- uint64_t gpu_addr;
+ struct amdgpu_bo *aobj;
int ret;
if (!handle) {
return -ENOENT;
}
- robj = gem_to_amdgpu_bo(obj);
- ret = amdgpu_bo_reserve(robj, false);
- if (unlikely(ret != 0))
- goto fail;
- ret = amdgpu_bo_pin_restricted(robj, AMDGPU_GEM_DOMAIN_VRAM,
- 0, 0, &gpu_addr);
- amdgpu_bo_unreserve(robj);
- if (ret)
- goto fail;
+ aobj = gem_to_amdgpu_bo(obj);
+ ret = amdgpu_bo_reserve(aobj, false);
+ if (ret != 0) {
+ drm_gem_object_unreference_unlocked(obj);
+ return ret;
+ }
+
+ ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+ amdgpu_bo_unreserve(aobj);
+ if (ret) {
+ DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
+ drm_gem_object_unreference_unlocked(obj);
+ return ret;
+ }
amdgpu_crtc->cursor_width = width;
amdgpu_crtc->cursor_height = height;
dce_v8_0_lock_cursor(crtc, true);
- dce_v8_0_set_cursor(crtc, obj, gpu_addr);
+
+ if (hot_x != amdgpu_crtc->cursor_hot_x ||
+ hot_y != amdgpu_crtc->cursor_hot_y) {
+ int x, y;
+
+ x = amdgpu_crtc->cursor_x + amdgpu_crtc->cursor_hot_x - hot_x;
+ y = amdgpu_crtc->cursor_y + amdgpu_crtc->cursor_hot_y - hot_y;
+
+ dce_v8_0_cursor_move_locked(crtc, x, y);
+
+ amdgpu_crtc->cursor_hot_x = hot_x;
+ amdgpu_crtc->cursor_hot_y = hot_y;
+ }
+
dce_v8_0_show_cursor(crtc);
dce_v8_0_lock_cursor(crtc, false);
unpin:
if (amdgpu_crtc->cursor_bo) {
- robj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
- ret = amdgpu_bo_reserve(robj, false);
+ struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+ ret = amdgpu_bo_reserve(aobj, false);
if (likely(ret == 0)) {
- amdgpu_bo_unpin(robj);
- amdgpu_bo_unreserve(robj);
+ amdgpu_bo_unpin(aobj);
+ amdgpu_bo_unreserve(aobj);
}
drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
}
amdgpu_crtc->cursor_bo = obj;
return 0;
-fail:
- drm_gem_object_unreference_unlocked(obj);
+}
- return ret;
+static void dce_v8_0_cursor_reset(struct drm_crtc *crtc)
+{
+ struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+ if (amdgpu_crtc->cursor_bo) {
+ dce_v8_0_lock_cursor(crtc, true);
+
+ dce_v8_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
+ amdgpu_crtc->cursor_y);
+
+ dce_v8_0_show_cursor(crtc);
+
+ dce_v8_0_lock_cursor(crtc, false);
+ }
}
static void dce_v8_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
}
static const struct drm_crtc_funcs dce_v8_0_crtc_funcs = {
- .cursor_set = dce_v8_0_crtc_cursor_set,
+ .cursor_set2 = dce_v8_0_crtc_cursor_set2,
.cursor_move = dce_v8_0_crtc_cursor_move,
.gamma_set = dce_v8_0_crtc_gamma_set,
.set_config = amdgpu_crtc_set_config,
dce_v8_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
amdgpu_atombios_crtc_overscan_setup(crtc, mode, adjusted_mode);
amdgpu_atombios_crtc_scaler_setup(crtc);
+ dce_v8_0_cursor_reset(crtc);
/* update the hw version fpr dpm */
amdgpu_crtc->hw_mode = *adjusted_mode;
amdgpu_atombios_scratch_regs_save(adev);
- dce_v8_0_hpd_fini(adev);
-
- return 0;
+ return dce_v8_0_hw_fini(handle);
}
static int dce_v8_0_resume(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int ret;
+
+ ret = dce_v8_0_hw_init(handle);
amdgpu_atombios_scratch_regs_restore(adev);
- /* init dig PHYs, disp eng pll */
- amdgpu_atombios_encoder_init_dig(adev);
- amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
/* turn on the BL */
if (adev->mode_info.bl_encoder) {
u8 bl_level = amdgpu_display_backlight_get_level(adev,
bl_level);
}
- /* initialize hpd */
- dce_v8_0_hpd_init(adev);
-
- return 0;
+ return ret;
}
static bool dce_v8_0_is_idle(void *handle)
unsigned type,
enum amdgpu_interrupt_state state)
{
- u32 reg, reg_block;
- /* now deal with page flip IRQ */
- switch (type) {
- case AMDGPU_PAGEFLIP_IRQ_D1:
- reg_block = CRTC0_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D2:
- reg_block = CRTC1_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D3:
- reg_block = CRTC2_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D4:
- reg_block = CRTC3_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D5:
- reg_block = CRTC4_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D6:
- reg_block = CRTC5_REGISTER_OFFSET;
- break;
- default:
- DRM_ERROR("invalid pageflip crtc %d\n", type);
- return -EINVAL;
+ u32 reg;
+
+ if (type >= adev->mode_info.num_crtc) {
+ DRM_ERROR("invalid pageflip crtc %d\n", type);
+ return -EINVAL;
}
- reg = RREG32(mmGRPH_INTERRUPT_CONTROL + reg_block);
+ reg = RREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type]);
if (state == AMDGPU_IRQ_STATE_DISABLE)
- WREG32(mmGRPH_INTERRUPT_CONTROL + reg_block, reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+ WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
+ reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
else
- WREG32(mmGRPH_INTERRUPT_CONTROL + reg_block, reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+ WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
+ reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
return 0;
}
struct amdgpu_irq_src *source,
struct amdgpu_iv_entry *entry)
{
- int reg_block;
unsigned long flags;
unsigned crtc_id;
struct amdgpu_crtc *amdgpu_crtc;
crtc_id = (entry->src_id - 8) >> 1;
amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
- /* ack the interrupt */
- switch(crtc_id){
- case AMDGPU_PAGEFLIP_IRQ_D1:
- reg_block = CRTC0_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D2:
- reg_block = CRTC1_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D3:
- reg_block = CRTC2_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D4:
- reg_block = CRTC3_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D5:
- reg_block = CRTC4_REGISTER_OFFSET;
- break;
- case AMDGPU_PAGEFLIP_IRQ_D6:
- reg_block = CRTC5_REGISTER_OFFSET;
- break;
- default:
- DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
- return -EINVAL;
+ if (crtc_id >= adev->mode_info.num_crtc) {
+ DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
+ return -EINVAL;
}
- if (RREG32(mmGRPH_INTERRUPT_STATUS + reg_block) & GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
- WREG32(mmGRPH_INTERRUPT_STATUS + reg_block, GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
+ if (RREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id]) &
+ GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
+ WREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id],
+ GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
/* IRQ could occur when in initial stage */
if (amdgpu_crtc == NULL)
ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
true, AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, toc_buf);
+ NULL, NULL, toc_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for TOC buffer\n");
return -ENOMEM;
ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE,
true, AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, smu_buf);
+ NULL, NULL, smu_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for SMU internal buffer\n");
return -ENOMEM;
r = amdgpu_bo_create(adev,
adev->gfx.mec.num_mec *adev->gfx.mec.num_pipe * MEC_HPD_SIZE * 2,
PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
+ AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
&adev->gfx.mec.hpd_eop_obj);
if (r) {
dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
r = amdgpu_bo_create(adev,
sizeof(struct bonaire_mqd),
PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
+ AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
&ring->mqd_obj);
if (r) {
dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
return 0;
}
-static void gfx_v7_0_ce_sync_me(struct amdgpu_ring *ring)
-{
- struct amdgpu_device *adev = ring->adev;
- u64 gpu_addr = adev->wb.gpu_addr + adev->gfx.ce_sync_offs * 4;
-
- /* instruct DE to set a magic number */
- amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
- amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
- WRITE_DATA_DST_SEL(5)));
- amdgpu_ring_write(ring, gpu_addr & 0xfffffffc);
- amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xffffffff);
- amdgpu_ring_write(ring, 1);
-
- /* let CE wait till condition satisfied */
- amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
- amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
- WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
- WAIT_REG_MEM_FUNCTION(3) | /* == */
- WAIT_REG_MEM_ENGINE(2))); /* ce */
- amdgpu_ring_write(ring, gpu_addr & 0xfffffffc);
- amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xffffffff);
- amdgpu_ring_write(ring, 1);
- amdgpu_ring_write(ring, 0xffffffff);
- amdgpu_ring_write(ring, 4); /* poll interval */
-
- /* instruct CE to reset wb of ce_sync to zero */
- amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
- amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
- WRITE_DATA_DST_SEL(5) |
- WR_CONFIRM));
- amdgpu_ring_write(ring, gpu_addr & 0xfffffffc);
- amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xffffffff);
- amdgpu_ring_write(ring, 0);
-}
-
/*
* vm
* VMID 0 is the physical GPU addresses as used by the kernel.
unsigned vm_id, uint64_t pd_addr)
{
int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+ if (usepfp) {
+ /* synce CE with ME to prevent CE fetch CEIB before context switch done */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
+ }
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
amdgpu_ring_write(ring, 0x0);
/* synce CE with ME to prevent CE fetch CEIB before context switch done */
- gfx_v7_0_ce_sync_me(ring);
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
}
}
r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->gfx.rlc.save_restore_obj);
+ NULL, NULL,
+ &adev->gfx.rlc.save_restore_obj);
if (r) {
dev_warn(adev->dev, "(%d) create RLC sr bo failed\n", r);
return r;
r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->gfx.rlc.clear_state_obj);
+ NULL, NULL,
+ &adev->gfx.rlc.clear_state_obj);
if (r) {
dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
gfx_v7_0_rlc_fini(adev);
r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, &adev->gfx.rlc.cp_table_obj);
+ NULL, NULL,
+ &adev->gfx.rlc.cp_table_obj);
if (r) {
dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
gfx_v7_0_rlc_fini(adev);
return r;
}
- r = amdgpu_wb_get(adev, &adev->gfx.ce_sync_offs);
- if (r) {
- DRM_ERROR("(%d) gfx.ce_sync_offs wb alloc failed\n", r);
- return r;
- }
-
for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
ring = &adev->gfx.gfx_ring[i];
ring->ring_obj = NULL;
r = amdgpu_bo_create(adev, adev->gds.mem.gfx_partition_size,
PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_GDS, 0,
- NULL, &adev->gds.gds_gfx_bo);
+ NULL, NULL, &adev->gds.gds_gfx_bo);
if (r)
return r;
r = amdgpu_bo_create(adev, adev->gds.gws.gfx_partition_size,
PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_GWS, 0,
- NULL, &adev->gds.gws_gfx_bo);
+ NULL, NULL, &adev->gds.gws_gfx_bo);
if (r)
return r;
r = amdgpu_bo_create(adev, adev->gds.oa.gfx_partition_size,
PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_OA, 0,
- NULL, &adev->gds.oa_gfx_bo);
+ NULL, NULL, &adev->gds.oa_gfx_bo);
if (r)
return r;
for (i = 0; i < adev->gfx.num_compute_rings; i++)
amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
- amdgpu_wb_free(adev, adev->gfx.ce_sync_offs);
-
gfx_v7_0_cp_compute_fini(adev);
gfx_v7_0_rlc_fini(adev);
gfx_v7_0_mec_fini(adev);
.set_powergating_state = gfx_v7_0_set_powergating_state,
};
-/**
- * gfx_v7_0_ring_is_lockup - check if the 3D engine is locked up
- *
- * @adev: amdgpu_device pointer
- * @ring: amdgpu_ring structure holding ring information
- *
- * Check if the 3D engine is locked up (CIK).
- * Returns true if the engine is locked, false if not.
- */
-static bool gfx_v7_0_ring_is_lockup(struct amdgpu_ring *ring)
-{
- if (gfx_v7_0_is_idle(ring->adev)) {
- amdgpu_ring_lockup_update(ring);
- return false;
- }
- return amdgpu_ring_test_lockup(ring);
-}
-
static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
.get_rptr = gfx_v7_0_ring_get_rptr_gfx,
.get_wptr = gfx_v7_0_ring_get_wptr_gfx,
.emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
.test_ring = gfx_v7_0_ring_test_ring,
.test_ib = gfx_v7_0_ring_test_ib,
- .is_lockup = gfx_v7_0_ring_is_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
.emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
.test_ring = gfx_v7_0_ring_test_ring,
.test_ib = gfx_v7_0_ring_test_ib,
- .is_lockup = gfx_v7_0_ring_is_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");
MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");
+MODULE_FIRMWARE("amdgpu/stoney_ce.bin");
+MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");
+MODULE_FIRMWARE("amdgpu/stoney_me.bin");
+MODULE_FIRMWARE("amdgpu/stoney_mec.bin");
+MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");
+
MODULE_FIRMWARE("amdgpu/tonga_ce.bin");
MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");
MODULE_FIRMWARE("amdgpu/tonga_me.bin");
mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
- mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
+ mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
- mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
+ mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+ mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+ mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
};
static const u32 golden_settings_fiji_a10[] =
mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
mmDB_DEBUG2, 0xf00fffff, 0x00000400,
mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
- mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x00000100,
mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+ mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
+ mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
- mmTCC_CTRL, 0x00100000, 0xf30fff7f,
+ mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+ mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
- mmTCP_CHAN_STEER_HI, 0xffffffff, 0x7d6cf5e4,
- mmTCP_CHAN_STEER_LO, 0xffffffff, 0x3928b1a0,
+ mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
};
static const u32 fiji_mgcg_cgcg_init[] =
{
- mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffc0,
+ mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+ mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+ mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+ mmPCIE_DATA, 0x000f0000, 0x00000000,
+ mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+ mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
};
mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
};
+static const u32 stoney_golden_settings_a11[] =
+{
+ mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+ mmGB_GPU_ID, 0x0000000f, 0x00000000,
+ mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+ mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+ mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
+ mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+ mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+ mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+ mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,
+ mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,
+};
+
+static const u32 stoney_golden_common_all[] =
+{
+ mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+ mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,
+ mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
+ mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
+ mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+ mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+ mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+ mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+};
+
+static const u32 stoney_mgcg_cgcg_init[] =
+{
+ mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+ mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
+ mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
+ mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
+ mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
+ mmATC_MISC_CG, 0xffffffff, 0x000c0200,
+};
+
static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
cz_golden_common_all,
(const u32)ARRAY_SIZE(cz_golden_common_all));
break;
+ case CHIP_STONEY:
+ amdgpu_program_register_sequence(adev,
+ stoney_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
+ amdgpu_program_register_sequence(adev,
+ stoney_golden_settings_a11,
+ (const u32)ARRAY_SIZE(stoney_golden_settings_a11));
+ amdgpu_program_register_sequence(adev,
+ stoney_golden_common_all,
+ (const u32)ARRAY_SIZE(stoney_golden_common_all));
+ break;
default:
break;
}
case CHIP_FIJI:
chip_name = "fiji";
break;
+ case CHIP_STONEY:
+ chip_name = "stoney";
+ break;
default:
BUG();
}
adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
- snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
- err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
- if (!err) {
- err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
- if (err)
- goto out;
- cp_hdr = (const struct gfx_firmware_header_v1_0 *)
- adev->gfx.mec2_fw->data;
- adev->gfx.mec2_fw_version = le32_to_cpu(
- cp_hdr->header.ucode_version);
- adev->gfx.mec2_feature_version = le32_to_cpu(
- cp_hdr->ucode_feature_version);
- } else {
- err = 0;
- adev->gfx.mec2_fw = NULL;
+ if (adev->asic_type != CHIP_STONEY) {
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+ err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+ if (!err) {
+ err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
+ if (err)
+ goto out;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.mec2_fw->data;
+ adev->gfx.mec2_fw_version =
+ le32_to_cpu(cp_hdr->header.ucode_version);
+ adev->gfx.mec2_feature_version =
+ le32_to_cpu(cp_hdr->ucode_feature_version);
+ } else {
+ err = 0;
+ adev->gfx.mec2_fw = NULL;
+ }
}
if (adev->firmware.smu_load) {
r = amdgpu_bo_create(adev,
adev->gfx.mec.num_mec *adev->gfx.mec.num_pipe * MEC_HPD_SIZE * 2,
PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
+ AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
&adev->gfx.mec.hpd_eop_obj);
if (r) {
dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
return 0;
}
+static void gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
+{
+ u32 gb_addr_config;
+ u32 mc_shared_chmap, mc_arb_ramcfg;
+ u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
+ u32 tmp;
+
+ switch (adev->asic_type) {
+ case CHIP_TOPAZ:
+ adev->gfx.config.max_shader_engines = 1;
+ adev->gfx.config.max_tile_pipes = 2;
+ adev->gfx.config.max_cu_per_sh = 6;
+ adev->gfx.config.max_sh_per_se = 1;
+ adev->gfx.config.max_backends_per_se = 2;
+ adev->gfx.config.max_texture_channel_caches = 2;
+ adev->gfx.config.max_gprs = 256;
+ adev->gfx.config.max_gs_threads = 32;
+ adev->gfx.config.max_hw_contexts = 8;
+
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+ gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
+ break;
+ case CHIP_FIJI:
+ adev->gfx.config.max_shader_engines = 4;
+ adev->gfx.config.max_tile_pipes = 16;
+ adev->gfx.config.max_cu_per_sh = 16;
+ adev->gfx.config.max_sh_per_se = 1;
+ adev->gfx.config.max_backends_per_se = 4;
+ adev->gfx.config.max_texture_channel_caches = 8;
+ adev->gfx.config.max_gprs = 256;
+ adev->gfx.config.max_gs_threads = 32;
+ adev->gfx.config.max_hw_contexts = 8;
+
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+ gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
+ break;
+ case CHIP_TONGA:
+ adev->gfx.config.max_shader_engines = 4;
+ adev->gfx.config.max_tile_pipes = 8;
+ adev->gfx.config.max_cu_per_sh = 8;
+ adev->gfx.config.max_sh_per_se = 1;
+ adev->gfx.config.max_backends_per_se = 2;
+ adev->gfx.config.max_texture_channel_caches = 8;
+ adev->gfx.config.max_gprs = 256;
+ adev->gfx.config.max_gs_threads = 32;
+ adev->gfx.config.max_hw_contexts = 8;
+
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+ gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
+ break;
+ case CHIP_CARRIZO:
+ adev->gfx.config.max_shader_engines = 1;
+ adev->gfx.config.max_tile_pipes = 2;
+ adev->gfx.config.max_sh_per_se = 1;
+ adev->gfx.config.max_backends_per_se = 2;
+
+ switch (adev->pdev->revision) {
+ case 0xc4:
+ case 0x84:
+ case 0xc8:
+ case 0xcc:
+ case 0xe1:
+ case 0xe3:
+ /* B10 */
+ adev->gfx.config.max_cu_per_sh = 8;
+ break;
+ case 0xc5:
+ case 0x81:
+ case 0x85:
+ case 0xc9:
+ case 0xcd:
+ case 0xe2:
+ case 0xe4:
+ /* B8 */
+ adev->gfx.config.max_cu_per_sh = 6;
+ break;
+ case 0xc6:
+ case 0xca:
+ case 0xce:
+ case 0x88:
+ /* B6 */
+ adev->gfx.config.max_cu_per_sh = 6;
+ break;
+ case 0xc7:
+ case 0x87:
+ case 0xcb:
+ case 0xe5:
+ case 0x89:
+ default:
+ /* B4 */
+ adev->gfx.config.max_cu_per_sh = 4;
+ break;
+ }
+
+ adev->gfx.config.max_texture_channel_caches = 2;
+ adev->gfx.config.max_gprs = 256;
+ adev->gfx.config.max_gs_threads = 32;
+ adev->gfx.config.max_hw_contexts = 8;
+
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+ gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
+ break;
+ case CHIP_STONEY:
+ adev->gfx.config.max_shader_engines = 1;
+ adev->gfx.config.max_tile_pipes = 2;
+ adev->gfx.config.max_sh_per_se = 1;
+ adev->gfx.config.max_backends_per_se = 1;
+
+ switch (adev->pdev->revision) {
+ case 0xc0:
+ case 0xc1:
+ case 0xc2:
+ case 0xc4:
+ case 0xc8:
+ case 0xc9:
+ adev->gfx.config.max_cu_per_sh = 3;
+ break;
+ case 0xd0:
+ case 0xd1:
+ case 0xd2:
+ default:
+ adev->gfx.config.max_cu_per_sh = 2;
+ break;
+ }
+
+ adev->gfx.config.max_texture_channel_caches = 2;
+ adev->gfx.config.max_gprs = 256;
+ adev->gfx.config.max_gs_threads = 16;
+ adev->gfx.config.max_hw_contexts = 8;
+
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+ gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
+ break;
+ default:
+ adev->gfx.config.max_shader_engines = 2;
+ adev->gfx.config.max_tile_pipes = 4;
+ adev->gfx.config.max_cu_per_sh = 2;
+ adev->gfx.config.max_sh_per_se = 1;
+ adev->gfx.config.max_backends_per_se = 2;
+ adev->gfx.config.max_texture_channel_caches = 4;
+ adev->gfx.config.max_gprs = 256;
+ adev->gfx.config.max_gs_threads = 32;
+ adev->gfx.config.max_hw_contexts = 8;
+
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+ gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
+ break;
+ }
+
+ mc_shared_chmap = RREG32(mmMC_SHARED_CHMAP);
+ adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
+ mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
+
+ adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
+ adev->gfx.config.mem_max_burst_length_bytes = 256;
+ if (adev->flags & AMD_IS_APU) {
+ /* Get memory bank mapping mode. */
+ tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
+ dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
+ dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
+
+ tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
+ dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
+ dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
+
+ /* Validate settings in case only one DIMM installed. */
+ if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
+ dimm00_addr_map = 0;
+ if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
+ dimm01_addr_map = 0;
+ if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
+ dimm10_addr_map = 0;
+ if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
+ dimm11_addr_map = 0;
+
+ /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
+ /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
+ if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
+ adev->gfx.config.mem_row_size_in_kb = 2;
+ else
+ adev->gfx.config.mem_row_size_in_kb = 1;
+ } else {
+ tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
+ adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
+ if (adev->gfx.config.mem_row_size_in_kb > 4)
+ adev->gfx.config.mem_row_size_in_kb = 4;
+ }
+
+ adev->gfx.config.shader_engine_tile_size = 32;
+ adev->gfx.config.num_gpus = 1;
+ adev->gfx.config.multi_gpu_tile_size = 64;
+
+ /* fix up row size */
+ switch (adev->gfx.config.mem_row_size_in_kb) {
+ case 1:
+ default:
+ gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
+ break;
+ case 2:
+ gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
+ break;
+ case 4:
+ gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
+ break;
+ }
+ adev->gfx.config.gb_addr_config = gb_addr_config;
+}
+
static int gfx_v8_0_sw_init(void *handle)
{
int i, r;
return r;
}
- r = amdgpu_wb_get(adev, &adev->gfx.ce_sync_offs);
- if (r) {
- DRM_ERROR("(%d) gfx.ce_sync_offs wb alloc failed\n", r);
- return r;
- }
-
/* set up the gfx ring */
for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
ring = &adev->gfx.gfx_ring[i];
/* reserve GDS, GWS and OA resource for gfx */
r = amdgpu_bo_create(adev, adev->gds.mem.gfx_partition_size,
PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GDS, 0,
+ AMDGPU_GEM_DOMAIN_GDS, 0, NULL,
NULL, &adev->gds.gds_gfx_bo);
if (r)
return r;
r = amdgpu_bo_create(adev, adev->gds.gws.gfx_partition_size,
PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_GWS, 0,
+ AMDGPU_GEM_DOMAIN_GWS, 0, NULL,
NULL, &adev->gds.gws_gfx_bo);
if (r)
return r;
r = amdgpu_bo_create(adev, adev->gds.oa.gfx_partition_size,
PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_OA, 0,
+ AMDGPU_GEM_DOMAIN_OA, 0, NULL,
NULL, &adev->gds.oa_gfx_bo);
if (r)
return r;
adev->gfx.ce_ram_size = 0x8000;
+ gfx_v8_0_gpu_early_init(adev);
+
return 0;
}
for (i = 0; i < adev->gfx.num_compute_rings; i++)
amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
- amdgpu_wb_free(adev, adev->gfx.ce_sync_offs);
-
gfx_v8_0_mec_fini(adev);
return 0;
case 6:
gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
- MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
- NUM_BANKS(ADDR_SURF_16_BANK));
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+ NUM_BANKS(ADDR_SURF_16_BANK));
+ break;
+ case 8:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+ NUM_BANKS(ADDR_SURF_16_BANK));
+ break;
+ case 9:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+ NUM_BANKS(ADDR_SURF_16_BANK));
+ break;
+ case 10:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_16_BANK));
+ break;
+ case 11:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_16_BANK));
+ break;
+ case 12:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
+ break;
+ case 13:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+ NUM_BANKS(ADDR_SURF_4_BANK));
+ break;
+ case 14:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+ NUM_BANKS(ADDR_SURF_4_BANK));
+ break;
+ case 7:
+ /* unused idx */
+ continue;
+ default:
+ gb_tile_moden = 0;
+ break;
+ };
+ adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden;
+ WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden);
+ }
+ break;
+ case CHIP_STONEY:
+ for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+ switch (reg_offset) {
+ case 0:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+ break;
+ case 1:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+ break;
+ case 2:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+ break;
+ case 3:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+ break;
+ case 4:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+ break;
+ case 5:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+ break;
+ case 6:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+ break;
+ case 8:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+ PIPE_CONFIG(ADDR_SURF_P2));
+ break;
+ case 9:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+ break;
+ case 10:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+ break;
+ case 11:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+ break;
+ case 13:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+ break;
+ case 14:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+ break;
+ case 15:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+ break;
+ case 16:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+ break;
+ case 18:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 19:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 20:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 21:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 22:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 24:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 25:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 26:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+ break;
+ case 27:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+ break;
+ case 28:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+ break;
+ case 29:
+ gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+ PIPE_CONFIG(ADDR_SURF_P2) |
+ MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+ SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+ break;
+ case 7:
+ case 12:
+ case 17:
+ case 23:
+ /* unused idx */
+ continue;
+ default:
+ gb_tile_moden = 0;
+ break;
+ };
+ adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
+ WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
+ }
+ for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
+ switch (reg_offset) {
+ case 0:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
+ break;
+ case 1:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
+ break;
+ case 2:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
+ break;
+ case 3:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
+ break;
+ case 4:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
+ break;
+ case 5:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
+ break;
+ case 6:
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
break;
case 8:
- gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
NUM_BANKS(ADDR_SURF_16_BANK));
break;
case 9:
- gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
NUM_BANKS(ADDR_SURF_16_BANK));
break;
case 10:
- gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
- BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
- MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
NUM_BANKS(ADDR_SURF_16_BANK));
break;
case 11:
- gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
- BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
- MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
NUM_BANKS(ADDR_SURF_16_BANK));
break;
case 12:
gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
- BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
- MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
- NUM_BANKS(ADDR_SURF_8_BANK));
+ BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+ NUM_BANKS(ADDR_SURF_16_BANK));
break;
case 13:
gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
- MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
- NUM_BANKS(ADDR_SURF_4_BANK));
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+ NUM_BANKS(ADDR_SURF_16_BANK));
break;
case 14:
gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
- MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
- NUM_BANKS(ADDR_SURF_4_BANK));
+ MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+ NUM_BANKS(ADDR_SURF_8_BANK));
break;
case 7:
/* unused idx */
static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
{
- u32 gb_addr_config;
- u32 mc_shared_chmap, mc_arb_ramcfg;
- u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
u32 tmp;
int i;
- switch (adev->asic_type) {
- case CHIP_TOPAZ:
- adev->gfx.config.max_shader_engines = 1;
- adev->gfx.config.max_tile_pipes = 2;
- adev->gfx.config.max_cu_per_sh = 6;
- adev->gfx.config.max_sh_per_se = 1;
- adev->gfx.config.max_backends_per_se = 2;
- adev->gfx.config.max_texture_channel_caches = 2;
- adev->gfx.config.max_gprs = 256;
- adev->gfx.config.max_gs_threads = 32;
- adev->gfx.config.max_hw_contexts = 8;
-
- adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
- adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
- adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
- adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
- gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
- break;
- case CHIP_FIJI:
- adev->gfx.config.max_shader_engines = 4;
- adev->gfx.config.max_tile_pipes = 16;
- adev->gfx.config.max_cu_per_sh = 16;
- adev->gfx.config.max_sh_per_se = 1;
- adev->gfx.config.max_backends_per_se = 4;
- adev->gfx.config.max_texture_channel_caches = 8;
- adev->gfx.config.max_gprs = 256;
- adev->gfx.config.max_gs_threads = 32;
- adev->gfx.config.max_hw_contexts = 8;
-
- adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
- adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
- adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
- adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
- gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
- break;
- case CHIP_TONGA:
- adev->gfx.config.max_shader_engines = 4;
- adev->gfx.config.max_tile_pipes = 8;
- adev->gfx.config.max_cu_per_sh = 8;
- adev->gfx.config.max_sh_per_se = 1;
- adev->gfx.config.max_backends_per_se = 2;
- adev->gfx.config.max_texture_channel_caches = 8;
- adev->gfx.config.max_gprs = 256;
- adev->gfx.config.max_gs_threads = 32;
- adev->gfx.config.max_hw_contexts = 8;
-
- adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
- adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
- adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
- adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
- gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
- break;
- case CHIP_CARRIZO:
- adev->gfx.config.max_shader_engines = 1;
- adev->gfx.config.max_tile_pipes = 2;
- adev->gfx.config.max_sh_per_se = 1;
- adev->gfx.config.max_backends_per_se = 2;
-
- switch (adev->pdev->revision) {
- case 0xc4:
- case 0x84:
- case 0xc8:
- case 0xcc:
- /* B10 */
- adev->gfx.config.max_cu_per_sh = 8;
- break;
- case 0xc5:
- case 0x81:
- case 0x85:
- case 0xc9:
- case 0xcd:
- /* B8 */
- adev->gfx.config.max_cu_per_sh = 6;
- break;
- case 0xc6:
- case 0xca:
- case 0xce:
- /* B6 */
- adev->gfx.config.max_cu_per_sh = 6;
- break;
- case 0xc7:
- case 0x87:
- case 0xcb:
- default:
- /* B4 */
- adev->gfx.config.max_cu_per_sh = 4;
- break;
- }
-
- adev->gfx.config.max_texture_channel_caches = 2;
- adev->gfx.config.max_gprs = 256;
- adev->gfx.config.max_gs_threads = 32;
- adev->gfx.config.max_hw_contexts = 8;
-
- adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
- adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
- adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
- adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
- gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
- break;
- default:
- adev->gfx.config.max_shader_engines = 2;
- adev->gfx.config.max_tile_pipes = 4;
- adev->gfx.config.max_cu_per_sh = 2;
- adev->gfx.config.max_sh_per_se = 1;
- adev->gfx.config.max_backends_per_se = 2;
- adev->gfx.config.max_texture_channel_caches = 4;
- adev->gfx.config.max_gprs = 256;
- adev->gfx.config.max_gs_threads = 32;
- adev->gfx.config.max_hw_contexts = 8;
-
- adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
- adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
- adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
- adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
- gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
- break;
- }
-
tmp = RREG32(mmGRBM_CNTL);
tmp = REG_SET_FIELD(tmp, GRBM_CNTL, READ_TIMEOUT, 0xff);
WREG32(mmGRBM_CNTL, tmp);
- mc_shared_chmap = RREG32(mmMC_SHARED_CHMAP);
- adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
- mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
-
- adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
- adev->gfx.config.mem_max_burst_length_bytes = 256;
- if (adev->flags & AMD_IS_APU) {
- /* Get memory bank mapping mode. */
- tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
- dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
- dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
-
- tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
- dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
- dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
-
- /* Validate settings in case only one DIMM installed. */
- if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
- dimm00_addr_map = 0;
- if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
- dimm01_addr_map = 0;
- if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
- dimm10_addr_map = 0;
- if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
- dimm11_addr_map = 0;
-
- /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
- /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
- if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
- adev->gfx.config.mem_row_size_in_kb = 2;
- else
- adev->gfx.config.mem_row_size_in_kb = 1;
- } else {
- tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
- adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
- if (adev->gfx.config.mem_row_size_in_kb > 4)
- adev->gfx.config.mem_row_size_in_kb = 4;
- }
-
- adev->gfx.config.shader_engine_tile_size = 32;
- adev->gfx.config.num_gpus = 1;
- adev->gfx.config.multi_gpu_tile_size = 64;
-
- /* fix up row size */
- switch (adev->gfx.config.mem_row_size_in_kb) {
- case 1:
- default:
- gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
- break;
- case 2:
- gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
- break;
- case 4:
- gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
- break;
- }
- adev->gfx.config.gb_addr_config = gb_addr_config;
-
- WREG32(mmGB_ADDR_CONFIG, gb_addr_config);
- WREG32(mmHDP_ADDR_CONFIG, gb_addr_config);
- WREG32(mmDMIF_ADDR_CALC, gb_addr_config);
+ WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
+ WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
+ WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
WREG32(mmSDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET,
- gb_addr_config & 0x70);
+ adev->gfx.config.gb_addr_config & 0x70);
WREG32(mmSDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET,
- gb_addr_config & 0x70);
- WREG32(mmUVD_UDEC_ADDR_CONFIG, gb_addr_config);
- WREG32(mmUVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
- WREG32(mmUVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
+ adev->gfx.config.gb_addr_config & 0x70);
+ WREG32(mmUVD_UDEC_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
+ WREG32(mmUVD_UDEC_DB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
+ WREG32(mmUVD_UDEC_DBW_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
gfx_v8_0_tiling_mode_table_init(adev);
if (i == 0) {
tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);
tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
- tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
+ tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
SH_MEM_ALIGNMENT_MODE_UNALIGNED);
WREG32(mmSH_MEM_CONFIG, tmp);
} else {
tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);
tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_NC);
- tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
+ tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
SH_MEM_ALIGNMENT_MODE_UNALIGNED);
WREG32(mmSH_MEM_CONFIG, tmp);
}
WREG32(mmRLC_CNTL, tmp);
/* carrizo do enable cp interrupt after cp inited */
- if (adev->asic_type != CHIP_CARRIZO)
+ if (!(adev->flags & AMD_IS_APU))
gfx_v8_0_enable_gui_idle_interrupt(adev, true);
udelay(50);
amdgpu_ring_write(ring, 0x00000002);
amdgpu_ring_write(ring, 0x00000000);
break;
+ case CHIP_STONEY:
+ amdgpu_ring_write(ring, 0x00000000);
+ amdgpu_ring_write(ring, 0x00000000);
+ break;
default:
BUG();
}
sizeof(struct vi_mqd),
PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
- &ring->mqd_obj);
+ NULL, &ring->mqd_obj);
if (r) {
dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
return r;
/* enable the doorbell if requested */
if (use_doorbell) {
if ((adev->asic_type == CHIP_CARRIZO) ||
- (adev->asic_type == CHIP_FIJI)) {
+ (adev->asic_type == CHIP_FIJI) ||
+ (adev->asic_type == CHIP_STONEY)) {
WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER,
AMDGPU_DOORBELL_KIQ << 2);
WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER,
{
int r;
- if (adev->asic_type != CHIP_CARRIZO)
+ if (!(adev->flags & AMD_IS_APU))
gfx_v8_0_enable_gui_idle_interrupt(adev, false);
if (!adev->firmware.smu_load) {
DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
amdgpu_ring_write(ring, lower_32_bits(seq));
amdgpu_ring_write(ring, upper_32_bits(seq));
+
}
/**
return true;
}
-static void gfx_v8_0_ce_sync_me(struct amdgpu_ring *ring)
+static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
+ unsigned vm_id, uint64_t pd_addr)
{
- struct amdgpu_device *adev = ring->adev;
- u64 gpu_addr = adev->wb.gpu_addr + adev->gfx.ce_sync_offs * 4;
-
- /* instruct DE to set a magic number */
- amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
- amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
- WRITE_DATA_DST_SEL(5)));
- amdgpu_ring_write(ring, gpu_addr & 0xfffffffc);
- amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xffffffff);
- amdgpu_ring_write(ring, 1);
+ int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+ uint32_t seq = ring->fence_drv.sync_seq[ring->idx];
+ uint64_t addr = ring->fence_drv.gpu_addr;
- /* let CE wait till condition satisfied */
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
- amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
- WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
- WAIT_REG_MEM_FUNCTION(3) | /* == */
- WAIT_REG_MEM_ENGINE(2))); /* ce */
- amdgpu_ring_write(ring, gpu_addr & 0xfffffffc);
- amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xffffffff);
- amdgpu_ring_write(ring, 1);
+ amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
+ WAIT_REG_MEM_FUNCTION(3))); /* equal */
+ amdgpu_ring_write(ring, addr & 0xfffffffc);
+ amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+ amdgpu_ring_write(ring, seq);
amdgpu_ring_write(ring, 0xffffffff);
amdgpu_ring_write(ring, 4); /* poll interval */
- /* instruct CE to reset wb of ce_sync to zero */
- amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
- amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
- WRITE_DATA_DST_SEL(5) |
- WR_CONFIRM));
- amdgpu_ring_write(ring, gpu_addr & 0xfffffffc);
- amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xffffffff);
- amdgpu_ring_write(ring, 0);
-}
-
-static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
- unsigned vm_id, uint64_t pd_addr)
-{
- int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+ if (usepfp) {
+ /* synce CE with ME to prevent CE fetch CEIB before context switch done */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
+ }
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
- WRITE_DATA_DST_SEL(0)));
+ WRITE_DATA_DST_SEL(0)) |
+ WR_CONFIRM);
if (vm_id < 8) {
amdgpu_ring_write(ring,
(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
/* sync PFP to ME, otherwise we might get invalid PFP reads */
amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
amdgpu_ring_write(ring, 0x0);
-
- /* synce CE with ME to prevent CE fetch CEIB before context switch done */
- gfx_v8_0_ce_sync_me(ring);
- }
-}
-
-static bool gfx_v8_0_ring_is_lockup(struct amdgpu_ring *ring)
-{
- if (gfx_v8_0_is_idle(ring->adev)) {
- amdgpu_ring_lockup_update(ring);
- return false;
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
}
- return amdgpu_ring_test_lockup(ring);
}
static u32 gfx_v8_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
EOP_TC_ACTION_EN |
+ EOP_TC_WB_ACTION_EN |
EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
EVENT_INDEX(5)));
amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
.test_ring = gfx_v8_0_ring_test_ring,
.test_ib = gfx_v8_0_ring_test_ib,
- .is_lockup = gfx_v8_0_ring_is_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
.test_ring = gfx_v8_0_ring_test_ring,
.test_ib = gfx_v8_0_ring_test_ib,
- .is_lockup = gfx_v8_0_ring_is_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
return 0;
}
+/**
+ * gmc_v8_0_set_fault_enable_default - update VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,
+ bool value)
+{
+ u32 tmp;
+
+ tmp = RREG32(mmVM_CONTEXT1_CNTL);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ WREG32(mmVM_CONTEXT1_CNTL, tmp);
+}
+
/**
* gmc_v7_0_gart_enable - gart enable
*
tmp = RREG32(mmVM_CONTEXT1_CNTL);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
amdgpu_vm_block_size - 9);
WREG32(mmVM_CONTEXT1_CNTL, tmp);
+ if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
+ gmc_v7_0_set_fault_enable_default(adev, false);
+ else
+ gmc_v7_0_set_fault_enable_default(adev, true);
if (adev->asic_type == CHIP_KAVERI) {
tmp = RREG32(mmCHUB_CONTROL);
if (adev->vm_manager.enabled) {
for (i = 0; i < AMDGPU_NUM_VM; ++i)
- amdgpu_fence_unref(&adev->vm_manager.active[i]);
+ fence_put(adev->vm_manager.active[i]);
gmc_v7_0_vm_fini(adev);
adev->vm_manager.enabled = false;
}
if (adev->vm_manager.enabled) {
for (i = 0; i < AMDGPU_NUM_VM; ++i)
- amdgpu_fence_unref(&adev->vm_manager.active[i]);
+ fence_put(adev->vm_manager.active[i]);
gmc_v7_0_vm_fini(adev);
adev->vm_manager.enabled = false;
}
addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
+ /* reset addr and status */
+ WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
+
+ if (!addr && !status)
+ return 0;
+
+ if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
+ gmc_v7_0_set_fault_enable_default(adev, false);
+
dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
entry->src_id, entry->src_data);
dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
status);
gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
- /* reset addr and status */
- WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
return 0;
}
mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
};
+static const u32 stoney_mgcg_cgcg_init[] =
+{
+ mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
+};
+
+
static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
{
switch (adev->asic_type) {
cz_mgcg_cgcg_init,
(const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
break;
+ case CHIP_STONEY:
+ amdgpu_program_register_sequence(adev,
+ stoney_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
+ break;
default:
break;
}
chip_name = "fiji";
break;
case CHIP_CARRIZO:
+ case CHIP_STONEY:
return 0;
default: BUG();
}
return 0;
}
+/**
+ * gmc_v8_0_set_fault_enable_default - update VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device *adev,
+ bool value)
+{
+ u32 tmp;
+
+ tmp = RREG32(mmVM_CONTEXT1_CNTL);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+ EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ WREG32(mmVM_CONTEXT1_CNTL, tmp);
+}
+
/**
* gmc_v8_0_gart_enable - gart enable
*
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
amdgpu_vm_block_size - 9);
WREG32(mmVM_CONTEXT1_CNTL, tmp);
+ if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
+ gmc_v8_0_set_fault_enable_default(adev, false);
+ else
+ gmc_v8_0_set_fault_enable_default(adev, true);
gmc_v8_0_gart_flush_gpu_tlb(adev, 0);
DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
if (adev->vm_manager.enabled) {
for (i = 0; i < AMDGPU_NUM_VM; ++i)
- amdgpu_fence_unref(&adev->vm_manager.active[i]);
+ fence_put(adev->vm_manager.active[i]);
gmc_v8_0_vm_fini(adev);
adev->vm_manager.enabled = false;
}
if (adev->vm_manager.enabled) {
for (i = 0; i < AMDGPU_NUM_VM; ++i)
- amdgpu_fence_unref(&adev->vm_manager.active[i]);
+ fence_put(adev->vm_manager.active[i]);
gmc_v8_0_vm_fini(adev);
adev->vm_manager.enabled = false;
}
addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
+ /* reset addr and status */
+ WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
+
+ if (!addr && !status)
+ return 0;
+
+ if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
+ gmc_v8_0_set_fault_enable_default(adev, false);
+
dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
entry->src_id, entry->src_data);
dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
status);
gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
- /* reset addr and status */
- WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
return 0;
}
ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
true, AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, toc_buf);
+ NULL, NULL, toc_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for TOC buffer\n");
return -ENOMEM;
{
/* powerdown unused blocks for now */
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int ret;
+
+ /* init the sysfs and debugfs files late */
+ ret = amdgpu_pm_sysfs_init(adev);
+ if (ret)
+ return ret;
kv_dpm_powergate_acp(adev, true);
kv_dpm_powergate_samu(adev, true);
adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
if (amdgpu_dpm == 1)
amdgpu_pm_print_power_states(adev);
- ret = amdgpu_pm_sysfs_init(adev);
- if (ret)
- goto dpm_failed;
mutex_unlock(&adev->pm.mutex);
DRM_INFO("amdgpu: dpm initialized\n");
{
const char *chip_name;
char fw_name[30];
- int err, i;
+ int err = 0, i;
struct amdgpu_firmware_info *info = NULL;
const struct common_firmware_header *header = NULL;
const struct sdma_firmware_header_v1_0 *hdr;
default: BUG();
}
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
if (i == 0)
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name);
else
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name);
- err = request_firmware(&adev->sdma[i].fw, fw_name, adev->dev);
+ err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);
if (err)
goto out;
- err = amdgpu_ucode_validate(adev->sdma[i].fw);
+ err = amdgpu_ucode_validate(adev->sdma.instance[i].fw);
if (err)
goto out;
- hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data;
- adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
- adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
- if (adev->sdma[i].feature_version >= 20)
- adev->sdma[i].burst_nop = true;
+ hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
+ adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
+ adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
+ if (adev->sdma.instance[i].feature_version >= 20)
+ adev->sdma.instance[i].burst_nop = true;
if (adev->firmware.smu_load) {
info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
- info->fw = adev->sdma[i].fw;
+ info->fw = adev->sdma.instance[i].fw;
header = (const struct common_firmware_header *)info->fw->data;
adev->firmware.fw_size +=
ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
printk(KERN_ERR
"sdma_v2_4: Failed to load firmware \"%s\"\n",
fw_name);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- release_firmware(adev->sdma[i].fw);
- adev->sdma[i].fw = NULL;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ release_firmware(adev->sdma.instance[i].fw);
+ adev->sdma.instance[i].fw = NULL;
}
}
return err;
static uint32_t sdma_v2_4_ring_get_wptr(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
- int me = (ring == &ring->adev->sdma[0].ring) ? 0 : 1;
+ int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
u32 wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me]) >> 2;
return wptr;
static void sdma_v2_4_ring_set_wptr(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
- int me = (ring == &ring->adev->sdma[0].ring) ? 0 : 1;
+ int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], ring->wptr << 2);
}
static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
{
- struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring);
+ struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring);
int i;
for (i = 0; i < count; i++)
{
u32 ref_and_mask = 0;
- if (ring == &ring->adev->sdma[0].ring)
+ if (ring == &ring->adev->sdma.instance[0].ring)
ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA0, 1);
else
ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA1, 1);
*/
static void sdma_v2_4_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma[1].ring;
+ struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
+ struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl, ib_cntl;
int i;
(adev->mman.buffer_funcs_ring == sdma1))
amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);
rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
sdma_v2_4_rlc_stop(adev);
}
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
f32_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]);
if (enable)
f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, 0);
u32 wb_offset;
int i, j, r;
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- ring = &adev->sdma[i].ring;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
wb_offset = (ring->rptr_offs * 4);
mutex_lock(&adev->srbm_mutex);
const __le32 *fw_data;
u32 fw_size;
int i, j;
- bool smc_loads_fw = false; /* XXX fix me */
-
- if (!adev->sdma[0].fw || !adev->sdma[1].fw)
- return -EINVAL;
/* halt the MEs */
sdma_v2_4_enable(adev, false);
- if (smc_loads_fw) {
- /* XXX query SMC for fw load complete */
- } else {
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data;
- amdgpu_ucode_print_sdma_hdr(&hdr->header);
- fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
- fw_data = (const __le32 *)
- (adev->sdma[i].fw->data +
- le32_to_cpu(hdr->header.ucode_array_offset_bytes));
- WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0);
- for (j = 0; j < fw_size; j++)
- WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++));
- WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma[i].fw_version);
- }
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (!adev->sdma.instance[i].fw)
+ return -EINVAL;
+ hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
+ amdgpu_ucode_print_sdma_hdr(&hdr->header);
+ fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+ fw_data = (const __le32 *)
+ (adev->sdma.instance[i].fw->data +
+ le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+ WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0);
+ for (j = 0; j < fw_size; j++)
+ WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++));
+ WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version);
}
return 0;
*/
static void sdma_v2_4_vm_pad_ib(struct amdgpu_ib *ib)
{
- struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring);
+ struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ib->ring);
u32 pad_count;
int i;
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ adev->sdma.num_instances = SDMA_MAX_INSTANCE;
+
sdma_v2_4_set_ring_funcs(adev);
sdma_v2_4_set_buffer_funcs(adev);
sdma_v2_4_set_vm_pte_funcs(adev);
static int sdma_v2_4_sw_init(void *handle)
{
struct amdgpu_ring *ring;
- int r;
+ int r, i;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
/* SDMA trap event */
- r = amdgpu_irq_add_id(adev, 224, &adev->sdma_trap_irq);
+ r = amdgpu_irq_add_id(adev, 224, &adev->sdma.trap_irq);
if (r)
return r;
/* SDMA Privileged inst */
- r = amdgpu_irq_add_id(adev, 241, &adev->sdma_illegal_inst_irq);
+ r = amdgpu_irq_add_id(adev, 241, &adev->sdma.illegal_inst_irq);
if (r)
return r;
/* SDMA Privileged inst */
- r = amdgpu_irq_add_id(adev, 247, &adev->sdma_illegal_inst_irq);
+ r = amdgpu_irq_add_id(adev, 247, &adev->sdma.illegal_inst_irq);
if (r)
return r;
return r;
}
- ring = &adev->sdma[0].ring;
- ring->ring_obj = NULL;
- ring->use_doorbell = false;
-
- ring = &adev->sdma[1].ring;
- ring->ring_obj = NULL;
- ring->use_doorbell = false;
-
- ring = &adev->sdma[0].ring;
- sprintf(ring->name, "sdma0");
- r = amdgpu_ring_init(adev, ring, 256 * 1024,
- SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
- &adev->sdma_trap_irq, AMDGPU_SDMA_IRQ_TRAP0,
- AMDGPU_RING_TYPE_SDMA);
- if (r)
- return r;
-
- ring = &adev->sdma[1].ring;
- sprintf(ring->name, "sdma1");
- r = amdgpu_ring_init(adev, ring, 256 * 1024,
- SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
- &adev->sdma_trap_irq, AMDGPU_SDMA_IRQ_TRAP1,
- AMDGPU_RING_TYPE_SDMA);
- if (r)
- return r;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
+ ring->ring_obj = NULL;
+ ring->use_doorbell = false;
+ sprintf(ring->name, "sdma%d", i);
+ r = amdgpu_ring_init(adev, ring, 256 * 1024,
+ SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
+ &adev->sdma.trap_irq,
+ (i == 0) ?
+ AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
+ AMDGPU_RING_TYPE_SDMA);
+ if (r)
+ return r;
+ }
return r;
}
static int sdma_v2_4_sw_fini(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int i;
- amdgpu_ring_fini(&adev->sdma[0].ring);
- amdgpu_ring_fini(&adev->sdma[1].ring);
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ amdgpu_ring_fini(&adev->sdma.instance[i].ring);
return 0;
}
dev_info(adev->dev, "VI SDMA registers\n");
dev_info(adev->dev, " SRBM_STATUS2=0x%08X\n",
RREG32(mmSRBM_STATUS2));
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
dev_info(adev->dev, " SDMA%d_STATUS_REG=0x%08X\n",
i, RREG32(mmSDMA0_STATUS_REG + sdma_offsets[i]));
dev_info(adev->dev, " SDMA%d_F32_CNTL=0x%08X\n",
case 0:
switch (queue_id) {
case 0:
- amdgpu_fence_process(&adev->sdma[0].ring);
+ amdgpu_fence_process(&adev->sdma.instance[0].ring);
break;
case 1:
/* XXX compute */
case 1:
switch (queue_id) {
case 0:
- amdgpu_fence_process(&adev->sdma[1].ring);
+ amdgpu_fence_process(&adev->sdma.instance[1].ring);
break;
case 1:
/* XXX compute */
.set_powergating_state = sdma_v2_4_set_powergating_state,
};
-/**
- * sdma_v2_4_ring_is_lockup - Check if the DMA engine is locked up
- *
- * @ring: amdgpu_ring structure holding ring information
- *
- * Check if the async DMA engine is locked up (VI).
- * Returns true if the engine appears to be locked up, false if not.
- */
-static bool sdma_v2_4_ring_is_lockup(struct amdgpu_ring *ring)
-{
-
- if (sdma_v2_4_is_idle(ring->adev)) {
- amdgpu_ring_lockup_update(ring);
- return false;
- }
- return amdgpu_ring_test_lockup(ring);
-}
-
static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {
.get_rptr = sdma_v2_4_ring_get_rptr,
.get_wptr = sdma_v2_4_ring_get_wptr,
.emit_hdp_flush = sdma_v2_4_ring_emit_hdp_flush,
.test_ring = sdma_v2_4_ring_test_ring,
.test_ib = sdma_v2_4_ring_test_ib,
- .is_lockup = sdma_v2_4_ring_is_lockup,
.insert_nop = sdma_v2_4_ring_insert_nop,
};
static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev)
{
- adev->sdma[0].ring.funcs = &sdma_v2_4_ring_funcs;
- adev->sdma[1].ring.funcs = &sdma_v2_4_ring_funcs;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ adev->sdma.instance[i].ring.funcs = &sdma_v2_4_ring_funcs;
}
static const struct amdgpu_irq_src_funcs sdma_v2_4_trap_irq_funcs = {
static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev)
{
- adev->sdma_trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
- adev->sdma_trap_irq.funcs = &sdma_v2_4_trap_irq_funcs;
- adev->sdma_illegal_inst_irq.funcs = &sdma_v2_4_illegal_inst_irq_funcs;
+ adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+ adev->sdma.trap_irq.funcs = &sdma_v2_4_trap_irq_funcs;
+ adev->sdma.illegal_inst_irq.funcs = &sdma_v2_4_illegal_inst_irq_funcs;
}
/**
{
if (adev->mman.buffer_funcs == NULL) {
adev->mman.buffer_funcs = &sdma_v2_4_buffer_funcs;
- adev->mman.buffer_funcs_ring = &adev->sdma[0].ring;
+ adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
}
}
{
if (adev->vm_manager.vm_pte_funcs == NULL) {
adev->vm_manager.vm_pte_funcs = &sdma_v2_4_vm_pte_funcs;
- adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring;
+ adev->vm_manager.vm_pte_funcs_ring = &adev->sdma.instance[0].ring;
adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true;
}
}
MODULE_FIRMWARE("amdgpu/carrizo_sdma1.bin");
MODULE_FIRMWARE("amdgpu/fiji_sdma.bin");
MODULE_FIRMWARE("amdgpu/fiji_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/stoney_sdma.bin");
static const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
{
mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100
};
+static const u32 stoney_golden_settings_a11[] =
+{
+ mmSDMA0_GFX_IB_CNTL, 0x00000100, 0x00000100,
+ mmSDMA0_POWER_CNTL, 0x00000800, 0x0003c800,
+ mmSDMA0_RLC0_IB_CNTL, 0x00000100, 0x00000100,
+ mmSDMA0_RLC1_IB_CNTL, 0x00000100, 0x00000100,
+};
+
+static const u32 stoney_mgcg_cgcg_init[] =
+{
+ mmSDMA0_CLK_CTRL, 0xffffffff, 0x00000100,
+};
+
/*
* sDMA - System DMA
* Starting with CIK, the GPU has new asynchronous
cz_golden_settings_a11,
(const u32)ARRAY_SIZE(cz_golden_settings_a11));
break;
+ case CHIP_STONEY:
+ amdgpu_program_register_sequence(adev,
+ stoney_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
+ amdgpu_program_register_sequence(adev,
+ stoney_golden_settings_a11,
+ (const u32)ARRAY_SIZE(stoney_golden_settings_a11));
+ break;
default:
break;
}
{
const char *chip_name;
char fw_name[30];
- int err, i;
+ int err = 0, i;
struct amdgpu_firmware_info *info = NULL;
const struct common_firmware_header *header = NULL;
const struct sdma_firmware_header_v1_0 *hdr;
case CHIP_CARRIZO:
chip_name = "carrizo";
break;
+ case CHIP_STONEY:
+ chip_name = "stoney";
+ break;
default: BUG();
}
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
if (i == 0)
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name);
else
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name);
- err = request_firmware(&adev->sdma[i].fw, fw_name, adev->dev);
+ err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);
if (err)
goto out;
- err = amdgpu_ucode_validate(adev->sdma[i].fw);
+ err = amdgpu_ucode_validate(adev->sdma.instance[i].fw);
if (err)
goto out;
- hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data;
- adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
- adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
- if (adev->sdma[i].feature_version >= 20)
- adev->sdma[i].burst_nop = true;
+ hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
+ adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
+ adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
+ if (adev->sdma.instance[i].feature_version >= 20)
+ adev->sdma.instance[i].burst_nop = true;
if (adev->firmware.smu_load) {
info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
- info->fw = adev->sdma[i].fw;
+ info->fw = adev->sdma.instance[i].fw;
header = (const struct common_firmware_header *)info->fw->data;
adev->firmware.fw_size +=
ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
printk(KERN_ERR
"sdma_v3_0: Failed to load firmware \"%s\"\n",
fw_name);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- release_firmware(adev->sdma[i].fw);
- adev->sdma[i].fw = NULL;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ release_firmware(adev->sdma.instance[i].fw);
+ adev->sdma.instance[i].fw = NULL;
}
}
return err;
/* XXX check if swapping is necessary on BE */
wptr = ring->adev->wb.wb[ring->wptr_offs] >> 2;
} else {
- int me = (ring == &ring->adev->sdma[0].ring) ? 0 : 1;
+ int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me]) >> 2;
}
adev->wb.wb[ring->wptr_offs] = ring->wptr << 2;
WDOORBELL32(ring->doorbell_index, ring->wptr << 2);
} else {
- int me = (ring == &ring->adev->sdma[0].ring) ? 0 : 1;
+ int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], ring->wptr << 2);
}
static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
{
- struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring);
+ struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring);
int i;
for (i = 0; i < count; i++)
{
u32 ref_and_mask = 0;
- if (ring == &ring->adev->sdma[0].ring)
+ if (ring == &ring->adev->sdma.instance[0].ring)
ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA0, 1);
else
ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA1, 1);
*/
static void sdma_v3_0_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma[1].ring;
+ struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
+ struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl, ib_cntl;
int i;
(adev->mman.buffer_funcs_ring == sdma1))
amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);
rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
u32 f32_cntl;
int i;
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
f32_cntl = RREG32(mmSDMA0_CNTL + sdma_offsets[i]);
if (enable)
f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
sdma_v3_0_rlc_stop(adev);
}
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
f32_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]);
if (enable)
f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, 0);
u32 doorbell;
int i, j, r;
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- ring = &adev->sdma[i].ring;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
wb_offset = (ring->rptr_offs * 4);
mutex_lock(&adev->srbm_mutex);
u32 fw_size;
int i, j;
- if (!adev->sdma[0].fw || !adev->sdma[1].fw)
- return -EINVAL;
-
/* halt the MEs */
sdma_v3_0_enable(adev, false);
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
- hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (!adev->sdma.instance[i].fw)
+ return -EINVAL;
+ hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
amdgpu_ucode_print_sdma_hdr(&hdr->header);
fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
fw_data = (const __le32 *)
- (adev->sdma[i].fw->data +
+ (adev->sdma.instance[i].fw->data +
le32_to_cpu(hdr->header.ucode_array_offset_bytes));
WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0);
for (j = 0; j < fw_size; j++)
WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++));
- WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma[i].fw_version);
+ WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version);
}
return 0;
*/
static int sdma_v3_0_start(struct amdgpu_device *adev)
{
- int r;
+ int r, i;
if (!adev->firmware.smu_load) {
r = sdma_v3_0_load_microcode(adev);
if (r)
return r;
} else {
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_SDMA0);
- if (r)
- return -EINVAL;
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_SDMA1);
- if (r)
- return -EINVAL;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+ (i == 0) ?
+ AMDGPU_UCODE_ID_SDMA0 :
+ AMDGPU_UCODE_ID_SDMA1);
+ if (r)
+ return -EINVAL;
+ }
}
/* unhalt the MEs */
*/
static void sdma_v3_0_vm_pad_ib(struct amdgpu_ib *ib)
{
- struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring);
+ struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ib->ring);
u32 pad_count;
int i;
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ switch (adev->asic_type) {
+ case CHIP_STONEY:
+ adev->sdma.num_instances = 1;
+ break;
+ default:
+ adev->sdma.num_instances = SDMA_MAX_INSTANCE;
+ break;
+ }
+
sdma_v3_0_set_ring_funcs(adev);
sdma_v3_0_set_buffer_funcs(adev);
sdma_v3_0_set_vm_pte_funcs(adev);
static int sdma_v3_0_sw_init(void *handle)
{
struct amdgpu_ring *ring;
- int r;
+ int r, i;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
/* SDMA trap event */
- r = amdgpu_irq_add_id(adev, 224, &adev->sdma_trap_irq);
+ r = amdgpu_irq_add_id(adev, 224, &adev->sdma.trap_irq);
if (r)
return r;
/* SDMA Privileged inst */
- r = amdgpu_irq_add_id(adev, 241, &adev->sdma_illegal_inst_irq);
+ r = amdgpu_irq_add_id(adev, 241, &adev->sdma.illegal_inst_irq);
if (r)
return r;
/* SDMA Privileged inst */
- r = amdgpu_irq_add_id(adev, 247, &adev->sdma_illegal_inst_irq);
+ r = amdgpu_irq_add_id(adev, 247, &adev->sdma.illegal_inst_irq);
if (r)
return r;
return r;
}
- ring = &adev->sdma[0].ring;
- ring->ring_obj = NULL;
- ring->use_doorbell = true;
- ring->doorbell_index = AMDGPU_DOORBELL_sDMA_ENGINE0;
-
- ring = &adev->sdma[1].ring;
- ring->ring_obj = NULL;
- ring->use_doorbell = true;
- ring->doorbell_index = AMDGPU_DOORBELL_sDMA_ENGINE1;
-
- ring = &adev->sdma[0].ring;
- sprintf(ring->name, "sdma0");
- r = amdgpu_ring_init(adev, ring, 256 * 1024,
- SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
- &adev->sdma_trap_irq, AMDGPU_SDMA_IRQ_TRAP0,
- AMDGPU_RING_TYPE_SDMA);
- if (r)
- return r;
-
- ring = &adev->sdma[1].ring;
- sprintf(ring->name, "sdma1");
- r = amdgpu_ring_init(adev, ring, 256 * 1024,
- SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
- &adev->sdma_trap_irq, AMDGPU_SDMA_IRQ_TRAP1,
- AMDGPU_RING_TYPE_SDMA);
- if (r)
- return r;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
+ ring->ring_obj = NULL;
+ ring->use_doorbell = true;
+ ring->doorbell_index = (i == 0) ?
+ AMDGPU_DOORBELL_sDMA_ENGINE0 : AMDGPU_DOORBELL_sDMA_ENGINE1;
+
+ sprintf(ring->name, "sdma%d", i);
+ r = amdgpu_ring_init(adev, ring, 256 * 1024,
+ SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
+ &adev->sdma.trap_irq,
+ (i == 0) ?
+ AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
+ AMDGPU_RING_TYPE_SDMA);
+ if (r)
+ return r;
+ }
return r;
}
static int sdma_v3_0_sw_fini(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int i;
- amdgpu_ring_fini(&adev->sdma[0].ring);
- amdgpu_ring_fini(&adev->sdma[1].ring);
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ amdgpu_ring_fini(&adev->sdma.instance[i].ring);
return 0;
}
dev_info(adev->dev, "VI SDMA registers\n");
dev_info(adev->dev, " SRBM_STATUS2=0x%08X\n",
RREG32(mmSRBM_STATUS2));
- for (i = 0; i < SDMA_MAX_INSTANCE; i++) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
dev_info(adev->dev, " SDMA%d_STATUS_REG=0x%08X\n",
i, RREG32(mmSDMA0_STATUS_REG + sdma_offsets[i]));
dev_info(adev->dev, " SDMA%d_F32_CNTL=0x%08X\n",
case 0:
switch (queue_id) {
case 0:
- amdgpu_fence_process(&adev->sdma[0].ring);
+ amdgpu_fence_process(&adev->sdma.instance[0].ring);
break;
case 1:
/* XXX compute */
case 1:
switch (queue_id) {
case 0:
- amdgpu_fence_process(&adev->sdma[1].ring);
+ amdgpu_fence_process(&adev->sdma.instance[1].ring);
break;
case 1:
/* XXX compute */
.set_powergating_state = sdma_v3_0_set_powergating_state,
};
-/**
- * sdma_v3_0_ring_is_lockup - Check if the DMA engine is locked up
- *
- * @ring: amdgpu_ring structure holding ring information
- *
- * Check if the async DMA engine is locked up (VI).
- * Returns true if the engine appears to be locked up, false if not.
- */
-static bool sdma_v3_0_ring_is_lockup(struct amdgpu_ring *ring)
-{
-
- if (sdma_v3_0_is_idle(ring->adev)) {
- amdgpu_ring_lockup_update(ring);
- return false;
- }
- return amdgpu_ring_test_lockup(ring);
-}
-
static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
.get_rptr = sdma_v3_0_ring_get_rptr,
.get_wptr = sdma_v3_0_ring_get_wptr,
.emit_hdp_flush = sdma_v3_0_ring_emit_hdp_flush,
.test_ring = sdma_v3_0_ring_test_ring,
.test_ib = sdma_v3_0_ring_test_ib,
- .is_lockup = sdma_v3_0_ring_is_lockup,
.insert_nop = sdma_v3_0_ring_insert_nop,
};
static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev)
{
- adev->sdma[0].ring.funcs = &sdma_v3_0_ring_funcs;
- adev->sdma[1].ring.funcs = &sdma_v3_0_ring_funcs;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ adev->sdma.instance[i].ring.funcs = &sdma_v3_0_ring_funcs;
}
static const struct amdgpu_irq_src_funcs sdma_v3_0_trap_irq_funcs = {
static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev)
{
- adev->sdma_trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
- adev->sdma_trap_irq.funcs = &sdma_v3_0_trap_irq_funcs;
- adev->sdma_illegal_inst_irq.funcs = &sdma_v3_0_illegal_inst_irq_funcs;
+ adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+ adev->sdma.trap_irq.funcs = &sdma_v3_0_trap_irq_funcs;
+ adev->sdma.illegal_inst_irq.funcs = &sdma_v3_0_illegal_inst_irq_funcs;
}
/**
{
if (adev->mman.buffer_funcs == NULL) {
adev->mman.buffer_funcs = &sdma_v3_0_buffer_funcs;
- adev->mman.buffer_funcs_ring = &adev->sdma[0].ring;
+ adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
}
}
{
if (adev->vm_manager.vm_pte_funcs == NULL) {
adev->vm_manager.vm_pte_funcs = &sdma_v3_0_vm_pte_funcs;
- adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring;
+ adev->vm_manager.vm_pte_funcs_ring = &adev->sdma.instance[0].ring;
adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true;
}
}
ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
true, AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, toc_buf);
+ NULL, NULL, toc_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for TOC buffer\n");
return -ENOMEM;
ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE,
true, AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
- NULL, smu_buf);
+ NULL, NULL, smu_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for SMU internal buffer\n");
return -ENOMEM;
int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- r = uvd_v4_2_hw_fini(adev);
+ r = amdgpu_uvd_suspend(adev);
if (r)
return r;
- r = amdgpu_uvd_suspend(adev);
+ r = uvd_v4_2_hw_fini(adev);
if (r)
return r;
.emit_semaphore = uvd_v4_2_ring_emit_semaphore,
.test_ring = uvd_v4_2_ring_test_ring,
.test_ib = uvd_v4_2_ring_test_ib,
- .is_lockup = amdgpu_ring_test_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- r = uvd_v5_0_hw_fini(adev);
+ r = amdgpu_uvd_suspend(adev);
if (r)
return r;
- r = amdgpu_uvd_suspend(adev);
+ r = uvd_v5_0_hw_fini(adev);
if (r)
return r;
.emit_semaphore = uvd_v5_0_ring_emit_semaphore,
.test_ring = uvd_v5_0_ring_test_ring,
.test_ib = uvd_v5_0_ring_test_ib,
- .is_lockup = amdgpu_ring_test_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ /* Skip this for APU for now */
+ if (!(adev->flags & AMD_IS_APU)) {
+ r = amdgpu_uvd_suspend(adev);
+ if (r)
+ return r;
+ }
r = uvd_v6_0_hw_fini(adev);
if (r)
return r;
- r = amdgpu_uvd_suspend(adev);
- if (r)
- return r;
-
return r;
}
int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- r = amdgpu_uvd_resume(adev);
- if (r)
- return r;
-
+ /* Skip this for APU for now */
+ if (!(adev->flags & AMD_IS_APU)) {
+ r = amdgpu_uvd_resume(adev);
+ if (r)
+ return r;
+ }
r = uvd_v6_0_hw_init(adev);
if (r)
return r;
.emit_semaphore = uvd_v6_0_ring_emit_semaphore,
.test_ring = uvd_v6_0_ring_test_ring,
.test_ib = uvd_v6_0_ring_test_ib,
- .is_lockup = amdgpu_ring_test_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
.emit_semaphore = amdgpu_vce_ring_emit_semaphore,
.test_ring = amdgpu_vce_ring_test_ring,
.test_ib = amdgpu_vce_ring_test_ib,
- .is_lockup = amdgpu_ring_test_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
u32 tmp;
unsigned ret;
- /* Fiji is single pipe */
- if (adev->asic_type == CHIP_FIJI) {
+ /* Fiji, Stoney are single pipe */
+ if ((adev->asic_type == CHIP_FIJI) ||
+ (adev->asic_type == CHIP_STONEY)){
ret = AMDGPU_VCE_HARVEST_VCE1;
return ret;
}
.emit_semaphore = amdgpu_vce_ring_emit_semaphore,
.test_ring = amdgpu_vce_ring_test_ring,
.test_ib = amdgpu_vce_ring_test_ib,
- .is_lockup = amdgpu_ring_test_lockup,
.insert_nop = amdgpu_ring_insert_nop,
};
mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
};
+static const u32 stoney_mgcg_cgcg_init[] =
+{
+ mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00000100,
+ mmHDP_XDP_CGTT_BLK_CTRL, 0xffffffff, 0x00000104,
+ mmHDP_HOST_PATH_CNTL, 0xffffffff, 0x0f000027,
+};
+
static void vi_init_golden_registers(struct amdgpu_device *adev)
{
/* Some of the registers might be dependent on GRBM_GFX_INDEX */
cz_mgcg_cgcg_init,
(const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
break;
+ case CHIP_STONEY:
+ amdgpu_program_register_sequence(adev,
+ stoney_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
+ break;
default:
break;
}
case CHIP_FIJI:
case CHIP_TONGA:
case CHIP_CARRIZO:
+ case CHIP_STONEY:
asic_register_table = cz_allowed_read_registers;
size = ARRAY_SIZE(cz_allowed_read_registers);
break;
RREG32(mmSRBM_STATUS2));
dev_info(adev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
RREG32(mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
- dev_info(adev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
- RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
+ if (adev->sdma.num_instances > 1) {
+ dev_info(adev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
+ RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
+ }
dev_info(adev->dev, " CP_STAT = 0x%08x\n", RREG32(mmCP_STAT));
dev_info(adev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
RREG32(mmCP_STALLED_STAT1));
reset_mask |= AMDGPU_RESET_DMA;
/* SDMA1_STATUS_REG */
- tmp = RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
- if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
- reset_mask |= AMDGPU_RESET_DMA1;
+ if (adev->sdma.num_instances > 1) {
+ tmp = RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
+ if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
+ reset_mask |= AMDGPU_RESET_DMA1;
+ }
#if 0
/* VCE_STATUS */
if (adev->asic_type != CHIP_TOPAZ) {
u32 mask;
int ret;
+ if (pci_is_root_bus(adev->pdev->bus))
+ return;
+
if (amdgpu_pcie_gen2 == 0)
return;
adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks);
break;
case CHIP_CARRIZO:
+ case CHIP_STONEY:
adev->ip_blocks = cz_ip_blocks;
adev->num_ip_blocks = ARRAY_SIZE(cz_ip_blocks);
break;
return 0;
}
+#define ATI_REV_ID_FUSE_MACRO__ADDRESS 0xC0014044
+#define ATI_REV_ID_FUSE_MACRO__SHIFT 9
+#define ATI_REV_ID_FUSE_MACRO__MASK 0x00001E00
+
static uint32_t vi_get_rev_id(struct amdgpu_device *adev)
{
if (adev->asic_type == CHIP_TOPAZ)
return (RREG32(mmPCIE_EFUSE4) & PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID_MASK)
>> PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID__SHIFT;
+ else if (adev->flags & AMD_IS_APU)
+ return (RREG32_SMC(ATI_REV_ID_FUSE_MACRO__ADDRESS) & ATI_REV_ID_FUSE_MACRO__MASK)
+ >> ATI_REV_ID_FUSE_MACRO__SHIFT;
else
return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
adev->cg_flags = 0;
adev->pg_flags = 0;
adev->external_rev_id = 0x1;
- if (amdgpu_smc_load_fw && smc_enabled)
- adev->firmware.smu_load = true;
break;
case CHIP_FIJI:
+ adev->has_uvd = true;
+ adev->cg_flags = 0;
+ adev->pg_flags = 0;
+ adev->external_rev_id = adev->rev_id + 0x3c;
+ break;
case CHIP_TONGA:
adev->has_uvd = true;
adev->cg_flags = 0;
adev->pg_flags = 0;
adev->external_rev_id = adev->rev_id + 0x14;
- if (amdgpu_smc_load_fw && smc_enabled)
- adev->firmware.smu_load = true;
break;
case CHIP_CARRIZO:
+ case CHIP_STONEY:
adev->has_uvd = true;
adev->cg_flags = 0;
- adev->pg_flags = AMDGPU_PG_SUPPORT_UVD | AMDGPU_PG_SUPPORT_VCE;
+ /* Disable UVD pg */
+ adev->pg_flags = /* AMDGPU_PG_SUPPORT_UVD | */AMDGPU_PG_SUPPORT_VCE;
adev->external_rev_id = adev->rev_id + 0x1;
- if (amdgpu_smc_load_fw && smc_enabled)
- adev->firmware.smu_load = true;
break;
default:
/* FIXME: not supported yet */
return -EINVAL;
}
+ if (amdgpu_smc_load_fw && smc_enabled)
+ adev->firmware.smu_load = true;
+
return 0;
}
CHIP_TONGA,
CHIP_FIJI,
CHIP_CARRIZO,
+ CHIP_STONEY,
CHIP_LAST,
};
--- /dev/null
+/*
+ * GFX_8_1 Register documentation
+ *
+ * Copyright (C) 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GFX_8_1_D_H
+#define GFX_8_1_D_H
+
+#define mmCB_BLEND_RED 0xa105
+#define mmCB_BLEND_GREEN 0xa106
+#define mmCB_BLEND_BLUE 0xa107
+#define mmCB_BLEND_ALPHA 0xa108
+#define mmCB_DCC_CONTROL 0xa109
+#define mmCB_COLOR_CONTROL 0xa202
+#define mmCB_BLEND0_CONTROL 0xa1e0
+#define mmCB_BLEND1_CONTROL 0xa1e1
+#define mmCB_BLEND2_CONTROL 0xa1e2
+#define mmCB_BLEND3_CONTROL 0xa1e3
+#define mmCB_BLEND4_CONTROL 0xa1e4
+#define mmCB_BLEND5_CONTROL 0xa1e5
+#define mmCB_BLEND6_CONTROL 0xa1e6
+#define mmCB_BLEND7_CONTROL 0xa1e7
+#define mmCB_COLOR0_BASE 0xa318
+#define mmCB_COLOR1_BASE 0xa327
+#define mmCB_COLOR2_BASE 0xa336
+#define mmCB_COLOR3_BASE 0xa345
+#define mmCB_COLOR4_BASE 0xa354
+#define mmCB_COLOR5_BASE 0xa363
+#define mmCB_COLOR6_BASE 0xa372
+#define mmCB_COLOR7_BASE 0xa381
+#define mmCB_COLOR0_PITCH 0xa319
+#define mmCB_COLOR1_PITCH 0xa328
+#define mmCB_COLOR2_PITCH 0xa337
+#define mmCB_COLOR3_PITCH 0xa346
+#define mmCB_COLOR4_PITCH 0xa355
+#define mmCB_COLOR5_PITCH 0xa364
+#define mmCB_COLOR6_PITCH 0xa373
+#define mmCB_COLOR7_PITCH 0xa382
+#define mmCB_COLOR0_SLICE 0xa31a
+#define mmCB_COLOR1_SLICE 0xa329
+#define mmCB_COLOR2_SLICE 0xa338
+#define mmCB_COLOR3_SLICE 0xa347
+#define mmCB_COLOR4_SLICE 0xa356
+#define mmCB_COLOR5_SLICE 0xa365
+#define mmCB_COLOR6_SLICE 0xa374
+#define mmCB_COLOR7_SLICE 0xa383
+#define mmCB_COLOR0_VIEW 0xa31b
+#define mmCB_COLOR1_VIEW 0xa32a
+#define mmCB_COLOR2_VIEW 0xa339
+#define mmCB_COLOR3_VIEW 0xa348
+#define mmCB_COLOR4_VIEW 0xa357
+#define mmCB_COLOR5_VIEW 0xa366
+#define mmCB_COLOR6_VIEW 0xa375
+#define mmCB_COLOR7_VIEW 0xa384
+#define mmCB_COLOR0_INFO 0xa31c
+#define mmCB_COLOR1_INFO 0xa32b
+#define mmCB_COLOR2_INFO 0xa33a
+#define mmCB_COLOR3_INFO 0xa349
+#define mmCB_COLOR4_INFO 0xa358
+#define mmCB_COLOR5_INFO 0xa367
+#define mmCB_COLOR6_INFO 0xa376
+#define mmCB_COLOR7_INFO 0xa385
+#define mmCB_COLOR0_ATTRIB 0xa31d
+#define mmCB_COLOR1_ATTRIB 0xa32c
+#define mmCB_COLOR2_ATTRIB 0xa33b
+#define mmCB_COLOR3_ATTRIB 0xa34a
+#define mmCB_COLOR4_ATTRIB 0xa359
+#define mmCB_COLOR5_ATTRIB 0xa368
+#define mmCB_COLOR6_ATTRIB 0xa377
+#define mmCB_COLOR7_ATTRIB 0xa386
+#define mmCB_COLOR0_DCC_CONTROL 0xa31e
+#define mmCB_COLOR1_DCC_CONTROL 0xa32d
+#define mmCB_COLOR2_DCC_CONTROL 0xa33c
+#define mmCB_COLOR3_DCC_CONTROL 0xa34b
+#define mmCB_COLOR4_DCC_CONTROL 0xa35a
+#define mmCB_COLOR5_DCC_CONTROL 0xa369
+#define mmCB_COLOR6_DCC_CONTROL 0xa378
+#define mmCB_COLOR7_DCC_CONTROL 0xa387
+#define mmCB_COLOR0_CMASK 0xa31f
+#define mmCB_COLOR1_CMASK 0xa32e
+#define mmCB_COLOR2_CMASK 0xa33d
+#define mmCB_COLOR3_CMASK 0xa34c
+#define mmCB_COLOR4_CMASK 0xa35b
+#define mmCB_COLOR5_CMASK 0xa36a
+#define mmCB_COLOR6_CMASK 0xa379
+#define mmCB_COLOR7_CMASK 0xa388
+#define mmCB_COLOR0_CMASK_SLICE 0xa320
+#define mmCB_COLOR1_CMASK_SLICE 0xa32f
+#define mmCB_COLOR2_CMASK_SLICE 0xa33e
+#define mmCB_COLOR3_CMASK_SLICE 0xa34d
+#define mmCB_COLOR4_CMASK_SLICE 0xa35c
+#define mmCB_COLOR5_CMASK_SLICE 0xa36b
+#define mmCB_COLOR6_CMASK_SLICE 0xa37a
+#define mmCB_COLOR7_CMASK_SLICE 0xa389
+#define mmCB_COLOR0_FMASK 0xa321
+#define mmCB_COLOR1_FMASK 0xa330
+#define mmCB_COLOR2_FMASK 0xa33f
+#define mmCB_COLOR3_FMASK 0xa34e
+#define mmCB_COLOR4_FMASK 0xa35d
+#define mmCB_COLOR5_FMASK 0xa36c
+#define mmCB_COLOR6_FMASK 0xa37b
+#define mmCB_COLOR7_FMASK 0xa38a
+#define mmCB_COLOR0_FMASK_SLICE 0xa322
+#define mmCB_COLOR1_FMASK_SLICE 0xa331
+#define mmCB_COLOR2_FMASK_SLICE 0xa340
+#define mmCB_COLOR3_FMASK_SLICE 0xa34f
+#define mmCB_COLOR4_FMASK_SLICE 0xa35e
+#define mmCB_COLOR5_FMASK_SLICE 0xa36d
+#define mmCB_COLOR6_FMASK_SLICE 0xa37c
+#define mmCB_COLOR7_FMASK_SLICE 0xa38b
+#define mmCB_COLOR0_CLEAR_WORD0 0xa323
+#define mmCB_COLOR1_CLEAR_WORD0 0xa332
+#define mmCB_COLOR2_CLEAR_WORD0 0xa341
+#define mmCB_COLOR3_CLEAR_WORD0 0xa350
+#define mmCB_COLOR4_CLEAR_WORD0 0xa35f
+#define mmCB_COLOR5_CLEAR_WORD0 0xa36e
+#define mmCB_COLOR6_CLEAR_WORD0 0xa37d
+#define mmCB_COLOR7_CLEAR_WORD0 0xa38c
+#define mmCB_COLOR0_CLEAR_WORD1 0xa324
+#define mmCB_COLOR1_CLEAR_WORD1 0xa333
+#define mmCB_COLOR2_CLEAR_WORD1 0xa342
+#define mmCB_COLOR3_CLEAR_WORD1 0xa351
+#define mmCB_COLOR4_CLEAR_WORD1 0xa360
+#define mmCB_COLOR5_CLEAR_WORD1 0xa36f
+#define mmCB_COLOR6_CLEAR_WORD1 0xa37e
+#define mmCB_COLOR7_CLEAR_WORD1 0xa38d
+#define mmCB_COLOR0_DCC_BASE 0xa325
+#define mmCB_COLOR1_DCC_BASE 0xa334
+#define mmCB_COLOR2_DCC_BASE 0xa343
+#define mmCB_COLOR3_DCC_BASE 0xa352
+#define mmCB_COLOR4_DCC_BASE 0xa361
+#define mmCB_COLOR5_DCC_BASE 0xa370
+#define mmCB_COLOR6_DCC_BASE 0xa37f
+#define mmCB_COLOR7_DCC_BASE 0xa38e
+#define mmCB_TARGET_MASK 0xa08e
+#define mmCB_SHADER_MASK 0xa08f
+#define mmCB_HW_CONTROL 0x2684
+#define mmCB_HW_CONTROL_1 0x2685
+#define mmCB_HW_CONTROL_2 0x2686
+#define mmCB_HW_CONTROL_3 0x2683
+#define mmCB_DCC_CONFIG 0x2687
+#define mmCB_PERFCOUNTER_FILTER 0xdc00
+#define mmCB_PERFCOUNTER0_SELECT 0xdc01
+#define mmCB_PERFCOUNTER0_SELECT1 0xdc02
+#define mmCB_PERFCOUNTER1_SELECT 0xdc03
+#define mmCB_PERFCOUNTER2_SELECT 0xdc04
+#define mmCB_PERFCOUNTER3_SELECT 0xdc05
+#define mmCB_PERFCOUNTER0_LO 0xd406
+#define mmCB_PERFCOUNTER1_LO 0xd408
+#define mmCB_PERFCOUNTER2_LO 0xd40a
+#define mmCB_PERFCOUNTER3_LO 0xd40c
+#define mmCB_PERFCOUNTER0_HI 0xd407
+#define mmCB_PERFCOUNTER1_HI 0xd409
+#define mmCB_PERFCOUNTER2_HI 0xd40b
+#define mmCB_PERFCOUNTER3_HI 0xd40d
+#define mmCB_CGTT_SCLK_CTRL 0xf0a8
+#define mmCB_DEBUG_BUS_1 0x2699
+#define mmCB_DEBUG_BUS_2 0x269a
+#define mmCB_DEBUG_BUS_3 0x269b
+#define mmCB_DEBUG_BUS_4 0x269c
+#define mmCB_DEBUG_BUS_5 0x269d
+#define mmCB_DEBUG_BUS_6 0x269e
+#define mmCB_DEBUG_BUS_7 0x269f
+#define mmCB_DEBUG_BUS_8 0x26a0
+#define mmCB_DEBUG_BUS_9 0x26a1
+#define mmCB_DEBUG_BUS_10 0x26a2
+#define mmCB_DEBUG_BUS_11 0x26a3
+#define mmCB_DEBUG_BUS_12 0x26a4
+#define mmCB_DEBUG_BUS_13 0x26a5
+#define mmCB_DEBUG_BUS_14 0x26a6
+#define mmCB_DEBUG_BUS_15 0x26a7
+#define mmCB_DEBUG_BUS_16 0x26a8
+#define mmCB_DEBUG_BUS_17 0x26a9
+#define mmCB_DEBUG_BUS_18 0x26aa
+#define mmCB_DEBUG_BUS_19 0x26ab
+#define mmCB_DEBUG_BUS_20 0x26ac
+#define mmCB_DEBUG_BUS_21 0x26ad
+#define mmCB_DEBUG_BUS_22 0x26ae
+#define mmCP_DFY_CNTL 0x3020
+#define mmCP_DFY_STAT 0x3021
+#define mmCP_DFY_ADDR_HI 0x3022
+#define mmCP_DFY_ADDR_LO 0x3023
+#define mmCP_DFY_DATA_0 0x3024
+#define mmCP_DFY_DATA_1 0x3025
+#define mmCP_DFY_DATA_2 0x3026
+#define mmCP_DFY_DATA_3 0x3027
+#define mmCP_DFY_DATA_4 0x3028
+#define mmCP_DFY_DATA_5 0x3029
+#define mmCP_DFY_DATA_6 0x302a
+#define mmCP_DFY_DATA_7 0x302b
+#define mmCP_DFY_DATA_8 0x302c
+#define mmCP_DFY_DATA_9 0x302d
+#define mmCP_DFY_DATA_10 0x302e
+#define mmCP_DFY_DATA_11 0x302f
+#define mmCP_DFY_DATA_12 0x3030
+#define mmCP_DFY_DATA_13 0x3031
+#define mmCP_DFY_DATA_14 0x3032
+#define mmCP_DFY_DATA_15 0x3033
+#define mmCP_DFY_CMD 0x3034
+#define mmCP_CPC_MGCG_SYNC_CNTL 0x3036
+#define mmCP_ATCL1_CNTL 0x303c
+#define mmCP_RB0_BASE 0x3040
+#define mmCP_RB0_BASE_HI 0x30b1
+#define mmCP_RB_BASE 0x3040
+#define mmCP_RB1_BASE 0x3060
+#define mmCP_RB1_BASE_HI 0x30b2
+#define mmCP_RB2_BASE 0x3065
+#define mmCP_RB0_CNTL 0x3041
+#define mmCP_RB_CNTL 0x3041
+#define mmCP_RB1_CNTL 0x3061
+#define mmCP_RB2_CNTL 0x3066
+#define mmCP_RB_RPTR_WR 0x3042
+#define mmCP_RB0_RPTR_ADDR 0x3043
+#define mmCP_RB_RPTR_ADDR 0x3043
+#define mmCP_RB1_RPTR_ADDR 0x3062
+#define mmCP_RB2_RPTR_ADDR 0x3067
+#define mmCP_RB0_RPTR_ADDR_HI 0x3044
+#define mmCP_RB_RPTR_ADDR_HI 0x3044
+#define mmCP_RB1_RPTR_ADDR_HI 0x3063
+#define mmCP_RB2_RPTR_ADDR_HI 0x3068
+#define mmCP_RB0_WPTR 0x3045
+#define mmCP_RB_WPTR 0x3045
+#define mmCP_RB1_WPTR 0x3064
+#define mmCP_RB2_WPTR 0x3069
+#define mmCP_RB_WPTR_POLL_ADDR_LO 0x3046
+#define mmCP_RB_WPTR_POLL_ADDR_HI 0x3047
+#define mmGC_PRIV_MODE 0x3048
+#define mmCP_INT_CNTL 0x3049
+#define mmCP_INT_CNTL_RING0 0x306a
+#define mmCP_INT_CNTL_RING1 0x306b
+#define mmCP_INT_CNTL_RING2 0x306c
+#define mmCP_INT_STATUS 0x304a
+#define mmCP_INT_STATUS_RING0 0x306d
+#define mmCP_INT_STATUS_RING1 0x306e
+#define mmCP_INT_STATUS_RING2 0x306f
+#define mmCP_DEVICE_ID 0x304b
+#define mmCP_RING_PRIORITY_CNTS 0x304c
+#define mmCP_ME0_PIPE_PRIORITY_CNTS 0x304c
+#define mmCP_RING0_PRIORITY 0x304d
+#define mmCP_ME0_PIPE0_PRIORITY 0x304d
+#define mmCP_RING1_PRIORITY 0x304e
+#define mmCP_ME0_PIPE1_PRIORITY 0x304e
+#define mmCP_RING2_PRIORITY 0x304f
+#define mmCP_ME0_PIPE2_PRIORITY 0x304f
+#define mmCP_ENDIAN_SWAP 0x3050
+#define mmCP_RB_VMID 0x3051
+#define mmCP_ME0_PIPE0_VMID 0x3052
+#define mmCP_ME0_PIPE1_VMID 0x3053
+#define mmCP_RB_DOORBELL_CONTROL 0x3059
+#define mmCP_RB_DOORBELL_RANGE_LOWER 0x305a
+#define mmCP_RB_DOORBELL_RANGE_UPPER 0x305b
+#define mmCP_MEC_DOORBELL_RANGE_LOWER 0x305c
+#define mmCP_MEC_DOORBELL_RANGE_UPPER 0x305d
+#define mmCP_PFP_UCODE_ADDR 0xf814
+#define mmCP_PFP_UCODE_DATA 0xf815
+#define mmCP_ME_RAM_RADDR 0xf816
+#define mmCP_ME_RAM_WADDR 0xf816
+#define mmCP_ME_RAM_DATA 0xf817
+#define mmCGTT_CPC_CLK_CTRL 0xf0b2
+#define mmCGTT_CPF_CLK_CTRL 0xf0b1
+#define mmCGTT_CP_CLK_CTRL 0xf0b0
+#define mmCP_CE_UCODE_ADDR 0xf818
+#define mmCP_CE_UCODE_DATA 0xf819
+#define mmCP_MEC_ME1_UCODE_ADDR 0xf81a
+#define mmCP_MEC_ME1_UCODE_DATA 0xf81b
+#define mmCP_MEC_ME2_UCODE_ADDR 0xf81c
+#define mmCP_MEC_ME2_UCODE_DATA 0xf81d
+#define mmCP_MEC1_F32_INT_DIS 0x30bd
+#define mmCP_MEC2_F32_INT_DIS 0x30be
+#define mmCP_PWR_CNTL 0x3078
+#define mmCP_MEM_SLP_CNTL 0x3079
+#define mmCP_ECC_FIRSTOCCURRENCE 0x307a
+#define mmCP_ECC_FIRSTOCCURRENCE_RING0 0x307b
+#define mmCP_ECC_FIRSTOCCURRENCE_RING1 0x307c
+#define mmCP_ECC_FIRSTOCCURRENCE_RING2 0x307d
+#define mmCP_CPF_DEBUG 0x3080
+#define mmCP_PQ_WPTR_POLL_CNTL 0x3083
+#define mmCP_PQ_WPTR_POLL_CNTL1 0x3084
+#define mmCPC_INT_CNTL 0x30b4
+#define mmCP_ME1_PIPE0_INT_CNTL 0x3085
+#define mmCP_ME1_PIPE1_INT_CNTL 0x3086
+#define mmCP_ME1_PIPE2_INT_CNTL 0x3087
+#define mmCP_ME1_PIPE3_INT_CNTL 0x3088
+#define mmCP_ME2_PIPE0_INT_CNTL 0x3089
+#define mmCP_ME2_PIPE1_INT_CNTL 0x308a
+#define mmCP_ME2_PIPE2_INT_CNTL 0x308b
+#define mmCP_ME2_PIPE3_INT_CNTL 0x308c
+#define mmCPC_INT_STATUS 0x30b5
+#define mmCP_ME1_PIPE0_INT_STATUS 0x308d
+#define mmCP_ME1_PIPE1_INT_STATUS 0x308e
+#define mmCP_ME1_PIPE2_INT_STATUS 0x308f
+#define mmCP_ME1_PIPE3_INT_STATUS 0x3090
+#define mmCP_ME2_PIPE0_INT_STATUS 0x3091
+#define mmCP_ME2_PIPE1_INT_STATUS 0x3092
+#define mmCP_ME2_PIPE2_INT_STATUS 0x3093
+#define mmCP_ME2_PIPE3_INT_STATUS 0x3094
+#define mmCP_ME1_INT_STAT_DEBUG 0x3095
+#define mmCP_ME2_INT_STAT_DEBUG 0x3096
+#define mmCP_ME1_PIPE_PRIORITY_CNTS 0x3099
+#define mmCP_ME1_PIPE0_PRIORITY 0x309a
+#define mmCP_ME1_PIPE1_PRIORITY 0x309b
+#define mmCP_ME1_PIPE2_PRIORITY 0x309c
+#define mmCP_ME1_PIPE3_PRIORITY 0x309d
+#define mmCP_ME2_PIPE_PRIORITY_CNTS 0x309e
+#define mmCP_ME2_PIPE0_PRIORITY 0x309f
+#define mmCP_ME2_PIPE1_PRIORITY 0x30a0
+#define mmCP_ME2_PIPE2_PRIORITY 0x30a1
+#define mmCP_ME2_PIPE3_PRIORITY 0x30a2
+#define mmCP_CE_PRGRM_CNTR_START 0x30a3
+#define mmCP_PFP_PRGRM_CNTR_START 0x30a4
+#define mmCP_ME_PRGRM_CNTR_START 0x30a5
+#define mmCP_MEC1_PRGRM_CNTR_START 0x30a6
+#define mmCP_MEC2_PRGRM_CNTR_START 0x30a7
+#define mmCP_CE_INTR_ROUTINE_START 0x30a8
+#define mmCP_PFP_INTR_ROUTINE_START 0x30a9
+#define mmCP_ME_INTR_ROUTINE_START 0x30aa
+#define mmCP_MEC1_INTR_ROUTINE_START 0x30ab
+#define mmCP_MEC2_INTR_ROUTINE_START 0x30ac
+#define mmCP_CONTEXT_CNTL 0x30ad
+#define mmCP_MAX_CONTEXT 0x30ae
+#define mmCP_IQ_WAIT_TIME1 0x30af
+#define mmCP_IQ_WAIT_TIME2 0x30b0
+#define mmCP_VMID_RESET 0x30b3
+#define mmCP_VMID_PREEMPT 0x30b6
+#define mmCP_VMID_STATUS 0x30bf
+#define mmCPC_INT_CNTX_ID 0x30b7
+#define mmCP_PQ_STATUS 0x30b8
+#define mmCP_CPC_IC_BASE_LO 0x30b9
+#define mmCP_CPC_IC_BASE_HI 0x30ba
+#define mmCP_CPC_IC_BASE_CNTL 0x30bb
+#define mmCP_CPC_IC_OP_CNTL 0x30bc
+#define mmCP_CPC_STATUS 0x2084
+#define mmCP_CPC_BUSY_STAT 0x2085
+#define mmCP_CPC_STALLED_STAT1 0x2086
+#define mmCP_CPF_STATUS 0x2087
+#define mmCP_CPF_BUSY_STAT 0x2088
+#define mmCP_CPF_STALLED_STAT1 0x2089
+#define mmCP_CPC_GRBM_FREE_COUNT 0x208b
+#define mmCP_MEC_CNTL 0x208d
+#define mmCP_MEC_ME1_HEADER_DUMP 0x208e
+#define mmCP_MEC_ME2_HEADER_DUMP 0x208f
+#define mmCP_CPC_SCRATCH_INDEX 0x2090
+#define mmCP_CPC_SCRATCH_DATA 0x2091
+#define mmCPG_PERFCOUNTER1_SELECT 0xd800
+#define mmCPG_PERFCOUNTER1_LO 0xd000
+#define mmCPG_PERFCOUNTER1_HI 0xd001
+#define mmCPG_PERFCOUNTER0_SELECT1 0xd801
+#define mmCPG_PERFCOUNTER0_SELECT 0xd802
+#define mmCPG_PERFCOUNTER0_LO 0xd002
+#define mmCPG_PERFCOUNTER0_HI 0xd003
+#define mmCPC_PERFCOUNTER1_SELECT 0xd803
+#define mmCPC_PERFCOUNTER1_LO 0xd004
+#define mmCPC_PERFCOUNTER1_HI 0xd005
+#define mmCPC_PERFCOUNTER0_SELECT1 0xd804
+#define mmCPC_PERFCOUNTER0_SELECT 0xd809
+#define mmCPC_PERFCOUNTER0_LO 0xd006
+#define mmCPC_PERFCOUNTER0_HI 0xd007
+#define mmCPF_PERFCOUNTER1_SELECT 0xd805
+#define mmCPF_PERFCOUNTER1_LO 0xd008
+#define mmCPF_PERFCOUNTER1_HI 0xd009
+#define mmCPF_PERFCOUNTER0_SELECT1 0xd806
+#define mmCPF_PERFCOUNTER0_SELECT 0xd807
+#define mmCPF_PERFCOUNTER0_LO 0xd00a
+#define mmCPF_PERFCOUNTER0_HI 0xd00b
+#define mmCP_CPC_HALT_HYST_COUNT 0x20a7
+#define mmCP_DRAW_OBJECT 0xd810
+#define mmCP_DRAW_OBJECT_COUNTER 0xd811
+#define mmCP_DRAW_WINDOW_MASK_HI 0xd812
+#define mmCP_DRAW_WINDOW_HI 0xd813
+#define mmCP_DRAW_WINDOW_LO 0xd814
+#define mmCP_DRAW_WINDOW_CNTL 0xd815
+#define mmCP_PRT_LOD_STATS_CNTL0 0x20ad
+#define mmCP_PRT_LOD_STATS_CNTL1 0x20ae
+#define mmCP_PRT_LOD_STATS_CNTL2 0x20af
+#define mmCP_CE_COMPARE_COUNT 0x20c0
+#define mmCP_CE_DE_COUNT 0x20c1
+#define mmCP_DE_CE_COUNT 0x20c2
+#define mmCP_DE_LAST_INVAL_COUNT 0x20c3
+#define mmCP_DE_DE_COUNT 0x20c4
+#define mmCP_EOP_DONE_EVENT_CNTL 0xc0d5
+#define mmCP_EOP_DONE_DATA_CNTL 0xc0d6
+#define mmCP_EOP_DONE_CNTX_ID 0xc0d7
+#define mmCP_EOP_DONE_ADDR_LO 0xc000
+#define mmCP_EOP_DONE_ADDR_HI 0xc001
+#define mmCP_EOP_DONE_DATA_LO 0xc002
+#define mmCP_EOP_DONE_DATA_HI 0xc003
+#define mmCP_EOP_LAST_FENCE_LO 0xc004
+#define mmCP_EOP_LAST_FENCE_HI 0xc005
+#define mmCP_STREAM_OUT_ADDR_LO 0xc006
+#define mmCP_STREAM_OUT_ADDR_HI 0xc007
+#define mmCP_NUM_PRIM_WRITTEN_COUNT0_LO 0xc008
+#define mmCP_NUM_PRIM_WRITTEN_COUNT0_HI 0xc009
+#define mmCP_NUM_PRIM_NEEDED_COUNT0_LO 0xc00a
+#define mmCP_NUM_PRIM_NEEDED_COUNT0_HI 0xc00b
+#define mmCP_NUM_PRIM_WRITTEN_COUNT1_LO 0xc00c
+#define mmCP_NUM_PRIM_WRITTEN_COUNT1_HI 0xc00d
+#define mmCP_NUM_PRIM_NEEDED_COUNT1_LO 0xc00e
+#define mmCP_NUM_PRIM_NEEDED_COUNT1_HI 0xc00f
+#define mmCP_NUM_PRIM_WRITTEN_COUNT2_LO 0xc010
+#define mmCP_NUM_PRIM_WRITTEN_COUNT2_HI 0xc011
+#define mmCP_NUM_PRIM_NEEDED_COUNT2_LO 0xc012
+#define mmCP_NUM_PRIM_NEEDED_COUNT2_HI 0xc013
+#define mmCP_NUM_PRIM_WRITTEN_COUNT3_LO 0xc014
+#define mmCP_NUM_PRIM_WRITTEN_COUNT3_HI 0xc015
+#define mmCP_NUM_PRIM_NEEDED_COUNT3_LO 0xc016
+#define mmCP_NUM_PRIM_NEEDED_COUNT3_HI 0xc017
+#define mmCP_PIPE_STATS_ADDR_LO 0xc018
+#define mmCP_PIPE_STATS_ADDR_HI 0xc019
+#define mmCP_VGT_IAVERT_COUNT_LO 0xc01a
+#define mmCP_VGT_IAVERT_COUNT_HI 0xc01b
+#define mmCP_VGT_IAPRIM_COUNT_LO 0xc01c
+#define mmCP_VGT_IAPRIM_COUNT_HI 0xc01d
+#define mmCP_VGT_GSPRIM_COUNT_LO 0xc01e
+#define mmCP_VGT_GSPRIM_COUNT_HI 0xc01f
+#define mmCP_VGT_VSINVOC_COUNT_LO 0xc020
+#define mmCP_VGT_VSINVOC_COUNT_HI 0xc021
+#define mmCP_VGT_GSINVOC_COUNT_LO 0xc022
+#define mmCP_VGT_GSINVOC_COUNT_HI 0xc023
+#define mmCP_VGT_HSINVOC_COUNT_LO 0xc024
+#define mmCP_VGT_HSINVOC_COUNT_HI 0xc025
+#define mmCP_VGT_DSINVOC_COUNT_LO 0xc026
+#define mmCP_VGT_DSINVOC_COUNT_HI 0xc027
+#define mmCP_PA_CINVOC_COUNT_LO 0xc028
+#define mmCP_PA_CINVOC_COUNT_HI 0xc029
+#define mmCP_PA_CPRIM_COUNT_LO 0xc02a
+#define mmCP_PA_CPRIM_COUNT_HI 0xc02b
+#define mmCP_SC_PSINVOC_COUNT0_LO 0xc02c
+#define mmCP_SC_PSINVOC_COUNT0_HI 0xc02d
+#define mmCP_SC_PSINVOC_COUNT1_LO 0xc02e
+#define mmCP_SC_PSINVOC_COUNT1_HI 0xc02f
+#define mmCP_VGT_CSINVOC_COUNT_LO 0xc030
+#define mmCP_VGT_CSINVOC_COUNT_HI 0xc031
+#define mmCP_PIPE_STATS_CONTROL 0xc03d
+#define mmCP_STREAM_OUT_CONTROL 0xc03e
+#define mmCP_STRMOUT_CNTL 0xc03f
+#define mmSCRATCH_REG0 0xc040
+#define mmSCRATCH_REG1 0xc041
+#define mmSCRATCH_REG2 0xc042
+#define mmSCRATCH_REG3 0xc043
+#define mmSCRATCH_REG4 0xc044
+#define mmSCRATCH_REG5 0xc045
+#define mmSCRATCH_REG6 0xc046
+#define mmSCRATCH_REG7 0xc047
+#define mmSCRATCH_UMSK 0xc050
+#define mmSCRATCH_ADDR 0xc051
+#define mmCP_PFP_ATOMIC_PREOP_LO 0xc052
+#define mmCP_PFP_ATOMIC_PREOP_HI 0xc053
+#define mmCP_PFP_GDS_ATOMIC0_PREOP_LO 0xc054
+#define mmCP_PFP_GDS_ATOMIC0_PREOP_HI 0xc055
+#define mmCP_PFP_GDS_ATOMIC1_PREOP_LO 0xc056
+#define mmCP_PFP_GDS_ATOMIC1_PREOP_HI 0xc057
+#define mmCP_APPEND_ADDR_LO 0xc058
+#define mmCP_APPEND_ADDR_HI 0xc059
+#define mmCP_APPEND_DATA 0xc05a
+#define mmCP_APPEND_LAST_CS_FENCE 0xc05b
+#define mmCP_APPEND_LAST_PS_FENCE 0xc05c
+#define mmCP_ATOMIC_PREOP_LO 0xc05d
+#define mmCP_ME_ATOMIC_PREOP_LO 0xc05d
+#define mmCP_ATOMIC_PREOP_HI 0xc05e
+#define mmCP_ME_ATOMIC_PREOP_HI 0xc05e
+#define mmCP_GDS_ATOMIC0_PREOP_LO 0xc05f
+#define mmCP_ME_GDS_ATOMIC0_PREOP_LO 0xc05f
+#define mmCP_GDS_ATOMIC0_PREOP_HI 0xc060
+#define mmCP_ME_GDS_ATOMIC0_PREOP_HI 0xc060
+#define mmCP_GDS_ATOMIC1_PREOP_LO 0xc061
+#define mmCP_ME_GDS_ATOMIC1_PREOP_LO 0xc061
+#define mmCP_GDS_ATOMIC1_PREOP_HI 0xc062
+#define mmCP_ME_GDS_ATOMIC1_PREOP_HI 0xc062
+#define mmCP_ME_MC_WADDR_LO 0xc069
+#define mmCP_ME_MC_WADDR_HI 0xc06a
+#define mmCP_ME_MC_WDATA_LO 0xc06b
+#define mmCP_ME_MC_WDATA_HI 0xc06c
+#define mmCP_ME_MC_RADDR_LO 0xc06d
+#define mmCP_ME_MC_RADDR_HI 0xc06e
+#define mmCP_SEM_WAIT_TIMER 0xc06f
+#define mmCP_SIG_SEM_ADDR_LO 0xc070
+#define mmCP_SIG_SEM_ADDR_HI 0xc071
+#define mmCP_WAIT_SEM_ADDR_LO 0xc075
+#define mmCP_WAIT_SEM_ADDR_HI 0xc076
+#define mmCP_WAIT_REG_MEM_TIMEOUT 0xc074
+#define mmCP_COHER_START_DELAY 0xc07b
+#define mmCP_COHER_CNTL 0xc07c
+#define mmCP_COHER_SIZE 0xc07d
+#define mmCP_COHER_SIZE_HI 0xc08c
+#define mmCP_COHER_BASE 0xc07e
+#define mmCP_COHER_BASE_HI 0xc079
+#define mmCP_COHER_STATUS 0xc07f
+#define mmCOHER_DEST_BASE_0 0xa092
+#define mmCOHER_DEST_BASE_1 0xa093
+#define mmCOHER_DEST_BASE_2 0xa07e
+#define mmCOHER_DEST_BASE_3 0xa07f
+#define mmCOHER_DEST_BASE_HI_0 0xa07a
+#define mmCOHER_DEST_BASE_HI_1 0xa07b
+#define mmCOHER_DEST_BASE_HI_2 0xa07c
+#define mmCOHER_DEST_BASE_HI_3 0xa07d
+#define mmCP_DMA_ME_SRC_ADDR 0xc080
+#define mmCP_DMA_ME_SRC_ADDR_HI 0xc081
+#define mmCP_DMA_ME_DST_ADDR 0xc082
+#define mmCP_DMA_ME_DST_ADDR_HI 0xc083
+#define mmCP_DMA_ME_CONTROL 0xc078
+#define mmCP_DMA_ME_COMMAND 0xc084
+#define mmCP_DMA_PFP_SRC_ADDR 0xc085
+#define mmCP_DMA_PFP_SRC_ADDR_HI 0xc086
+#define mmCP_DMA_PFP_DST_ADDR 0xc087
+#define mmCP_DMA_PFP_DST_ADDR_HI 0xc088
+#define mmCP_DMA_PFP_CONTROL 0xc077
+#define mmCP_DMA_PFP_COMMAND 0xc089
+#define mmCP_DMA_CNTL 0xc08a
+#define mmCP_DMA_READ_TAGS 0xc08b
+#define mmCP_PFP_IB_CONTROL 0xc08d
+#define mmCP_PFP_LOAD_CONTROL 0xc08e
+#define mmCP_SCRATCH_INDEX 0xc08f
+#define mmCP_SCRATCH_DATA 0xc090
+#define mmCP_RB_OFFSET 0xc091
+#define mmCP_IB1_OFFSET 0xc092
+#define mmCP_IB2_OFFSET 0xc093
+#define mmCP_IB1_PREAMBLE_BEGIN 0xc094
+#define mmCP_IB1_PREAMBLE_END 0xc095
+#define mmCP_IB2_PREAMBLE_BEGIN 0xc096
+#define mmCP_IB2_PREAMBLE_END 0xc097
+#define mmCP_CE_IB1_OFFSET 0xc098
+#define mmCP_CE_IB2_OFFSET 0xc099
+#define mmCP_CE_COUNTER 0xc09a
+#define mmCP_CE_RB_OFFSET 0xc09b
+#define mmCP_PFP_COMPLETION_STATUS 0xc0ec
+#define mmCP_CE_COMPLETION_STATUS 0xc0ed
+#define mmCP_PRED_NOT_VISIBLE 0xc0ee
+#define mmCP_PFP_METADATA_BASE_ADDR 0xc0f0
+#define mmCP_PFP_METADATA_BASE_ADDR_HI 0xc0f1
+#define mmCP_CE_METADATA_BASE_ADDR 0xc0f2
+#define mmCP_CE_METADATA_BASE_ADDR_HI 0xc0f3
+#define mmCP_DRAW_INDX_INDR_ADDR 0xc0f4
+#define mmCP_DRAW_INDX_INDR_ADDR_HI 0xc0f5
+#define mmCP_DISPATCH_INDR_ADDR 0xc0f6
+#define mmCP_DISPATCH_INDR_ADDR_HI 0xc0f7
+#define mmCP_INDEX_BASE_ADDR 0xc0f8
+#define mmCP_INDEX_BASE_ADDR_HI 0xc0f9
+#define mmCP_INDEX_TYPE 0xc0fa
+#define mmCP_GDS_BKUP_ADDR 0xc0fb
+#define mmCP_GDS_BKUP_ADDR_HI 0xc0fc
+#define mmCP_SAMPLE_STATUS 0xc0fd
+#define mmCP_STALLED_STAT1 0x219d
+#define mmCP_STALLED_STAT2 0x219e
+#define mmCP_STALLED_STAT3 0x219c
+#define mmCP_BUSY_STAT 0x219f
+#define mmCP_STAT 0x21a0
+#define mmCP_ME_HEADER_DUMP 0x21a1
+#define mmCP_PFP_HEADER_DUMP 0x21a2
+#define mmCP_GRBM_FREE_COUNT 0x21a3
+#define mmCP_CE_HEADER_DUMP 0x21a4
+#define mmCP_CSF_STAT 0x21b4
+#define mmCP_CSF_CNTL 0x21b5
+#define mmCP_ME_CNTL 0x21b6
+#define mmCP_CNTX_STAT 0x21b8
+#define mmCP_ME_PREEMPTION 0x21b9
+#define mmCP_RB0_RPTR 0x21c0
+#define mmCP_RB_RPTR 0x21c0
+#define mmCP_RB1_RPTR 0x21bf
+#define mmCP_RB2_RPTR 0x21be
+#define mmCP_RB_WPTR_DELAY 0x21c1
+#define mmCP_RB_WPTR_POLL_CNTL 0x21c2
+#define mmCP_CE_INIT_BASE_LO 0xc0c3
+#define mmCP_CE_INIT_BASE_HI 0xc0c4
+#define mmCP_CE_INIT_BUFSZ 0xc0c5
+#define mmCP_CE_IB1_BASE_LO 0xc0c6
+#define mmCP_CE_IB1_BASE_HI 0xc0c7
+#define mmCP_CE_IB1_BUFSZ 0xc0c8
+#define mmCP_CE_IB2_BASE_LO 0xc0c9
+#define mmCP_CE_IB2_BASE_HI 0xc0ca
+#define mmCP_CE_IB2_BUFSZ 0xc0cb
+#define mmCP_IB1_BASE_LO 0xc0cc
+#define mmCP_IB1_BASE_HI 0xc0cd
+#define mmCP_IB1_BUFSZ 0xc0ce
+#define mmCP_IB2_BASE_LO 0xc0cf
+#define mmCP_IB2_BASE_HI 0xc0d0
+#define mmCP_IB2_BUFSZ 0xc0d1
+#define mmCP_ST_BASE_LO 0xc0d2
+#define mmCP_ST_BASE_HI 0xc0d3
+#define mmCP_ST_BUFSZ 0xc0d4
+#define mmCP_ROQ_THRESHOLDS 0x21bc
+#define mmCP_MEQ_STQ_THRESHOLD 0x21bd
+#define mmCP_ROQ1_THRESHOLDS 0x21d5
+#define mmCP_ROQ2_THRESHOLDS 0x21d6
+#define mmCP_STQ_THRESHOLDS 0x21d7
+#define mmCP_QUEUE_THRESHOLDS 0x21d8
+#define mmCP_MEQ_THRESHOLDS 0x21d9
+#define mmCP_ROQ_AVAIL 0x21da
+#define mmCP_STQ_AVAIL 0x21db
+#define mmCP_ROQ2_AVAIL 0x21dc
+#define mmCP_MEQ_AVAIL 0x21dd
+#define mmCP_CMD_INDEX 0x21de
+#define mmCP_CMD_DATA 0x21df
+#define mmCP_ROQ_RB_STAT 0x21e0
+#define mmCP_ROQ_IB1_STAT 0x21e1
+#define mmCP_ROQ_IB2_STAT 0x21e2
+#define mmCP_STQ_STAT 0x21e3
+#define mmCP_STQ_WR_STAT 0x21e4
+#define mmCP_MEQ_STAT 0x21e5
+#define mmCP_CEQ1_AVAIL 0x21e6
+#define mmCP_CEQ2_AVAIL 0x21e7
+#define mmCP_CE_ROQ_RB_STAT 0x21e8
+#define mmCP_CE_ROQ_IB1_STAT 0x21e9
+#define mmCP_CE_ROQ_IB2_STAT 0x21ea
+#define mmCP_INT_STAT_DEBUG 0x21f7
+#define mmCP_PERFMON_CNTL 0xd808
+#define mmCP_PERFMON_CNTX_CNTL 0xa0d8
+#define mmCP_RINGID 0xa0d9
+#define mmCP_PIPEID 0xa0d9
+#define mmCP_VMID 0xa0da
+#define mmCP_HPD_ROQ_OFFSETS 0x3240
+#define mmCP_HPD_STATUS0 0x3241
+#define mmCP_MQD_BASE_ADDR 0x3245
+#define mmCP_MQD_BASE_ADDR_HI 0x3246
+#define mmCP_HQD_ACTIVE 0x3247
+#define mmCP_HQD_VMID 0x3248
+#define mmCP_HQD_PERSISTENT_STATE 0x3249
+#define mmCP_HQD_PIPE_PRIORITY 0x324a
+#define mmCP_HQD_QUEUE_PRIORITY 0x324b
+#define mmCP_HQD_QUANTUM 0x324c
+#define mmCP_HQD_PQ_BASE 0x324d
+#define mmCP_HQD_PQ_BASE_HI 0x324e
+#define mmCP_HQD_PQ_RPTR 0x324f
+#define mmCP_HQD_PQ_RPTR_REPORT_ADDR 0x3250
+#define mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI 0x3251
+#define mmCP_HQD_PQ_WPTR_POLL_ADDR 0x3252
+#define mmCP_HQD_PQ_WPTR_POLL_ADDR_HI 0x3253
+#define mmCP_HQD_PQ_DOORBELL_CONTROL 0x3254
+#define mmCP_HQD_PQ_WPTR 0x3255
+#define mmCP_HQD_PQ_CONTROL 0x3256
+#define mmCP_HQD_IB_BASE_ADDR 0x3257
+#define mmCP_HQD_IB_BASE_ADDR_HI 0x3258
+#define mmCP_HQD_IB_RPTR 0x3259
+#define mmCP_HQD_IB_CONTROL 0x325a
+#define mmCP_HQD_IQ_TIMER 0x325b
+#define mmCP_HQD_IQ_RPTR 0x325c
+#define mmCP_HQD_DEQUEUE_REQUEST 0x325d
+#define mmCP_HQD_DMA_OFFLOAD 0x325e
+#define mmCP_HQD_OFFLOAD 0x325e
+#define mmCP_HQD_SEMA_CMD 0x325f
+#define mmCP_HQD_MSG_TYPE 0x3260
+#define mmCP_HQD_ATOMIC0_PREOP_LO 0x3261
+#define mmCP_HQD_ATOMIC0_PREOP_HI 0x3262
+#define mmCP_HQD_ATOMIC1_PREOP_LO 0x3263
+#define mmCP_HQD_ATOMIC1_PREOP_HI 0x3264
+#define mmCP_HQD_HQ_SCHEDULER0 0x3265
+#define mmCP_HQD_HQ_STATUS0 0x3265
+#define mmCP_HQD_HQ_SCHEDULER1 0x3266
+#define mmCP_HQD_HQ_CONTROL0 0x3266
+#define mmCP_MQD_CONTROL 0x3267
+#define mmCP_HQD_HQ_STATUS1 0x3268
+#define mmCP_HQD_HQ_CONTROL1 0x3269
+#define mmCP_HQD_EOP_BASE_ADDR 0x326a
+#define mmCP_HQD_EOP_BASE_ADDR_HI 0x326b
+#define mmCP_HQD_EOP_CONTROL 0x326c
+#define mmCP_HQD_EOP_RPTR 0x326d
+#define mmCP_HQD_EOP_WPTR 0x326e
+#define mmCP_HQD_EOP_EVENTS 0x326f
+#define mmCP_HQD_CTX_SAVE_BASE_ADDR_LO 0x3270
+#define mmCP_HQD_CTX_SAVE_BASE_ADDR_HI 0x3271
+#define mmCP_HQD_CTX_SAVE_CONTROL 0x3272
+#define mmCP_HQD_CNTL_STACK_OFFSET 0x3273
+#define mmCP_HQD_CNTL_STACK_SIZE 0x3274
+#define mmCP_HQD_WG_STATE_OFFSET 0x3275
+#define mmCP_HQD_CTX_SAVE_SIZE 0x3276
+#define mmCP_HQD_GDS_RESOURCE_STATE 0x3277
+#define mmCP_HQD_ERROR 0x3278
+#define mmCP_HQD_EOP_WPTR_MEM 0x3279
+#define mmCP_HQD_EOP_DONES 0x327a
+#define mmDB_Z_READ_BASE 0xa012
+#define mmDB_STENCIL_READ_BASE 0xa013
+#define mmDB_Z_WRITE_BASE 0xa014
+#define mmDB_STENCIL_WRITE_BASE 0xa015
+#define mmDB_DEPTH_INFO 0xa00f
+#define mmDB_Z_INFO 0xa010
+#define mmDB_STENCIL_INFO 0xa011
+#define mmDB_DEPTH_SIZE 0xa016
+#define mmDB_DEPTH_SLICE 0xa017
+#define mmDB_DEPTH_VIEW 0xa002
+#define mmDB_RENDER_CONTROL 0xa000
+#define mmDB_COUNT_CONTROL 0xa001
+#define mmDB_RENDER_OVERRIDE 0xa003
+#define mmDB_RENDER_OVERRIDE2 0xa004
+#define mmDB_EQAA 0xa201
+#define mmDB_SHADER_CONTROL 0xa203
+#define mmDB_DEPTH_BOUNDS_MIN 0xa008
+#define mmDB_DEPTH_BOUNDS_MAX 0xa009
+#define mmDB_STENCIL_CLEAR 0xa00a
+#define mmDB_DEPTH_CLEAR 0xa00b
+#define mmDB_HTILE_DATA_BASE 0xa005
+#define mmDB_HTILE_SURFACE 0xa2af
+#define mmDB_PRELOAD_CONTROL 0xa2b2
+#define mmDB_STENCILREFMASK 0xa10c
+#define mmDB_STENCILREFMASK_BF 0xa10d
+#define mmDB_SRESULTS_COMPARE_STATE0 0xa2b0
+#define mmDB_SRESULTS_COMPARE_STATE1 0xa2b1
+#define mmDB_DEPTH_CONTROL 0xa200
+#define mmDB_STENCIL_CONTROL 0xa10b
+#define mmDB_ALPHA_TO_MASK 0xa2dc
+#define mmDB_PERFCOUNTER0_SELECT 0xdc40
+#define mmDB_PERFCOUNTER1_SELECT 0xdc42
+#define mmDB_PERFCOUNTER2_SELECT 0xdc44
+#define mmDB_PERFCOUNTER3_SELECT 0xdc46
+#define mmDB_PERFCOUNTER0_SELECT1 0xdc41
+#define mmDB_PERFCOUNTER1_SELECT1 0xdc43
+#define mmDB_PERFCOUNTER0_LO 0xd440
+#define mmDB_PERFCOUNTER1_LO 0xd442
+#define mmDB_PERFCOUNTER2_LO 0xd444
+#define mmDB_PERFCOUNTER3_LO 0xd446
+#define mmDB_PERFCOUNTER0_HI 0xd441
+#define mmDB_PERFCOUNTER1_HI 0xd443
+#define mmDB_PERFCOUNTER2_HI 0xd445
+#define mmDB_PERFCOUNTER3_HI 0xd447
+#define mmDB_DEBUG 0x260c
+#define mmDB_DEBUG2 0x260d
+#define mmDB_DEBUG3 0x260e
+#define mmDB_DEBUG4 0x260f
+#define mmDB_CREDIT_LIMIT 0x2614
+#define mmDB_WATERMARKS 0x2615
+#define mmDB_SUBTILE_CONTROL 0x2616
+#define mmDB_FREE_CACHELINES 0x2617
+#define mmDB_FIFO_DEPTH1 0x2618
+#define mmDB_FIFO_DEPTH2 0x2619
+#define mmDB_CGTT_CLK_CTRL_0 0xf0a4
+#define mmDB_ZPASS_COUNT_LOW 0xc3fe
+#define mmDB_ZPASS_COUNT_HI 0xc3ff
+#define mmDB_RING_CONTROL 0x261b
+#define mmDB_READ_DEBUG_0 0x2620
+#define mmDB_READ_DEBUG_1 0x2621
+#define mmDB_READ_DEBUG_2 0x2622
+#define mmDB_READ_DEBUG_3 0x2623
+#define mmDB_READ_DEBUG_4 0x2624
+#define mmDB_READ_DEBUG_5 0x2625
+#define mmDB_READ_DEBUG_6 0x2626
+#define mmDB_READ_DEBUG_7 0x2627
+#define mmDB_READ_DEBUG_8 0x2628
+#define mmDB_READ_DEBUG_9 0x2629
+#define mmDB_READ_DEBUG_A 0x262a
+#define mmDB_READ_DEBUG_B 0x262b
+#define mmDB_READ_DEBUG_C 0x262c
+#define mmDB_READ_DEBUG_D 0x262d
+#define mmDB_READ_DEBUG_E 0x262e
+#define mmDB_READ_DEBUG_F 0x262f
+#define mmDB_OCCLUSION_COUNT0_LOW 0xc3c0
+#define mmDB_OCCLUSION_COUNT0_HI 0xc3c1
+#define mmDB_OCCLUSION_COUNT1_LOW 0xc3c2
+#define mmDB_OCCLUSION_COUNT1_HI 0xc3c3
+#define mmDB_OCCLUSION_COUNT2_LOW 0xc3c4
+#define mmDB_OCCLUSION_COUNT2_HI 0xc3c5
+#define mmDB_OCCLUSION_COUNT3_LOW 0xc3c6
+#define mmDB_OCCLUSION_COUNT3_HI 0xc3c7
+#define mmCC_RB_REDUNDANCY 0x263c
+#define mmCC_RB_BACKEND_DISABLE 0x263d
+#define mmGC_USER_RB_REDUNDANCY 0x26de
+#define mmGC_USER_RB_BACKEND_DISABLE 0x26df
+#define mmGB_ADDR_CONFIG 0x263e
+#define mmGB_BACKEND_MAP 0x263f
+#define mmGB_GPU_ID 0x2640
+#define mmCC_RB_DAISY_CHAIN 0x2641
+#define mmGB_TILE_MODE0 0x2644
+#define mmGB_TILE_MODE1 0x2645
+#define mmGB_TILE_MODE2 0x2646
+#define mmGB_TILE_MODE3 0x2647
+#define mmGB_TILE_MODE4 0x2648
+#define mmGB_TILE_MODE5 0x2649
+#define mmGB_TILE_MODE6 0x264a
+#define mmGB_TILE_MODE7 0x264b
+#define mmGB_TILE_MODE8 0x264c
+#define mmGB_TILE_MODE9 0x264d
+#define mmGB_TILE_MODE10 0x264e
+#define mmGB_TILE_MODE11 0x264f
+#define mmGB_TILE_MODE12 0x2650
+#define mmGB_TILE_MODE13 0x2651
+#define mmGB_TILE_MODE14 0x2652
+#define mmGB_TILE_MODE15 0x2653
+#define mmGB_TILE_MODE16 0x2654
+#define mmGB_TILE_MODE17 0x2655
+#define mmGB_TILE_MODE18 0x2656
+#define mmGB_TILE_MODE19 0x2657
+#define mmGB_TILE_MODE20 0x2658
+#define mmGB_TILE_MODE21 0x2659
+#define mmGB_TILE_MODE22 0x265a
+#define mmGB_TILE_MODE23 0x265b
+#define mmGB_TILE_MODE24 0x265c
+#define mmGB_TILE_MODE25 0x265d
+#define mmGB_TILE_MODE26 0x265e
+#define mmGB_TILE_MODE27 0x265f
+#define mmGB_TILE_MODE28 0x2660
+#define mmGB_TILE_MODE29 0x2661
+#define mmGB_TILE_MODE30 0x2662
+#define mmGB_TILE_MODE31 0x2663
+#define mmGB_MACROTILE_MODE0 0x2664
+#define mmGB_MACROTILE_MODE1 0x2665
+#define mmGB_MACROTILE_MODE2 0x2666
+#define mmGB_MACROTILE_MODE3 0x2667
+#define mmGB_MACROTILE_MODE4 0x2668
+#define mmGB_MACROTILE_MODE5 0x2669
+#define mmGB_MACROTILE_MODE6 0x266a
+#define mmGB_MACROTILE_MODE7 0x266b
+#define mmGB_MACROTILE_MODE8 0x266c
+#define mmGB_MACROTILE_MODE9 0x266d
+#define mmGB_MACROTILE_MODE10 0x266e
+#define mmGB_MACROTILE_MODE11 0x266f
+#define mmGB_MACROTILE_MODE12 0x2670
+#define mmGB_MACROTILE_MODE13 0x2671
+#define mmGB_MACROTILE_MODE14 0x2672
+#define mmGB_MACROTILE_MODE15 0x2673
+#define mmGB_EDC_MODE 0x307e
+#define mmCC_GC_EDC_CONFIG 0x3098
+#define mmRAS_SIGNATURE_CONTROL 0x3380
+#define mmRAS_SIGNATURE_MASK 0x3381
+#define mmRAS_SX_SIGNATURE0 0x3382
+#define mmRAS_SX_SIGNATURE1 0x3383
+#define mmRAS_SX_SIGNATURE2 0x3384
+#define mmRAS_SX_SIGNATURE3 0x3385
+#define mmRAS_DB_SIGNATURE0 0x338b
+#define mmRAS_PA_SIGNATURE0 0x338c
+#define mmRAS_VGT_SIGNATURE0 0x338d
+#define mmRAS_SC_SIGNATURE0 0x338f
+#define mmRAS_SC_SIGNATURE1 0x3390
+#define mmRAS_SC_SIGNATURE2 0x3391
+#define mmRAS_SC_SIGNATURE3 0x3392
+#define mmRAS_SC_SIGNATURE4 0x3393
+#define mmRAS_SC_SIGNATURE5 0x3394
+#define mmRAS_SC_SIGNATURE6 0x3395
+#define mmRAS_SC_SIGNATURE7 0x3396
+#define mmRAS_IA_SIGNATURE0 0x3397
+#define mmRAS_IA_SIGNATURE1 0x3398
+#define mmRAS_SPI_SIGNATURE0 0x3399
+#define mmRAS_SPI_SIGNATURE1 0x339a
+#define mmRAS_TA_SIGNATURE0 0x339b
+#define mmRAS_TD_SIGNATURE0 0x339c
+#define mmRAS_CB_SIGNATURE0 0x339d
+#define mmRAS_BCI_SIGNATURE0 0x339e
+#define mmRAS_BCI_SIGNATURE1 0x339f
+#define mmRAS_TA_SIGNATURE1 0x33a0
+#define mmGRBM_HYP_CAM_INDEX 0xf83e
+#define mmGRBM_CAM_INDEX 0xf83e
+#define mmGRBM_HYP_CAM_DATA 0xf83f
+#define mmGRBM_CAM_DATA 0xf83f
+#define mmGRBM_CNTL 0x2000
+#define mmGRBM_SKEW_CNTL 0x2001
+#define mmGRBM_PWR_CNTL 0x2003
+#define mmGRBM_STATUS 0x2004
+#define mmGRBM_STATUS2 0x2002
+#define mmGRBM_STATUS_SE0 0x2005
+#define mmGRBM_STATUS_SE1 0x2006
+#define mmGRBM_STATUS_SE2 0x200e
+#define mmGRBM_STATUS_SE3 0x200f
+#define mmGRBM_SOFT_RESET 0x2008
+#define mmGRBM_DEBUG_CNTL 0x2009
+#define mmGRBM_DEBUG_DATA 0x200a
+#define mmGRBM_CGTT_CLK_CNTL 0x200b
+#define mmGRBM_GFX_INDEX 0xc200
+#define mmGRBM_GFX_CLKEN_CNTL 0x200c
+#define mmGRBM_WAIT_IDLE_CLOCKS 0x200d
+#define mmGRBM_DEBUG 0x2014
+#define mmGRBM_DEBUG_SNAPSHOT 0x2015
+#define mmGRBM_READ_ERROR 0x2016
+#define mmGRBM_READ_ERROR2 0x2017
+#define mmGRBM_INT_CNTL 0x2018
+#define mmGRBM_TRAP_OP 0x2019
+#define mmGRBM_TRAP_ADDR 0x201a
+#define mmGRBM_TRAP_ADDR_MSK 0x201b
+#define mmGRBM_TRAP_WD 0x201c
+#define mmGRBM_TRAP_WD_MSK 0x201d
+#define mmGRBM_DSM_BYPASS 0x201e
+#define mmGRBM_WRITE_ERROR 0x201f
+#define mmGRBM_PERFCOUNTER0_SELECT 0xd840
+#define mmGRBM_PERFCOUNTER1_SELECT 0xd841
+#define mmGRBM_SE0_PERFCOUNTER_SELECT 0xd842
+#define mmGRBM_SE1_PERFCOUNTER_SELECT 0xd843
+#define mmGRBM_SE2_PERFCOUNTER_SELECT 0xd844
+#define mmGRBM_SE3_PERFCOUNTER_SELECT 0xd845
+#define mmGRBM_PERFCOUNTER0_LO 0xd040
+#define mmGRBM_PERFCOUNTER0_HI 0xd041
+#define mmGRBM_PERFCOUNTER1_LO 0xd043
+#define mmGRBM_PERFCOUNTER1_HI 0xd044
+#define mmGRBM_SE0_PERFCOUNTER_LO 0xd045
+#define mmGRBM_SE0_PERFCOUNTER_HI 0xd046
+#define mmGRBM_SE1_PERFCOUNTER_LO 0xd047
+#define mmGRBM_SE1_PERFCOUNTER_HI 0xd048
+#define mmGRBM_SE2_PERFCOUNTER_LO 0xd049
+#define mmGRBM_SE2_PERFCOUNTER_HI 0xd04a
+#define mmGRBM_SE3_PERFCOUNTER_LO 0xd04b
+#define mmGRBM_SE3_PERFCOUNTER_HI 0xd04c
+#define mmGRBM_SCRATCH_REG0 0x2040
+#define mmGRBM_SCRATCH_REG1 0x2041
+#define mmGRBM_SCRATCH_REG2 0x2042
+#define mmGRBM_SCRATCH_REG3 0x2043
+#define mmGRBM_SCRATCH_REG4 0x2044
+#define mmGRBM_SCRATCH_REG5 0x2045
+#define mmGRBM_SCRATCH_REG6 0x2046
+#define mmGRBM_SCRATCH_REG7 0x2047
+#define mmDEBUG_INDEX 0x203c
+#define mmDEBUG_DATA 0x203d
+#define mmGRBM_NOWHERE 0x203f
+#define mmPA_CL_VPORT_XSCALE 0xa10f
+#define mmPA_CL_VPORT_XOFFSET 0xa110
+#define mmPA_CL_VPORT_YSCALE 0xa111
+#define mmPA_CL_VPORT_YOFFSET 0xa112
+#define mmPA_CL_VPORT_ZSCALE 0xa113
+#define mmPA_CL_VPORT_ZOFFSET 0xa114
+#define mmPA_CL_VPORT_XSCALE_1 0xa115
+#define mmPA_CL_VPORT_XSCALE_2 0xa11b
+#define mmPA_CL_VPORT_XSCALE_3 0xa121
+#define mmPA_CL_VPORT_XSCALE_4 0xa127
+#define mmPA_CL_VPORT_XSCALE_5 0xa12d
+#define mmPA_CL_VPORT_XSCALE_6 0xa133
+#define mmPA_CL_VPORT_XSCALE_7 0xa139
+#define mmPA_CL_VPORT_XSCALE_8 0xa13f
+#define mmPA_CL_VPORT_XSCALE_9 0xa145
+#define mmPA_CL_VPORT_XSCALE_10 0xa14b
+#define mmPA_CL_VPORT_XSCALE_11 0xa151
+#define mmPA_CL_VPORT_XSCALE_12 0xa157
+#define mmPA_CL_VPORT_XSCALE_13 0xa15d
+#define mmPA_CL_VPORT_XSCALE_14 0xa163
+#define mmPA_CL_VPORT_XSCALE_15 0xa169
+#define mmPA_CL_VPORT_XOFFSET_1 0xa116
+#define mmPA_CL_VPORT_XOFFSET_2 0xa11c
+#define mmPA_CL_VPORT_XOFFSET_3 0xa122
+#define mmPA_CL_VPORT_XOFFSET_4 0xa128
+#define mmPA_CL_VPORT_XOFFSET_5 0xa12e
+#define mmPA_CL_VPORT_XOFFSET_6 0xa134
+#define mmPA_CL_VPORT_XOFFSET_7 0xa13a
+#define mmPA_CL_VPORT_XOFFSET_8 0xa140
+#define mmPA_CL_VPORT_XOFFSET_9 0xa146
+#define mmPA_CL_VPORT_XOFFSET_10 0xa14c
+#define mmPA_CL_VPORT_XOFFSET_11 0xa152
+#define mmPA_CL_VPORT_XOFFSET_12 0xa158
+#define mmPA_CL_VPORT_XOFFSET_13 0xa15e
+#define mmPA_CL_VPORT_XOFFSET_14 0xa164
+#define mmPA_CL_VPORT_XOFFSET_15 0xa16a
+#define mmPA_CL_VPORT_YSCALE_1 0xa117
+#define mmPA_CL_VPORT_YSCALE_2 0xa11d
+#define mmPA_CL_VPORT_YSCALE_3 0xa123
+#define mmPA_CL_VPORT_YSCALE_4 0xa129
+#define mmPA_CL_VPORT_YSCALE_5 0xa12f
+#define mmPA_CL_VPORT_YSCALE_6 0xa135
+#define mmPA_CL_VPORT_YSCALE_7 0xa13b
+#define mmPA_CL_VPORT_YSCALE_8 0xa141
+#define mmPA_CL_VPORT_YSCALE_9 0xa147
+#define mmPA_CL_VPORT_YSCALE_10 0xa14d
+#define mmPA_CL_VPORT_YSCALE_11 0xa153
+#define mmPA_CL_VPORT_YSCALE_12 0xa159
+#define mmPA_CL_VPORT_YSCALE_13 0xa15f
+#define mmPA_CL_VPORT_YSCALE_14 0xa165
+#define mmPA_CL_VPORT_YSCALE_15 0xa16b
+#define mmPA_CL_VPORT_YOFFSET_1 0xa118
+#define mmPA_CL_VPORT_YOFFSET_2 0xa11e
+#define mmPA_CL_VPORT_YOFFSET_3 0xa124
+#define mmPA_CL_VPORT_YOFFSET_4 0xa12a
+#define mmPA_CL_VPORT_YOFFSET_5 0xa130
+#define mmPA_CL_VPORT_YOFFSET_6 0xa136
+#define mmPA_CL_VPORT_YOFFSET_7 0xa13c
+#define mmPA_CL_VPORT_YOFFSET_8 0xa142
+#define mmPA_CL_VPORT_YOFFSET_9 0xa148
+#define mmPA_CL_VPORT_YOFFSET_10 0xa14e
+#define mmPA_CL_VPORT_YOFFSET_11 0xa154
+#define mmPA_CL_VPORT_YOFFSET_12 0xa15a
+#define mmPA_CL_VPORT_YOFFSET_13 0xa160
+#define mmPA_CL_VPORT_YOFFSET_14 0xa166
+#define mmPA_CL_VPORT_YOFFSET_15 0xa16c
+#define mmPA_CL_VPORT_ZSCALE_1 0xa119
+#define mmPA_CL_VPORT_ZSCALE_2 0xa11f
+#define mmPA_CL_VPORT_ZSCALE_3 0xa125
+#define mmPA_CL_VPORT_ZSCALE_4 0xa12b
+#define mmPA_CL_VPORT_ZSCALE_5 0xa131
+#define mmPA_CL_VPORT_ZSCALE_6 0xa137
+#define mmPA_CL_VPORT_ZSCALE_7 0xa13d
+#define mmPA_CL_VPORT_ZSCALE_8 0xa143
+#define mmPA_CL_VPORT_ZSCALE_9 0xa149
+#define mmPA_CL_VPORT_ZSCALE_10 0xa14f
+#define mmPA_CL_VPORT_ZSCALE_11 0xa155
+#define mmPA_CL_VPORT_ZSCALE_12 0xa15b
+#define mmPA_CL_VPORT_ZSCALE_13 0xa161
+#define mmPA_CL_VPORT_ZSCALE_14 0xa167
+#define mmPA_CL_VPORT_ZSCALE_15 0xa16d
+#define mmPA_CL_VPORT_ZOFFSET_1 0xa11a
+#define mmPA_CL_VPORT_ZOFFSET_2 0xa120
+#define mmPA_CL_VPORT_ZOFFSET_3 0xa126
+#define mmPA_CL_VPORT_ZOFFSET_4 0xa12c
+#define mmPA_CL_VPORT_ZOFFSET_5 0xa132
+#define mmPA_CL_VPORT_ZOFFSET_6 0xa138
+#define mmPA_CL_VPORT_ZOFFSET_7 0xa13e
+#define mmPA_CL_VPORT_ZOFFSET_8 0xa144
+#define mmPA_CL_VPORT_ZOFFSET_9 0xa14a
+#define mmPA_CL_VPORT_ZOFFSET_10 0xa150
+#define mmPA_CL_VPORT_ZOFFSET_11 0xa156
+#define mmPA_CL_VPORT_ZOFFSET_12 0xa15c
+#define mmPA_CL_VPORT_ZOFFSET_13 0xa162
+#define mmPA_CL_VPORT_ZOFFSET_14 0xa168
+#define mmPA_CL_VPORT_ZOFFSET_15 0xa16e
+#define mmPA_CL_VTE_CNTL 0xa206
+#define mmPA_CL_VS_OUT_CNTL 0xa207
+#define mmPA_CL_NANINF_CNTL 0xa208
+#define mmPA_CL_CLIP_CNTL 0xa204
+#define mmPA_CL_GB_VERT_CLIP_ADJ 0xa2fa
+#define mmPA_CL_GB_VERT_DISC_ADJ 0xa2fb
+#define mmPA_CL_GB_HORZ_CLIP_ADJ 0xa2fc
+#define mmPA_CL_GB_HORZ_DISC_ADJ 0xa2fd
+#define mmPA_CL_UCP_0_X 0xa16f
+#define mmPA_CL_UCP_0_Y 0xa170
+#define mmPA_CL_UCP_0_Z 0xa171
+#define mmPA_CL_UCP_0_W 0xa172
+#define mmPA_CL_UCP_1_X 0xa173
+#define mmPA_CL_UCP_1_Y 0xa174
+#define mmPA_CL_UCP_1_Z 0xa175
+#define mmPA_CL_UCP_1_W 0xa176
+#define mmPA_CL_UCP_2_X 0xa177
+#define mmPA_CL_UCP_2_Y 0xa178
+#define mmPA_CL_UCP_2_Z 0xa179
+#define mmPA_CL_UCP_2_W 0xa17a
+#define mmPA_CL_UCP_3_X 0xa17b
+#define mmPA_CL_UCP_3_Y 0xa17c
+#define mmPA_CL_UCP_3_Z 0xa17d
+#define mmPA_CL_UCP_3_W 0xa17e
+#define mmPA_CL_UCP_4_X 0xa17f
+#define mmPA_CL_UCP_4_Y 0xa180
+#define mmPA_CL_UCP_4_Z 0xa181
+#define mmPA_CL_UCP_4_W 0xa182
+#define mmPA_CL_UCP_5_X 0xa183
+#define mmPA_CL_UCP_5_Y 0xa184
+#define mmPA_CL_UCP_5_Z 0xa185
+#define mmPA_CL_UCP_5_W 0xa186
+#define mmPA_CL_POINT_X_RAD 0xa1f5
+#define mmPA_CL_POINT_Y_RAD 0xa1f6
+#define mmPA_CL_POINT_SIZE 0xa1f7
+#define mmPA_CL_POINT_CULL_RAD 0xa1f8
+#define mmPA_CL_ENHANCE 0x2285
+#define mmPA_CL_RESET_DEBUG 0x2286
+#define mmPA_SU_VTX_CNTL 0xa2f9
+#define mmPA_SU_POINT_SIZE 0xa280
+#define mmPA_SU_POINT_MINMAX 0xa281
+#define mmPA_SU_LINE_CNTL 0xa282
+#define mmPA_SU_LINE_STIPPLE_CNTL 0xa209
+#define mmPA_SU_LINE_STIPPLE_SCALE 0xa20a
+#define mmPA_SU_PRIM_FILTER_CNTL 0xa20b
+#define mmPA_SU_SC_MODE_CNTL 0xa205
+#define mmPA_SU_POLY_OFFSET_DB_FMT_CNTL 0xa2de
+#define mmPA_SU_POLY_OFFSET_CLAMP 0xa2df
+#define mmPA_SU_POLY_OFFSET_FRONT_SCALE 0xa2e0
+#define mmPA_SU_POLY_OFFSET_FRONT_OFFSET 0xa2e1
+#define mmPA_SU_POLY_OFFSET_BACK_SCALE 0xa2e2
+#define mmPA_SU_POLY_OFFSET_BACK_OFFSET 0xa2e3
+#define mmPA_SU_HARDWARE_SCREEN_OFFSET 0xa08d
+#define mmPA_SU_LINE_STIPPLE_VALUE 0xc280
+#define mmPA_SU_PERFCOUNTER0_SELECT 0xd900
+#define mmPA_SU_PERFCOUNTER0_SELECT1 0xd901
+#define mmPA_SU_PERFCOUNTER1_SELECT 0xd902
+#define mmPA_SU_PERFCOUNTER1_SELECT1 0xd903
+#define mmPA_SU_PERFCOUNTER2_SELECT 0xd904
+#define mmPA_SU_PERFCOUNTER3_SELECT 0xd905
+#define mmPA_SU_PERFCOUNTER0_LO 0xd100
+#define mmPA_SU_PERFCOUNTER0_HI 0xd101
+#define mmPA_SU_PERFCOUNTER1_LO 0xd102
+#define mmPA_SU_PERFCOUNTER1_HI 0xd103
+#define mmPA_SU_PERFCOUNTER2_LO 0xd104
+#define mmPA_SU_PERFCOUNTER2_HI 0xd105
+#define mmPA_SU_PERFCOUNTER3_LO 0xd106
+#define mmPA_SU_PERFCOUNTER3_HI 0xd107
+#define mmPA_SC_AA_CONFIG 0xa2f8
+#define mmPA_SC_AA_MASK_X0Y0_X1Y0 0xa30e
+#define mmPA_SC_AA_MASK_X0Y1_X1Y1 0xa30f
+#define mmPA_SC_SHADER_CONTROL 0xa310
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 0xa2fe
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 0xa2ff
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 0xa300
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 0xa301
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 0xa302
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 0xa303
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 0xa304
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 0xa305
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 0xa306
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 0xa307
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 0xa308
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 0xa309
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 0xa30a
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 0xa30b
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 0xa30c
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 0xa30d
+#define mmPA_SC_CENTROID_PRIORITY_0 0xa2f5
+#define mmPA_SC_CENTROID_PRIORITY_1 0xa2f6
+#define mmPA_SC_CLIPRECT_0_TL 0xa084
+#define mmPA_SC_CLIPRECT_0_BR 0xa085
+#define mmPA_SC_CLIPRECT_1_TL 0xa086
+#define mmPA_SC_CLIPRECT_1_BR 0xa087
+#define mmPA_SC_CLIPRECT_2_TL 0xa088
+#define mmPA_SC_CLIPRECT_2_BR 0xa089
+#define mmPA_SC_CLIPRECT_3_TL 0xa08a
+#define mmPA_SC_CLIPRECT_3_BR 0xa08b
+#define mmPA_SC_CLIPRECT_RULE 0xa083
+#define mmPA_SC_EDGERULE 0xa08c
+#define mmPA_SC_LINE_CNTL 0xa2f7
+#define mmPA_SC_LINE_STIPPLE 0xa283
+#define mmPA_SC_MODE_CNTL_0 0xa292
+#define mmPA_SC_MODE_CNTL_1 0xa293
+#define mmPA_SC_RASTER_CONFIG 0xa0d4
+#define mmPA_SC_RASTER_CONFIG_1 0xa0d5
+#define mmPA_SC_SCREEN_EXTENT_CONTROL 0xa0d6
+#define mmPA_SC_GENERIC_SCISSOR_TL 0xa090
+#define mmPA_SC_GENERIC_SCISSOR_BR 0xa091
+#define mmPA_SC_SCREEN_SCISSOR_TL 0xa00c
+#define mmPA_SC_SCREEN_SCISSOR_BR 0xa00d
+#define mmPA_SC_WINDOW_OFFSET 0xa080
+#define mmPA_SC_WINDOW_SCISSOR_TL 0xa081
+#define mmPA_SC_WINDOW_SCISSOR_BR 0xa082
+#define mmPA_SC_VPORT_SCISSOR_0_TL 0xa094
+#define mmPA_SC_VPORT_SCISSOR_1_TL 0xa096
+#define mmPA_SC_VPORT_SCISSOR_2_TL 0xa098
+#define mmPA_SC_VPORT_SCISSOR_3_TL 0xa09a
+#define mmPA_SC_VPORT_SCISSOR_4_TL 0xa09c
+#define mmPA_SC_VPORT_SCISSOR_5_TL 0xa09e
+#define mmPA_SC_VPORT_SCISSOR_6_TL 0xa0a0
+#define mmPA_SC_VPORT_SCISSOR_7_TL 0xa0a2
+#define mmPA_SC_VPORT_SCISSOR_8_TL 0xa0a4
+#define mmPA_SC_VPORT_SCISSOR_9_TL 0xa0a6
+#define mmPA_SC_VPORT_SCISSOR_10_TL 0xa0a8
+#define mmPA_SC_VPORT_SCISSOR_11_TL 0xa0aa
+#define mmPA_SC_VPORT_SCISSOR_12_TL 0xa0ac
+#define mmPA_SC_VPORT_SCISSOR_13_TL 0xa0ae
+#define mmPA_SC_VPORT_SCISSOR_14_TL 0xa0b0
+#define mmPA_SC_VPORT_SCISSOR_15_TL 0xa0b2
+#define mmPA_SC_VPORT_SCISSOR_0_BR 0xa095
+#define mmPA_SC_VPORT_SCISSOR_1_BR 0xa097
+#define mmPA_SC_VPORT_SCISSOR_2_BR 0xa099
+#define mmPA_SC_VPORT_SCISSOR_3_BR 0xa09b
+#define mmPA_SC_VPORT_SCISSOR_4_BR 0xa09d
+#define mmPA_SC_VPORT_SCISSOR_5_BR 0xa09f
+#define mmPA_SC_VPORT_SCISSOR_6_BR 0xa0a1
+#define mmPA_SC_VPORT_SCISSOR_7_BR 0xa0a3
+#define mmPA_SC_VPORT_SCISSOR_8_BR 0xa0a5
+#define mmPA_SC_VPORT_SCISSOR_9_BR 0xa0a7
+#define mmPA_SC_VPORT_SCISSOR_10_BR 0xa0a9
+#define mmPA_SC_VPORT_SCISSOR_11_BR 0xa0ab
+#define mmPA_SC_VPORT_SCISSOR_12_BR 0xa0ad
+#define mmPA_SC_VPORT_SCISSOR_13_BR 0xa0af
+#define mmPA_SC_VPORT_SCISSOR_14_BR 0xa0b1
+#define mmPA_SC_VPORT_SCISSOR_15_BR 0xa0b3
+#define mmPA_SC_VPORT_ZMIN_0 0xa0b4
+#define mmPA_SC_VPORT_ZMIN_1 0xa0b6
+#define mmPA_SC_VPORT_ZMIN_2 0xa0b8
+#define mmPA_SC_VPORT_ZMIN_3 0xa0ba
+#define mmPA_SC_VPORT_ZMIN_4 0xa0bc
+#define mmPA_SC_VPORT_ZMIN_5 0xa0be
+#define mmPA_SC_VPORT_ZMIN_6 0xa0c0
+#define mmPA_SC_VPORT_ZMIN_7 0xa0c2
+#define mmPA_SC_VPORT_ZMIN_8 0xa0c4
+#define mmPA_SC_VPORT_ZMIN_9 0xa0c6
+#define mmPA_SC_VPORT_ZMIN_10 0xa0c8
+#define mmPA_SC_VPORT_ZMIN_11 0xa0ca
+#define mmPA_SC_VPORT_ZMIN_12 0xa0cc
+#define mmPA_SC_VPORT_ZMIN_13 0xa0ce
+#define mmPA_SC_VPORT_ZMIN_14 0xa0d0
+#define mmPA_SC_VPORT_ZMIN_15 0xa0d2
+#define mmPA_SC_VPORT_ZMAX_0 0xa0b5
+#define mmPA_SC_VPORT_ZMAX_1 0xa0b7
+#define mmPA_SC_VPORT_ZMAX_2 0xa0b9
+#define mmPA_SC_VPORT_ZMAX_3 0xa0bb
+#define mmPA_SC_VPORT_ZMAX_4 0xa0bd
+#define mmPA_SC_VPORT_ZMAX_5 0xa0bf
+#define mmPA_SC_VPORT_ZMAX_6 0xa0c1
+#define mmPA_SC_VPORT_ZMAX_7 0xa0c3
+#define mmPA_SC_VPORT_ZMAX_8 0xa0c5
+#define mmPA_SC_VPORT_ZMAX_9 0xa0c7
+#define mmPA_SC_VPORT_ZMAX_10 0xa0c9
+#define mmPA_SC_VPORT_ZMAX_11 0xa0cb
+#define mmPA_SC_VPORT_ZMAX_12 0xa0cd
+#define mmPA_SC_VPORT_ZMAX_13 0xa0cf
+#define mmPA_SC_VPORT_ZMAX_14 0xa0d1
+#define mmPA_SC_VPORT_ZMAX_15 0xa0d3
+#define mmPA_SC_ENHANCE 0x22fc
+#define mmPA_SC_ENHANCE_1 0x22fd
+#define mmPA_SC_DSM_CNTL 0x22fe
+#define mmPA_SC_FIFO_SIZE 0x22f3
+#define mmPA_SC_IF_FIFO_SIZE 0x22f5
+#define mmPA_SC_FORCE_EOV_MAX_CNTS 0x22c9
+#define mmPA_SC_LINE_STIPPLE_STATE 0xc281
+#define mmPA_SC_SCREEN_EXTENT_MIN_0 0xc284
+#define mmPA_SC_SCREEN_EXTENT_MAX_0 0xc285
+#define mmPA_SC_SCREEN_EXTENT_MIN_1 0xc286
+#define mmPA_SC_SCREEN_EXTENT_MAX_1 0xc28b
+#define mmPA_SC_PERFCOUNTER0_SELECT 0xd940
+#define mmPA_SC_PERFCOUNTER0_SELECT1 0xd941
+#define mmPA_SC_PERFCOUNTER1_SELECT 0xd942
+#define mmPA_SC_PERFCOUNTER2_SELECT 0xd943
+#define mmPA_SC_PERFCOUNTER3_SELECT 0xd944
+#define mmPA_SC_PERFCOUNTER4_SELECT 0xd945
+#define mmPA_SC_PERFCOUNTER5_SELECT 0xd946
+#define mmPA_SC_PERFCOUNTER6_SELECT 0xd947
+#define mmPA_SC_PERFCOUNTER7_SELECT 0xd948
+#define mmPA_SC_PERFCOUNTER0_LO 0xd140
+#define mmPA_SC_PERFCOUNTER0_HI 0xd141
+#define mmPA_SC_PERFCOUNTER1_LO 0xd142
+#define mmPA_SC_PERFCOUNTER1_HI 0xd143
+#define mmPA_SC_PERFCOUNTER2_LO 0xd144
+#define mmPA_SC_PERFCOUNTER2_HI 0xd145
+#define mmPA_SC_PERFCOUNTER3_LO 0xd146
+#define mmPA_SC_PERFCOUNTER3_HI 0xd147
+#define mmPA_SC_PERFCOUNTER4_LO 0xd148
+#define mmPA_SC_PERFCOUNTER4_HI 0xd149
+#define mmPA_SC_PERFCOUNTER5_LO 0xd14a
+#define mmPA_SC_PERFCOUNTER5_HI 0xd14b
+#define mmPA_SC_PERFCOUNTER6_LO 0xd14c
+#define mmPA_SC_PERFCOUNTER6_HI 0xd14d
+#define mmPA_SC_PERFCOUNTER7_LO 0xd14e
+#define mmPA_SC_PERFCOUNTER7_HI 0xd14f
+#define mmPA_SC_P3D_TRAP_SCREEN_HV_EN 0xc2a0
+#define mmPA_SC_P3D_TRAP_SCREEN_H 0xc2a1
+#define mmPA_SC_P3D_TRAP_SCREEN_V 0xc2a2
+#define mmPA_SC_P3D_TRAP_SCREEN_OCCURRENCE 0xc2a3
+#define mmPA_SC_P3D_TRAP_SCREEN_COUNT 0xc2a4
+#define mmPA_SC_HP3D_TRAP_SCREEN_HV_EN 0xc2a8
+#define mmPA_SC_HP3D_TRAP_SCREEN_H 0xc2a9
+#define mmPA_SC_HP3D_TRAP_SCREEN_V 0xc2aa
+#define mmPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE 0xc2ab
+#define mmPA_SC_HP3D_TRAP_SCREEN_COUNT 0xc2ac
+#define mmPA_SC_TRAP_SCREEN_HV_EN 0xc2b0
+#define mmPA_SC_TRAP_SCREEN_H 0xc2b1
+#define mmPA_SC_TRAP_SCREEN_V 0xc2b2
+#define mmPA_SC_TRAP_SCREEN_OCCURRENCE 0xc2b3
+#define mmPA_SC_TRAP_SCREEN_COUNT 0xc2b4
+#define mmPA_SC_P3D_TRAP_SCREEN_HV_LOCK 0x22c0
+#define mmPA_SC_HP3D_TRAP_SCREEN_HV_LOCK 0x22c1
+#define mmPA_SC_TRAP_SCREEN_HV_LOCK 0x22c2
+#define mmPA_CL_CNTL_STATUS 0x2284
+#define mmPA_SU_CNTL_STATUS 0x2294
+#define mmPA_SC_FIFO_DEPTH_CNTL 0x2295
+#define mmCGTT_PA_CLK_CTRL 0xf088
+#define mmCGTT_SC_CLK_CTRL 0xf089
+#define mmPA_SU_DEBUG_CNTL 0x2280
+#define mmPA_SU_DEBUG_DATA 0x2281
+#define mmPA_SC_DEBUG_CNTL 0x22f6
+#define mmPA_SC_DEBUG_DATA 0x22f7
+#define ixCLIPPER_DEBUG_REG00 0x0
+#define ixCLIPPER_DEBUG_REG01 0x1
+#define ixCLIPPER_DEBUG_REG02 0x2
+#define ixCLIPPER_DEBUG_REG03 0x3
+#define ixCLIPPER_DEBUG_REG04 0x4
+#define ixCLIPPER_DEBUG_REG05 0x5
+#define ixCLIPPER_DEBUG_REG06 0x6
+#define ixCLIPPER_DEBUG_REG07 0x7
+#define ixCLIPPER_DEBUG_REG08 0x8
+#define ixCLIPPER_DEBUG_REG09 0x9
+#define ixCLIPPER_DEBUG_REG10 0xa
+#define ixCLIPPER_DEBUG_REG11 0xb
+#define ixCLIPPER_DEBUG_REG12 0xc
+#define ixCLIPPER_DEBUG_REG13 0xd
+#define ixCLIPPER_DEBUG_REG14 0xe
+#define ixCLIPPER_DEBUG_REG15 0xf
+#define ixCLIPPER_DEBUG_REG16 0x10
+#define ixCLIPPER_DEBUG_REG17 0x11
+#define ixCLIPPER_DEBUG_REG18 0x12
+#define ixCLIPPER_DEBUG_REG19 0x13
+#define ixSXIFCCG_DEBUG_REG0 0x14
+#define ixSXIFCCG_DEBUG_REG1 0x15
+#define ixSXIFCCG_DEBUG_REG2 0x16
+#define ixSXIFCCG_DEBUG_REG3 0x17
+#define ixSETUP_DEBUG_REG0 0x18
+#define ixSETUP_DEBUG_REG1 0x19
+#define ixSETUP_DEBUG_REG2 0x1a
+#define ixSETUP_DEBUG_REG3 0x1b
+#define ixSETUP_DEBUG_REG4 0x1c
+#define ixSETUP_DEBUG_REG5 0x1d
+#define ixPA_SC_DEBUG_REG0 0x0
+#define ixPA_SC_DEBUG_REG1 0x1
+#define mmCOMPUTE_DISPATCH_INITIATOR 0x2e00
+#define mmCOMPUTE_DIM_X 0x2e01
+#define mmCOMPUTE_DIM_Y 0x2e02
+#define mmCOMPUTE_DIM_Z 0x2e03
+#define mmCOMPUTE_START_X 0x2e04
+#define mmCOMPUTE_START_Y 0x2e05
+#define mmCOMPUTE_START_Z 0x2e06
+#define mmCOMPUTE_NUM_THREAD_X 0x2e07
+#define mmCOMPUTE_NUM_THREAD_Y 0x2e08
+#define mmCOMPUTE_NUM_THREAD_Z 0x2e09
+#define mmCOMPUTE_PIPELINESTAT_ENABLE 0x2e0a
+#define mmCOMPUTE_PERFCOUNT_ENABLE 0x2e0b
+#define mmCOMPUTE_PGM_LO 0x2e0c
+#define mmCOMPUTE_PGM_HI 0x2e0d
+#define mmCOMPUTE_TBA_LO 0x2e0e
+#define mmCOMPUTE_TBA_HI 0x2e0f
+#define mmCOMPUTE_TMA_LO 0x2e10
+#define mmCOMPUTE_TMA_HI 0x2e11
+#define mmCOMPUTE_PGM_RSRC1 0x2e12
+#define mmCOMPUTE_PGM_RSRC2 0x2e13
+#define mmCOMPUTE_VMID 0x2e14
+#define mmCOMPUTE_RESOURCE_LIMITS 0x2e15
+#define mmCOMPUTE_STATIC_THREAD_MGMT_SE0 0x2e16
+#define mmCOMPUTE_STATIC_THREAD_MGMT_SE1 0x2e17
+#define mmCOMPUTE_TMPRING_SIZE 0x2e18
+#define mmCOMPUTE_STATIC_THREAD_MGMT_SE2 0x2e19
+#define mmCOMPUTE_STATIC_THREAD_MGMT_SE3 0x2e1a
+#define mmCOMPUTE_RESTART_X 0x2e1b
+#define mmCOMPUTE_RESTART_Y 0x2e1c
+#define mmCOMPUTE_RESTART_Z 0x2e1d
+#define mmCOMPUTE_THREAD_TRACE_ENABLE 0x2e1e
+#define mmCOMPUTE_MISC_RESERVED 0x2e1f
+#define mmCOMPUTE_DISPATCH_ID 0x2e20
+#define mmCOMPUTE_THREADGROUP_ID 0x2e21
+#define mmCOMPUTE_RELAUNCH 0x2e22
+#define mmCOMPUTE_WAVE_RESTORE_ADDR_LO 0x2e23
+#define mmCOMPUTE_WAVE_RESTORE_ADDR_HI 0x2e24
+#define mmCOMPUTE_WAVE_RESTORE_CONTROL 0x2e25
+#define mmCOMPUTE_USER_DATA_0 0x2e40
+#define mmCOMPUTE_USER_DATA_1 0x2e41
+#define mmCOMPUTE_USER_DATA_2 0x2e42
+#define mmCOMPUTE_USER_DATA_3 0x2e43
+#define mmCOMPUTE_USER_DATA_4 0x2e44
+#define mmCOMPUTE_USER_DATA_5 0x2e45
+#define mmCOMPUTE_USER_DATA_6 0x2e46
+#define mmCOMPUTE_USER_DATA_7 0x2e47
+#define mmCOMPUTE_USER_DATA_8 0x2e48
+#define mmCOMPUTE_USER_DATA_9 0x2e49
+#define mmCOMPUTE_USER_DATA_10 0x2e4a
+#define mmCOMPUTE_USER_DATA_11 0x2e4b
+#define mmCOMPUTE_USER_DATA_12 0x2e4c
+#define mmCOMPUTE_USER_DATA_13 0x2e4d
+#define mmCOMPUTE_USER_DATA_14 0x2e4e
+#define mmCOMPUTE_USER_DATA_15 0x2e4f
+#define mmCOMPUTE_NOWHERE 0x2e7f
+#define mmCSPRIV_CONNECT 0x0
+#define mmCSPRIV_THREAD_TRACE_TG0 0x1e
+#define mmCSPRIV_THREAD_TRACE_TG1 0x1e
+#define mmCSPRIV_THREAD_TRACE_TG2 0x1e
+#define mmCSPRIV_THREAD_TRACE_TG3 0x1e
+#define mmCSPRIV_THREAD_TRACE_EVENT 0x1f
+#define mmRLC_CNTL 0xec00
+#define mmRLC_DEBUG_SELECT 0xec01
+#define mmRLC_DEBUG 0xec02
+#define mmRLC_MC_CNTL 0xec03
+#define mmRLC_STAT 0xec04
+#define mmRLC_SAFE_MODE 0xec05
+#define mmRLC_MEM_SLP_CNTL 0xec06
+#define mmSMU_RLC_RESPONSE 0xec07
+#define mmRLC_RLCV_SAFE_MODE 0xec08
+#define mmRLC_SMU_SAFE_MODE 0xec09
+#define mmRLC_RLCV_COMMAND 0xec0a
+#define mmRLC_CLK_CNTL 0xec0b
+#define mmRLC_PERFMON_CLK_CNTL 0xdcbf
+#define mmRLC_PERFMON_CNTL 0xdcc0
+#define mmRLC_PERFCOUNTER0_SELECT 0xdcc1
+#define mmRLC_PERFCOUNTER1_SELECT 0xdcc2
+#define mmRLC_PERFCOUNTER0_LO 0xd480
+#define mmRLC_PERFCOUNTER1_LO 0xd482
+#define mmRLC_PERFCOUNTER0_HI 0xd481
+#define mmRLC_PERFCOUNTER1_HI 0xd483
+#define mmCGTT_RLC_CLK_CTRL 0xf0b8
+#define mmRLC_LB_CNTL 0xec19
+#define mmRLC_LB_CNTR_MAX 0xec12
+#define mmRLC_LB_CNTR_INIT 0xec1b
+#define mmRLC_LOAD_BALANCE_CNTR 0xec1c
+#define mmRLC_JUMP_TABLE_RESTORE 0xec1e
+#define mmRLC_PG_DELAY_2 0xec1f
+#define mmRLC_GPM_DEBUG_SELECT 0xec20
+#define mmRLC_GPM_DEBUG 0xec21
+#define mmRLC_GPM_DEBUG_INST_A 0xec22
+#define mmRLC_GPM_DEBUG_INST_B 0xec23
+#define mmRLC_GPM_DEBUG_INST_ADDR 0xec1d
+#define mmRLC_GPM_UCODE_ADDR 0xf83c
+#define mmRLC_GPM_UCODE_DATA 0xf83d
+#define mmGPU_BIST_CONTROL 0xf835
+#define mmRLC_ROM_CNTL 0xf836
+#define mmRLC_GPU_CLOCK_COUNT_LSB 0xec24
+#define mmRLC_GPU_CLOCK_COUNT_MSB 0xec25
+#define mmRLC_CAPTURE_GPU_CLOCK_COUNT 0xec26
+#define mmRLC_UCODE_CNTL 0xec27
+#define mmRLC_GPM_STAT 0xec40
+#define mmRLC_GPU_CLOCK_32_RES_SEL 0xec41
+#define mmRLC_GPU_CLOCK_32 0xec42
+#define mmRLC_PG_CNTL 0xec43
+#define mmRLC_GPM_THREAD_PRIORITY 0xec44
+#define mmRLC_GPM_THREAD_ENABLE 0xec45
+#define mmRLC_GPM_VMID_THREAD0 0xec46
+#define mmRLC_GPM_VMID_THREAD1 0xec47
+#define mmRLC_CGTT_MGCG_OVERRIDE 0xec48
+#define mmRLC_CGCG_CGLS_CTRL 0xec49
+#define mmRLC_CGCG_RAMP_CTRL 0xec4a
+#define mmRLC_DYN_PG_STATUS 0xec4b
+#define mmRLC_DYN_PG_REQUEST 0xec4c
+#define mmRLC_PG_DELAY 0xec4d
+#define mmRLC_CU_STATUS 0xec4e
+#define mmRLC_LB_INIT_CU_MASK 0xec4f
+#define mmRLC_LB_ALWAYS_ACTIVE_CU_MASK 0xec50
+#define mmRLC_LB_PARAMS 0xec51
+#define mmRLC_THREAD1_DELAY 0xec52
+#define mmRLC_PG_ALWAYS_ON_CU_MASK 0xec53
+#define mmRLC_MAX_PG_CU 0xec54
+#define mmRLC_AUTO_PG_CTRL 0xec55
+#define mmRLC_SMU_GRBM_REG_SAVE_CTRL 0xec56
+#define mmRLC_SERDES_RD_MASTER_INDEX 0xec59
+#define mmRLC_SERDES_RD_DATA_0 0xec5a
+#define mmRLC_SERDES_RD_DATA_1 0xec5b
+#define mmRLC_SERDES_RD_DATA_2 0xec5c
+#define mmRLC_SERDES_WR_CU_MASTER_MASK 0xec5d
+#define mmRLC_SERDES_WR_NONCU_MASTER_MASK 0xec5e
+#define mmRLC_SERDES_WR_CTRL 0xec5f
+#define mmRLC_SERDES_WR_DATA 0xec60
+#define mmRLC_SERDES_CU_MASTER_BUSY 0xec61
+#define mmRLC_SERDES_NONCU_MASTER_BUSY 0xec62
+#define mmRLC_GPM_GENERAL_0 0xec63
+#define mmRLC_GPM_GENERAL_1 0xec64
+#define mmRLC_GPM_GENERAL_2 0xec65
+#define mmRLC_GPM_GENERAL_3 0xec66
+#define mmRLC_GPM_GENERAL_4 0xec67
+#define mmRLC_GPM_GENERAL_5 0xec68
+#define mmRLC_GPM_GENERAL_6 0xec69
+#define mmRLC_GPM_GENERAL_7 0xec6a
+#define mmRLC_GPM_SCRATCH_ADDR 0xec6c
+#define mmRLC_GPM_SCRATCH_DATA 0xec6d
+#define mmRLC_STATIC_PG_STATUS 0xec6e
+#define mmRLC_GPM_PERF_COUNT_0 0xec6f
+#define mmRLC_GPM_PERF_COUNT_1 0xec70
+#define mmRLC_GPR_REG1 0xec79
+#define mmRLC_GPR_REG2 0xec7a
+#define mmRLC_MGCG_CTRL 0xec1a
+#define mmRLC_GPM_THREAD_RESET 0xec28
+#define mmRLC_SPM_VMID 0xec71
+#define mmRLC_SPM_INT_CNTL 0xec72
+#define mmRLC_SPM_INT_STATUS 0xec73
+#define mmRLC_SPM_DEBUG_SELECT 0xec74
+#define mmRLC_SPM_DEBUG 0xec75
+#define mmRLC_SMU_MESSAGE 0xec76
+#define mmRLC_GPM_LOG_SIZE 0xec77
+#define mmRLC_GPM_LOG_CONT 0xec7b
+#define mmRLC_PG_DELAY_3 0xec78
+#define mmRLC_GPM_INT_DISABLE_TH0 0xec7c
+#define mmRLC_GPM_INT_DISABLE_TH1 0xec7d
+#define mmRLC_GPM_INT_FORCE_TH0 0xec7e
+#define mmRLC_GPM_INT_FORCE_TH1 0xec7f
+#define mmRLC_SRM_CNTL 0xec80
+#define mmRLC_SRM_DEBUG_SELECT 0xec81
+#define mmRLC_SRM_DEBUG 0xec82
+#define mmRLC_SRM_ARAM_ADDR 0xec83
+#define mmRLC_SRM_ARAM_DATA 0xec84
+#define mmRLC_SRM_DRAM_ADDR 0xec85
+#define mmRLC_SRM_DRAM_DATA 0xec86
+#define mmRLC_SRM_GPM_COMMAND 0xec87
+#define mmRLC_SRM_GPM_COMMAND_STATUS 0xec88
+#define mmRLC_SRM_RLCV_COMMAND 0xec89
+#define mmRLC_SRM_RLCV_COMMAND_STATUS 0xec8a
+#define mmRLC_SRM_INDEX_CNTL_ADDR_0 0xec8b
+#define mmRLC_SRM_INDEX_CNTL_ADDR_1 0xec8c
+#define mmRLC_SRM_INDEX_CNTL_ADDR_2 0xec8d
+#define mmRLC_SRM_INDEX_CNTL_ADDR_3 0xec8e
+#define mmRLC_SRM_INDEX_CNTL_ADDR_4 0xec8f
+#define mmRLC_SRM_INDEX_CNTL_ADDR_5 0xec90
+#define mmRLC_SRM_INDEX_CNTL_ADDR_6 0xec91
+#define mmRLC_SRM_INDEX_CNTL_ADDR_7 0xec92
+#define mmRLC_SRM_INDEX_CNTL_DATA_0 0xec93
+#define mmRLC_SRM_INDEX_CNTL_DATA_1 0xec94
+#define mmRLC_SRM_INDEX_CNTL_DATA_2 0xec95
+#define mmRLC_SRM_INDEX_CNTL_DATA_3 0xec96
+#define mmRLC_SRM_INDEX_CNTL_DATA_4 0xec97
+#define mmRLC_SRM_INDEX_CNTL_DATA_5 0xec98
+#define mmRLC_SRM_INDEX_CNTL_DATA_6 0xec99
+#define mmRLC_SRM_INDEX_CNTL_DATA_7 0xec9a
+#define mmRLC_SRM_STAT 0xec9b
+#define mmRLC_SRM_GPM_ABORT 0xec9c
+#define mmRLC_CSIB_ADDR_LO 0xeca2
+#define mmRLC_CSIB_ADDR_HI 0xeca3
+#define mmRLC_CSIB_LENGTH 0xeca4
+#define mmRLC_CP_RESPONSE0 0xeca5
+#define mmRLC_CP_RESPONSE1 0xeca6
+#define mmRLC_CP_RESPONSE2 0xeca7
+#define mmRLC_CP_RESPONSE3 0xeca8
+#define mmRLC_SMU_COMMAND 0xeca9
+#define mmRLC_CP_SCHEDULERS 0xecaa
+#define mmRLC_SMU_ARGUMENT_1 0xecab
+#define mmRLC_SMU_ARGUMENT_2 0xecac
+#define mmRLC_GPM_GENERAL_8 0xecad
+#define mmRLC_GPM_GENERAL_9 0xecae
+#define mmRLC_GPM_GENERAL_10 0xecaf
+#define mmRLC_GPM_GENERAL_11 0xecb0
+#define mmRLC_GPM_GENERAL_12 0xecb1
+#define mmRLC_SPM_PERFMON_CNTL 0xdc80
+#define mmRLC_SPM_PERFMON_RING_BASE_LO 0xdc81
+#define mmRLC_SPM_PERFMON_RING_BASE_HI 0xdc82
+#define mmRLC_SPM_PERFMON_RING_SIZE 0xdc83
+#define mmRLC_SPM_PERFMON_SEGMENT_SIZE 0xdc84
+#define mmRLC_SPM_SE_MUXSEL_ADDR 0xdc85
+#define mmRLC_SPM_SE_MUXSEL_DATA 0xdc86
+#define mmRLC_SPM_CPG_PERFMON_SAMPLE_DELAY 0xdc87
+#define mmRLC_SPM_CPC_PERFMON_SAMPLE_DELAY 0xdc88
+#define mmRLC_SPM_CPF_PERFMON_SAMPLE_DELAY 0xdc89
+#define mmRLC_SPM_CB_PERFMON_SAMPLE_DELAY 0xdc8a
+#define mmRLC_SPM_DB_PERFMON_SAMPLE_DELAY 0xdc8b
+#define mmRLC_SPM_PA_PERFMON_SAMPLE_DELAY 0xdc8c
+#define mmRLC_SPM_GDS_PERFMON_SAMPLE_DELAY 0xdc8d
+#define mmRLC_SPM_IA_PERFMON_SAMPLE_DELAY 0xdc8e
+#define mmRLC_SPM_SC_PERFMON_SAMPLE_DELAY 0xdc90
+#define mmRLC_SPM_TCC_PERFMON_SAMPLE_DELAY 0xdc91
+#define mmRLC_SPM_TCA_PERFMON_SAMPLE_DELAY 0xdc92
+#define mmRLC_SPM_TCP_PERFMON_SAMPLE_DELAY 0xdc93
+#define mmRLC_SPM_TA_PERFMON_SAMPLE_DELAY 0xdc94
+#define mmRLC_SPM_TD_PERFMON_SAMPLE_DELAY 0xdc95
+#define mmRLC_SPM_VGT_PERFMON_SAMPLE_DELAY 0xdc96
+#define mmRLC_SPM_SPI_PERFMON_SAMPLE_DELAY 0xdc97
+#define mmRLC_SPM_SQG_PERFMON_SAMPLE_DELAY 0xdc98
+#define mmRLC_SPM_SX_PERFMON_SAMPLE_DELAY 0xdc9a
+#define mmRLC_SPM_GLOBAL_MUXSEL_ADDR 0xdc9b
+#define mmRLC_SPM_GLOBAL_MUXSEL_DATA 0xdc9c
+#define mmRLC_SPM_RING_RDPTR 0xdc9d
+#define mmRLC_SPM_SEGMENT_THRESHOLD 0xdc9e
+#define mmRLC_GPU_IOV_VF_ENABLE 0xfb00
+#define mmRLC_GPU_IOV_RLC_RESPONSE 0xfb4d
+#define mmRLC_GPU_IOV_ACTIVE_FCN_ID 0xfb40
+#define mmSPI_PS_INPUT_CNTL_0 0xa191
+#define mmSPI_PS_INPUT_CNTL_1 0xa192
+#define mmSPI_PS_INPUT_CNTL_2 0xa193
+#define mmSPI_PS_INPUT_CNTL_3 0xa194
+#define mmSPI_PS_INPUT_CNTL_4 0xa195
+#define mmSPI_PS_INPUT_CNTL_5 0xa196
+#define mmSPI_PS_INPUT_CNTL_6 0xa197
+#define mmSPI_PS_INPUT_CNTL_7 0xa198
+#define mmSPI_PS_INPUT_CNTL_8 0xa199
+#define mmSPI_PS_INPUT_CNTL_9 0xa19a
+#define mmSPI_PS_INPUT_CNTL_10 0xa19b
+#define mmSPI_PS_INPUT_CNTL_11 0xa19c
+#define mmSPI_PS_INPUT_CNTL_12 0xa19d
+#define mmSPI_PS_INPUT_CNTL_13 0xa19e
+#define mmSPI_PS_INPUT_CNTL_14 0xa19f
+#define mmSPI_PS_INPUT_CNTL_15 0xa1a0
+#define mmSPI_PS_INPUT_CNTL_16 0xa1a1
+#define mmSPI_PS_INPUT_CNTL_17 0xa1a2
+#define mmSPI_PS_INPUT_CNTL_18 0xa1a3
+#define mmSPI_PS_INPUT_CNTL_19 0xa1a4
+#define mmSPI_PS_INPUT_CNTL_20 0xa1a5
+#define mmSPI_PS_INPUT_CNTL_21 0xa1a6
+#define mmSPI_PS_INPUT_CNTL_22 0xa1a7
+#define mmSPI_PS_INPUT_CNTL_23 0xa1a8
+#define mmSPI_PS_INPUT_CNTL_24 0xa1a9
+#define mmSPI_PS_INPUT_CNTL_25 0xa1aa
+#define mmSPI_PS_INPUT_CNTL_26 0xa1ab
+#define mmSPI_PS_INPUT_CNTL_27 0xa1ac
+#define mmSPI_PS_INPUT_CNTL_28 0xa1ad
+#define mmSPI_PS_INPUT_CNTL_29 0xa1ae
+#define mmSPI_PS_INPUT_CNTL_30 0xa1af
+#define mmSPI_PS_INPUT_CNTL_31 0xa1b0
+#define mmSPI_VS_OUT_CONFIG 0xa1b1
+#define mmSPI_PS_INPUT_ENA 0xa1b3
+#define mmSPI_PS_INPUT_ADDR 0xa1b4
+#define mmSPI_INTERP_CONTROL_0 0xa1b5
+#define mmSPI_PS_IN_CONTROL 0xa1b6
+#define mmSPI_BARYC_CNTL 0xa1b8
+#define mmSPI_TMPRING_SIZE 0xa1ba
+#define mmSPI_SHADER_POS_FORMAT 0xa1c3
+#define mmSPI_SHADER_Z_FORMAT 0xa1c4
+#define mmSPI_SHADER_COL_FORMAT 0xa1c5
+#define mmSPI_ARB_PRIORITY 0x31c0
+#define mmSPI_ARB_CYCLES_0 0x31c1
+#define mmSPI_ARB_CYCLES_1 0x31c2
+#define mmSPI_CDBG_SYS_GFX 0x31c3
+#define mmSPI_CDBG_SYS_HP3D 0x31c4
+#define mmSPI_CDBG_SYS_CS0 0x31c5
+#define mmSPI_CDBG_SYS_CS1 0x31c6
+#define mmSPI_WCL_PIPE_PERCENT_GFX 0x31c7
+#define mmSPI_WCL_PIPE_PERCENT_HP3D 0x31c8
+#define mmSPI_WCL_PIPE_PERCENT_CS0 0x31c9
+#define mmSPI_WCL_PIPE_PERCENT_CS1 0x31ca
+#define mmSPI_WCL_PIPE_PERCENT_CS2 0x31cb
+#define mmSPI_WCL_PIPE_PERCENT_CS3 0x31cc
+#define mmSPI_WCL_PIPE_PERCENT_CS4 0x31cd
+#define mmSPI_WCL_PIPE_PERCENT_CS5 0x31ce
+#define mmSPI_WCL_PIPE_PERCENT_CS6 0x31cf
+#define mmSPI_WCL_PIPE_PERCENT_CS7 0x31d0
+#define mmSPI_GDBG_WAVE_CNTL 0x31d1
+#define mmSPI_GDBG_TRAP_CONFIG 0x31d2
+#define mmSPI_GDBG_TRAP_MASK 0x31d3
+#define mmSPI_GDBG_TBA_LO 0x31d4
+#define mmSPI_GDBG_TBA_HI 0x31d5
+#define mmSPI_GDBG_TMA_LO 0x31d6
+#define mmSPI_GDBG_TMA_HI 0x31d7
+#define mmSPI_GDBG_TRAP_DATA0 0x31d8
+#define mmSPI_GDBG_TRAP_DATA1 0x31d9
+#define mmSPI_RESET_DEBUG 0x31da
+#define mmSPI_COMPUTE_QUEUE_RESET 0x31db
+#define mmSPI_RESOURCE_RESERVE_CU_0 0x31dc
+#define mmSPI_RESOURCE_RESERVE_CU_1 0x31dd
+#define mmSPI_RESOURCE_RESERVE_CU_2 0x31de
+#define mmSPI_RESOURCE_RESERVE_CU_3 0x31df
+#define mmSPI_RESOURCE_RESERVE_CU_4 0x31e0
+#define mmSPI_RESOURCE_RESERVE_CU_5 0x31e1
+#define mmSPI_RESOURCE_RESERVE_CU_6 0x31e2
+#define mmSPI_RESOURCE_RESERVE_CU_7 0x31e3
+#define mmSPI_RESOURCE_RESERVE_CU_8 0x31e4
+#define mmSPI_RESOURCE_RESERVE_CU_9 0x31e5
+#define mmSPI_RESOURCE_RESERVE_CU_10 0x31f0
+#define mmSPI_RESOURCE_RESERVE_CU_11 0x31f1
+#define mmSPI_RESOURCE_RESERVE_CU_12 0x31f4
+#define mmSPI_RESOURCE_RESERVE_CU_13 0x31f5
+#define mmSPI_RESOURCE_RESERVE_CU_14 0x31f6
+#define mmSPI_RESOURCE_RESERVE_CU_15 0x31f7
+#define mmSPI_RESOURCE_RESERVE_EN_CU_0 0x31e6
+#define mmSPI_RESOURCE_RESERVE_EN_CU_1 0x31e7
+#define mmSPI_RESOURCE_RESERVE_EN_CU_2 0x31e8
+#define mmSPI_RESOURCE_RESERVE_EN_CU_3 0x31e9
+#define mmSPI_RESOURCE_RESERVE_EN_CU_4 0x31ea
+#define mmSPI_RESOURCE_RESERVE_EN_CU_5 0x31eb
+#define mmSPI_RESOURCE_RESERVE_EN_CU_6 0x31ec
+#define mmSPI_RESOURCE_RESERVE_EN_CU_7 0x31ed
+#define mmSPI_RESOURCE_RESERVE_EN_CU_8 0x31ee
+#define mmSPI_RESOURCE_RESERVE_EN_CU_9 0x31ef
+#define mmSPI_RESOURCE_RESERVE_EN_CU_10 0x31f2
+#define mmSPI_RESOURCE_RESERVE_EN_CU_11 0x31f3
+#define mmSPI_RESOURCE_RESERVE_EN_CU_12 0x31f8
+#define mmSPI_RESOURCE_RESERVE_EN_CU_13 0x31f9
+#define mmSPI_RESOURCE_RESERVE_EN_CU_14 0x31fa
+#define mmSPI_RESOURCE_RESERVE_EN_CU_15 0x31fb
+#define mmSPI_COMPUTE_WF_CTX_SAVE 0x31fc
+#define mmSPI_PS_MAX_WAVE_ID 0x243a
+#define mmSPI_START_PHASE 0x243b
+#define mmSPI_GFX_CNTL 0x243c
+#define mmSPI_CONFIG_CNTL 0x2440
+#define mmSPI_DEBUG_CNTL 0x2441
+#define mmSPI_DEBUG_READ 0x2442
+#define mmSPI_DSM_CNTL 0x2443
+#define mmSPI_EDC_CNT 0x2444
+#define mmSPI_PERFCOUNTER0_SELECT 0xd980
+#define mmSPI_PERFCOUNTER1_SELECT 0xd981
+#define mmSPI_PERFCOUNTER2_SELECT 0xd982
+#define mmSPI_PERFCOUNTER3_SELECT 0xd983
+#define mmSPI_PERFCOUNTER0_SELECT1 0xd984
+#define mmSPI_PERFCOUNTER1_SELECT1 0xd985
+#define mmSPI_PERFCOUNTER2_SELECT1 0xd986
+#define mmSPI_PERFCOUNTER3_SELECT1 0xd987
+#define mmSPI_PERFCOUNTER4_SELECT 0xd988
+#define mmSPI_PERFCOUNTER5_SELECT 0xd989
+#define mmSPI_PERFCOUNTER_BINS 0xd98a
+#define mmSPI_PERFCOUNTER0_HI 0xd180
+#define mmSPI_PERFCOUNTER0_LO 0xd181
+#define mmSPI_PERFCOUNTER1_HI 0xd182
+#define mmSPI_PERFCOUNTER1_LO 0xd183
+#define mmSPI_PERFCOUNTER2_HI 0xd184
+#define mmSPI_PERFCOUNTER2_LO 0xd185
+#define mmSPI_PERFCOUNTER3_HI 0xd186
+#define mmSPI_PERFCOUNTER3_LO 0xd187
+#define mmSPI_PERFCOUNTER4_HI 0xd188
+#define mmSPI_PERFCOUNTER4_LO 0xd189
+#define mmSPI_PERFCOUNTER5_HI 0xd18a
+#define mmSPI_PERFCOUNTER5_LO 0xd18b
+#define mmSPI_CONFIG_CNTL_1 0x244f
+#define mmSPI_DEBUG_BUSY 0x2450
+#define mmSPI_CONFIG_CNTL_2 0x2451
+#define mmCGTS_SM_CTRL_REG 0xf000
+#define mmCGTS_RD_CTRL_REG 0xf001
+#define mmCGTS_RD_REG 0xf002
+#define mmCGTS_TCC_DISABLE 0xf003
+#define mmCGTS_USER_TCC_DISABLE 0xf004
+#define mmCGTS_CU0_SP0_CTRL_REG 0xf008
+#define mmCGTS_CU0_LDS_SQ_CTRL_REG 0xf009
+#define mmCGTS_CU0_TA_SQC_CTRL_REG 0xf00a
+#define mmCGTS_CU0_SP1_CTRL_REG 0xf00b
+#define mmCGTS_CU0_TD_TCP_CTRL_REG 0xf00c
+#define mmCGTS_CU1_SP0_CTRL_REG 0xf00d
+#define mmCGTS_CU1_LDS_SQ_CTRL_REG 0xf00e
+#define mmCGTS_CU1_TA_CTRL_REG 0xf00f
+#define mmCGTS_CU1_SP1_CTRL_REG 0xf010
+#define mmCGTS_CU1_TD_TCP_CTRL_REG 0xf011
+#define mmCGTS_CU2_SP0_CTRL_REG 0xf012
+#define mmCGTS_CU2_LDS_SQ_CTRL_REG 0xf013
+#define mmCGTS_CU2_TA_CTRL_REG 0xf014
+#define mmCGTS_CU2_SP1_CTRL_REG 0xf015
+#define mmCGTS_CU2_TD_TCP_CTRL_REG 0xf016
+#define mmCGTS_CU3_SP0_CTRL_REG 0xf017
+#define mmCGTS_CU3_LDS_SQ_CTRL_REG 0xf018
+#define mmCGTS_CU3_TA_CTRL_REG 0xf019
+#define mmCGTS_CU3_SP1_CTRL_REG 0xf01a
+#define mmCGTS_CU3_TD_TCP_CTRL_REG 0xf01b
+#define mmCGTS_CU4_SP0_CTRL_REG 0xf01c
+#define mmCGTS_CU4_LDS_SQ_CTRL_REG 0xf01d
+#define mmCGTS_CU4_TA_SQC_CTRL_REG 0xf01e
+#define mmCGTS_CU4_SP1_CTRL_REG 0xf01f
+#define mmCGTS_CU4_TD_TCP_CTRL_REG 0xf020
+#define mmCGTS_CU5_SP0_CTRL_REG 0xf021
+#define mmCGTS_CU5_LDS_SQ_CTRL_REG 0xf022
+#define mmCGTS_CU5_TA_CTRL_REG 0xf023
+#define mmCGTS_CU5_SP1_CTRL_REG 0xf024
+#define mmCGTS_CU5_TD_TCP_CTRL_REG 0xf025
+#define mmCGTS_CU6_SP0_CTRL_REG 0xf026
+#define mmCGTS_CU6_LDS_SQ_CTRL_REG 0xf027
+#define mmCGTS_CU6_TA_CTRL_REG 0xf028
+#define mmCGTS_CU6_SP1_CTRL_REG 0xf029
+#define mmCGTS_CU6_TD_TCP_CTRL_REG 0xf02a
+#define mmCGTS_CU7_SP0_CTRL_REG 0xf02b
+#define mmCGTS_CU7_LDS_SQ_CTRL_REG 0xf02c
+#define mmCGTS_CU7_TA_CTRL_REG 0xf02d
+#define mmCGTS_CU7_SP1_CTRL_REG 0xf02e
+#define mmCGTS_CU7_TD_TCP_CTRL_REG 0xf02f
+#define mmCGTS_CU8_SP0_CTRL_REG 0xf030
+#define mmCGTS_CU8_LDS_SQ_CTRL_REG 0xf031
+#define mmCGTS_CU8_TA_SQC_CTRL_REG 0xf032
+#define mmCGTS_CU8_SP1_CTRL_REG 0xf033
+#define mmCGTS_CU8_TD_TCP_CTRL_REG 0xf034
+#define mmCGTS_CU9_SP0_CTRL_REG 0xf035
+#define mmCGTS_CU9_LDS_SQ_CTRL_REG 0xf036
+#define mmCGTS_CU9_TA_CTRL_REG 0xf037
+#define mmCGTS_CU9_SP1_CTRL_REG 0xf038
+#define mmCGTS_CU9_TD_TCP_CTRL_REG 0xf039
+#define mmCGTS_CU10_SP0_CTRL_REG 0xf03a
+#define mmCGTS_CU10_LDS_SQ_CTRL_REG 0xf03b
+#define mmCGTS_CU10_TA_CTRL_REG 0xf03c
+#define mmCGTS_CU10_SP1_CTRL_REG 0xf03d
+#define mmCGTS_CU10_TD_TCP_CTRL_REG 0xf03e
+#define mmCGTS_CU11_SP0_CTRL_REG 0xf03f
+#define mmCGTS_CU11_LDS_SQ_CTRL_REG 0xf040
+#define mmCGTS_CU11_TA_CTRL_REG 0xf041
+#define mmCGTS_CU11_SP1_CTRL_REG 0xf042
+#define mmCGTS_CU11_TD_TCP_CTRL_REG 0xf043
+#define mmCGTS_CU12_SP0_CTRL_REG 0xf044
+#define mmCGTS_CU12_LDS_SQ_CTRL_REG 0xf045
+#define mmCGTS_CU12_TA_SQC_CTRL_REG 0xf046
+#define mmCGTS_CU12_SP1_CTRL_REG 0xf047
+#define mmCGTS_CU12_TD_TCP_CTRL_REG 0xf048
+#define mmCGTS_CU13_SP0_CTRL_REG 0xf049
+#define mmCGTS_CU13_LDS_SQ_CTRL_REG 0xf04a
+#define mmCGTS_CU13_TA_CTRL_REG 0xf04b
+#define mmCGTS_CU13_SP1_CTRL_REG 0xf04c
+#define mmCGTS_CU13_TD_TCP_CTRL_REG 0xf04d
+#define mmCGTS_CU14_SP0_CTRL_REG 0xf04e
+#define mmCGTS_CU14_LDS_SQ_CTRL_REG 0xf04f
+#define mmCGTS_CU14_TA_CTRL_REG 0xf050
+#define mmCGTS_CU14_SP1_CTRL_REG 0xf051
+#define mmCGTS_CU14_TD_TCP_CTRL_REG 0xf052
+#define mmCGTS_CU15_SP0_CTRL_REG 0xf053
+#define mmCGTS_CU15_LDS_SQ_CTRL_REG 0xf054
+#define mmCGTS_CU15_TA_CTRL_REG 0xf055
+#define mmCGTS_CU15_SP1_CTRL_REG 0xf056
+#define mmCGTS_CU15_TD_TCP_CTRL_REG 0xf057
+#define mmCGTT_SPI_CLK_CTRL 0xf080
+#define mmCGTT_PC_CLK_CTRL 0xf081
+#define mmCGTT_BCI_CLK_CTRL 0xf082
+#define mmSPI_WF_LIFETIME_CNTL 0x24aa
+#define mmSPI_WF_LIFETIME_LIMIT_0 0x24ab
+#define mmSPI_WF_LIFETIME_LIMIT_1 0x24ac
+#define mmSPI_WF_LIFETIME_LIMIT_2 0x24ad
+#define mmSPI_WF_LIFETIME_LIMIT_3 0x24ae
+#define mmSPI_WF_LIFETIME_LIMIT_4 0x24af
+#define mmSPI_WF_LIFETIME_LIMIT_5 0x24b0
+#define mmSPI_WF_LIFETIME_LIMIT_6 0x24b1
+#define mmSPI_WF_LIFETIME_LIMIT_7 0x24b2
+#define mmSPI_WF_LIFETIME_LIMIT_8 0x24b3
+#define mmSPI_WF_LIFETIME_LIMIT_9 0x24b4
+#define mmSPI_WF_LIFETIME_STATUS_0 0x24b5
+#define mmSPI_WF_LIFETIME_STATUS_1 0x24b6
+#define mmSPI_WF_LIFETIME_STATUS_2 0x24b7
+#define mmSPI_WF_LIFETIME_STATUS_3 0x24b8
+#define mmSPI_WF_LIFETIME_STATUS_4 0x24b9
+#define mmSPI_WF_LIFETIME_STATUS_5 0x24ba
+#define mmSPI_WF_LIFETIME_STATUS_6 0x24bb
+#define mmSPI_WF_LIFETIME_STATUS_7 0x24bc
+#define mmSPI_WF_LIFETIME_STATUS_8 0x24bd
+#define mmSPI_WF_LIFETIME_STATUS_9 0x24be
+#define mmSPI_WF_LIFETIME_STATUS_10 0x24bf
+#define mmSPI_WF_LIFETIME_STATUS_11 0x24c0
+#define mmSPI_WF_LIFETIME_STATUS_12 0x24c1
+#define mmSPI_WF_LIFETIME_STATUS_13 0x24c2
+#define mmSPI_WF_LIFETIME_STATUS_14 0x24c3
+#define mmSPI_WF_LIFETIME_STATUS_15 0x24c4
+#define mmSPI_WF_LIFETIME_STATUS_16 0x24c5
+#define mmSPI_WF_LIFETIME_STATUS_17 0x24c6
+#define mmSPI_WF_LIFETIME_STATUS_18 0x24c7
+#define mmSPI_WF_LIFETIME_STATUS_19 0x24c8
+#define mmSPI_WF_LIFETIME_STATUS_20 0x24c9
+#define mmSPI_WF_LIFETIME_DEBUG 0x24ca
+#define mmSPI_SLAVE_DEBUG_BUSY 0x24d3
+#define mmSPI_LB_CTR_CTRL 0x24d4
+#define mmSPI_LB_CU_MASK 0x24d5
+#define mmSPI_LB_DATA_REG 0x24d6
+#define mmSPI_PG_ENABLE_STATIC_CU_MASK 0x24d7
+#define mmSPI_GDS_CREDITS 0x24d8
+#define mmSPI_SX_EXPORT_BUFFER_SIZES 0x24d9
+#define mmSPI_SX_SCOREBOARD_BUFFER_SIZES 0x24da
+#define mmSPI_CSQ_WF_ACTIVE_STATUS 0x24db
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_0 0x24dc
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_1 0x24dd
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_2 0x24de
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_3 0x24df
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_4 0x24e0
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_5 0x24e1
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_6 0x24e2
+#define mmSPI_CSQ_WF_ACTIVE_COUNT_7 0x24e3
+#define mmBCI_DEBUG_READ 0x24eb
+#define mmSPI_P0_TRAP_SCREEN_PSBA_LO 0x24ec
+#define mmSPI_P0_TRAP_SCREEN_PSBA_HI 0x24ed
+#define mmSPI_P0_TRAP_SCREEN_PSMA_LO 0x24ee
+#define mmSPI_P0_TRAP_SCREEN_PSMA_HI 0x24ef
+#define mmSPI_P0_TRAP_SCREEN_GPR_MIN 0x24f0
+#define mmSPI_P1_TRAP_SCREEN_PSBA_LO 0x24f1
+#define mmSPI_P1_TRAP_SCREEN_PSBA_HI 0x24f2
+#define mmSPI_P1_TRAP_SCREEN_PSMA_LO 0x24f3
+#define mmSPI_P1_TRAP_SCREEN_PSMA_HI 0x24f4
+#define mmSPI_P1_TRAP_SCREEN_GPR_MIN 0x24f5
+#define mmSPI_SHADER_TBA_LO_PS 0x2c00
+#define mmSPI_SHADER_TBA_HI_PS 0x2c01
+#define mmSPI_SHADER_TMA_LO_PS 0x2c02
+#define mmSPI_SHADER_TMA_HI_PS 0x2c03
+#define mmSPI_SHADER_PGM_LO_PS 0x2c08
+#define mmSPI_SHADER_PGM_HI_PS 0x2c09
+#define mmSPI_SHADER_PGM_RSRC1_PS 0x2c0a
+#define mmSPI_SHADER_PGM_RSRC2_PS 0x2c0b
+#define mmSPI_SHADER_PGM_RSRC3_PS 0x2c07
+#define mmSPI_SHADER_USER_DATA_PS_0 0x2c0c
+#define mmSPI_SHADER_USER_DATA_PS_1 0x2c0d
+#define mmSPI_SHADER_USER_DATA_PS_2 0x2c0e
+#define mmSPI_SHADER_USER_DATA_PS_3 0x2c0f
+#define mmSPI_SHADER_USER_DATA_PS_4 0x2c10
+#define mmSPI_SHADER_USER_DATA_PS_5 0x2c11
+#define mmSPI_SHADER_USER_DATA_PS_6 0x2c12
+#define mmSPI_SHADER_USER_DATA_PS_7 0x2c13
+#define mmSPI_SHADER_USER_DATA_PS_8 0x2c14
+#define mmSPI_SHADER_USER_DATA_PS_9 0x2c15
+#define mmSPI_SHADER_USER_DATA_PS_10 0x2c16
+#define mmSPI_SHADER_USER_DATA_PS_11 0x2c17
+#define mmSPI_SHADER_USER_DATA_PS_12 0x2c18
+#define mmSPI_SHADER_USER_DATA_PS_13 0x2c19
+#define mmSPI_SHADER_USER_DATA_PS_14 0x2c1a
+#define mmSPI_SHADER_USER_DATA_PS_15 0x2c1b
+#define mmSPI_SHADER_TBA_LO_VS 0x2c40
+#define mmSPI_SHADER_TBA_HI_VS 0x2c41
+#define mmSPI_SHADER_TMA_LO_VS 0x2c42
+#define mmSPI_SHADER_TMA_HI_VS 0x2c43
+#define mmSPI_SHADER_PGM_LO_VS 0x2c48
+#define mmSPI_SHADER_PGM_HI_VS 0x2c49
+#define mmSPI_SHADER_PGM_RSRC1_VS 0x2c4a
+#define mmSPI_SHADER_PGM_RSRC2_VS 0x2c4b
+#define mmSPI_SHADER_PGM_RSRC3_VS 0x2c46
+#define mmSPI_SHADER_LATE_ALLOC_VS 0x2c47
+#define mmSPI_SHADER_USER_DATA_VS_0 0x2c4c
+#define mmSPI_SHADER_USER_DATA_VS_1 0x2c4d
+#define mmSPI_SHADER_USER_DATA_VS_2 0x2c4e
+#define mmSPI_SHADER_USER_DATA_VS_3 0x2c4f
+#define mmSPI_SHADER_USER_DATA_VS_4 0x2c50
+#define mmSPI_SHADER_USER_DATA_VS_5 0x2c51
+#define mmSPI_SHADER_USER_DATA_VS_6 0x2c52
+#define mmSPI_SHADER_USER_DATA_VS_7 0x2c53
+#define mmSPI_SHADER_USER_DATA_VS_8 0x2c54
+#define mmSPI_SHADER_USER_DATA_VS_9 0x2c55
+#define mmSPI_SHADER_USER_DATA_VS_10 0x2c56
+#define mmSPI_SHADER_USER_DATA_VS_11 0x2c57
+#define mmSPI_SHADER_USER_DATA_VS_12 0x2c58
+#define mmSPI_SHADER_USER_DATA_VS_13 0x2c59
+#define mmSPI_SHADER_USER_DATA_VS_14 0x2c5a
+#define mmSPI_SHADER_USER_DATA_VS_15 0x2c5b
+#define mmSPI_SHADER_PGM_RSRC2_ES_VS 0x2c7c
+#define mmSPI_SHADER_PGM_RSRC2_LS_VS 0x2c7d
+#define mmSPI_SHADER_TBA_LO_GS 0x2c80
+#define mmSPI_SHADER_TBA_HI_GS 0x2c81
+#define mmSPI_SHADER_TMA_LO_GS 0x2c82
+#define mmSPI_SHADER_TMA_HI_GS 0x2c83
+#define mmSPI_SHADER_PGM_LO_GS 0x2c88
+#define mmSPI_SHADER_PGM_HI_GS 0x2c89
+#define mmSPI_SHADER_PGM_RSRC1_GS 0x2c8a
+#define mmSPI_SHADER_PGM_RSRC2_GS 0x2c8b
+#define mmSPI_SHADER_PGM_RSRC3_GS 0x2c87
+#define mmSPI_SHADER_USER_DATA_GS_0 0x2c8c
+#define mmSPI_SHADER_USER_DATA_GS_1 0x2c8d
+#define mmSPI_SHADER_USER_DATA_GS_2 0x2c8e
+#define mmSPI_SHADER_USER_DATA_GS_3 0x2c8f
+#define mmSPI_SHADER_USER_DATA_GS_4 0x2c90
+#define mmSPI_SHADER_USER_DATA_GS_5 0x2c91
+#define mmSPI_SHADER_USER_DATA_GS_6 0x2c92
+#define mmSPI_SHADER_USER_DATA_GS_7 0x2c93
+#define mmSPI_SHADER_USER_DATA_GS_8 0x2c94
+#define mmSPI_SHADER_USER_DATA_GS_9 0x2c95
+#define mmSPI_SHADER_USER_DATA_GS_10 0x2c96
+#define mmSPI_SHADER_USER_DATA_GS_11 0x2c97
+#define mmSPI_SHADER_USER_DATA_GS_12 0x2c98
+#define mmSPI_SHADER_USER_DATA_GS_13 0x2c99
+#define mmSPI_SHADER_USER_DATA_GS_14 0x2c9a
+#define mmSPI_SHADER_USER_DATA_GS_15 0x2c9b
+#define mmSPI_SHADER_PGM_RSRC2_ES_GS 0x2cbc
+#define mmSPI_SHADER_TBA_LO_ES 0x2cc0
+#define mmSPI_SHADER_TBA_HI_ES 0x2cc1
+#define mmSPI_SHADER_TMA_LO_ES 0x2cc2
+#define mmSPI_SHADER_TMA_HI_ES 0x2cc3
+#define mmSPI_SHADER_PGM_LO_ES 0x2cc8
+#define mmSPI_SHADER_PGM_HI_ES 0x2cc9
+#define mmSPI_SHADER_PGM_RSRC1_ES 0x2cca
+#define mmSPI_SHADER_PGM_RSRC2_ES 0x2ccb
+#define mmSPI_SHADER_PGM_RSRC3_ES 0x2cc7
+#define mmSPI_SHADER_USER_DATA_ES_0 0x2ccc
+#define mmSPI_SHADER_USER_DATA_ES_1 0x2ccd
+#define mmSPI_SHADER_USER_DATA_ES_2 0x2cce
+#define mmSPI_SHADER_USER_DATA_ES_3 0x2ccf
+#define mmSPI_SHADER_USER_DATA_ES_4 0x2cd0
+#define mmSPI_SHADER_USER_DATA_ES_5 0x2cd1
+#define mmSPI_SHADER_USER_DATA_ES_6 0x2cd2
+#define mmSPI_SHADER_USER_DATA_ES_7 0x2cd3
+#define mmSPI_SHADER_USER_DATA_ES_8 0x2cd4
+#define mmSPI_SHADER_USER_DATA_ES_9 0x2cd5
+#define mmSPI_SHADER_USER_DATA_ES_10 0x2cd6
+#define mmSPI_SHADER_USER_DATA_ES_11 0x2cd7
+#define mmSPI_SHADER_USER_DATA_ES_12 0x2cd8
+#define mmSPI_SHADER_USER_DATA_ES_13 0x2cd9
+#define mmSPI_SHADER_USER_DATA_ES_14 0x2cda
+#define mmSPI_SHADER_USER_DATA_ES_15 0x2cdb
+#define mmSPI_SHADER_PGM_RSRC2_LS_ES 0x2cfd
+#define mmSPI_SHADER_TBA_LO_HS 0x2d00
+#define mmSPI_SHADER_TBA_HI_HS 0x2d01
+#define mmSPI_SHADER_TMA_LO_HS 0x2d02
+#define mmSPI_SHADER_TMA_HI_HS 0x2d03
+#define mmSPI_SHADER_PGM_LO_HS 0x2d08
+#define mmSPI_SHADER_PGM_HI_HS 0x2d09
+#define mmSPI_SHADER_PGM_RSRC1_HS 0x2d0a
+#define mmSPI_SHADER_PGM_RSRC2_HS 0x2d0b
+#define mmSPI_SHADER_PGM_RSRC3_HS 0x2d07
+#define mmSPI_SHADER_USER_DATA_HS_0 0x2d0c
+#define mmSPI_SHADER_USER_DATA_HS_1 0x2d0d
+#define mmSPI_SHADER_USER_DATA_HS_2 0x2d0e
+#define mmSPI_SHADER_USER_DATA_HS_3 0x2d0f
+#define mmSPI_SHADER_USER_DATA_HS_4 0x2d10
+#define mmSPI_SHADER_USER_DATA_HS_5 0x2d11
+#define mmSPI_SHADER_USER_DATA_HS_6 0x2d12
+#define mmSPI_SHADER_USER_DATA_HS_7 0x2d13
+#define mmSPI_SHADER_USER_DATA_HS_8 0x2d14
+#define mmSPI_SHADER_USER_DATA_HS_9 0x2d15
+#define mmSPI_SHADER_USER_DATA_HS_10 0x2d16
+#define mmSPI_SHADER_USER_DATA_HS_11 0x2d17
+#define mmSPI_SHADER_USER_DATA_HS_12 0x2d18
+#define mmSPI_SHADER_USER_DATA_HS_13 0x2d19
+#define mmSPI_SHADER_USER_DATA_HS_14 0x2d1a
+#define mmSPI_SHADER_USER_DATA_HS_15 0x2d1b
+#define mmSPI_SHADER_PGM_RSRC2_LS_HS 0x2d3d
+#define mmSPI_SHADER_TBA_LO_LS 0x2d40
+#define mmSPI_SHADER_TBA_HI_LS 0x2d41
+#define mmSPI_SHADER_TMA_LO_LS 0x2d42
+#define mmSPI_SHADER_TMA_HI_LS 0x2d43
+#define mmSPI_SHADER_PGM_LO_LS 0x2d48
+#define mmSPI_SHADER_PGM_HI_LS 0x2d49
+#define mmSPI_SHADER_PGM_RSRC1_LS 0x2d4a
+#define mmSPI_SHADER_PGM_RSRC2_LS 0x2d4b
+#define mmSPI_SHADER_PGM_RSRC3_LS 0x2d47
+#define mmSPI_SHADER_USER_DATA_LS_0 0x2d4c
+#define mmSPI_SHADER_USER_DATA_LS_1 0x2d4d
+#define mmSPI_SHADER_USER_DATA_LS_2 0x2d4e
+#define mmSPI_SHADER_USER_DATA_LS_3 0x2d4f
+#define mmSPI_SHADER_USER_DATA_LS_4 0x2d50
+#define mmSPI_SHADER_USER_DATA_LS_5 0x2d51
+#define mmSPI_SHADER_USER_DATA_LS_6 0x2d52
+#define mmSPI_SHADER_USER_DATA_LS_7 0x2d53
+#define mmSPI_SHADER_USER_DATA_LS_8 0x2d54
+#define mmSPI_SHADER_USER_DATA_LS_9 0x2d55
+#define mmSPI_SHADER_USER_DATA_LS_10 0x2d56
+#define mmSPI_SHADER_USER_DATA_LS_11 0x2d57
+#define mmSPI_SHADER_USER_DATA_LS_12 0x2d58
+#define mmSPI_SHADER_USER_DATA_LS_13 0x2d59
+#define mmSPI_SHADER_USER_DATA_LS_14 0x2d5a
+#define mmSPI_SHADER_USER_DATA_LS_15 0x2d5b
+#define mmSQ_CONFIG 0x2300
+#define mmSQC_CONFIG 0x2301
+#define mmSQC_CACHES 0xc348
+#define mmSQC_WRITEBACK 0xc349
+#define mmSQC_DSM_CNTL 0x230f
+#define mmSQ_RANDOM_WAVE_PRI 0x2303
+#define mmSQ_REG_CREDITS 0x2304
+#define mmSQ_FIFO_SIZES 0x2305
+#define mmSQ_DSM_CNTL 0x2306
+#define mmCC_GC_SHADER_RATE_CONFIG 0x2312
+#define mmGC_USER_SHADER_RATE_CONFIG 0x2313
+#define mmSQ_INTERRUPT_AUTO_MASK 0x2314
+#define mmSQ_INTERRUPT_MSG_CTRL 0x2315
+#define mmSQ_PERFCOUNTER_CTRL 0xd9e0
+#define mmSQ_PERFCOUNTER_MASK 0xd9e1
+#define mmSQ_PERFCOUNTER_CTRL2 0xd9e2
+#define mmCC_SQC_BANK_DISABLE 0x2307
+#define mmUSER_SQC_BANK_DISABLE 0x2308
+#define mmSQ_PERFCOUNTER0_LO 0xd1c0
+#define mmSQ_PERFCOUNTER1_LO 0xd1c2
+#define mmSQ_PERFCOUNTER2_LO 0xd1c4
+#define mmSQ_PERFCOUNTER3_LO 0xd1c6
+#define mmSQ_PERFCOUNTER4_LO 0xd1c8
+#define mmSQ_PERFCOUNTER5_LO 0xd1ca
+#define mmSQ_PERFCOUNTER6_LO 0xd1cc
+#define mmSQ_PERFCOUNTER7_LO 0xd1ce
+#define mmSQ_PERFCOUNTER8_LO 0xd1d0
+#define mmSQ_PERFCOUNTER9_LO 0xd1d2
+#define mmSQ_PERFCOUNTER10_LO 0xd1d4
+#define mmSQ_PERFCOUNTER11_LO 0xd1d6
+#define mmSQ_PERFCOUNTER12_LO 0xd1d8
+#define mmSQ_PERFCOUNTER13_LO 0xd1da
+#define mmSQ_PERFCOUNTER14_LO 0xd1dc
+#define mmSQ_PERFCOUNTER15_LO 0xd1de
+#define mmSQ_PERFCOUNTER0_HI 0xd1c1
+#define mmSQ_PERFCOUNTER1_HI 0xd1c3
+#define mmSQ_PERFCOUNTER2_HI 0xd1c5
+#define mmSQ_PERFCOUNTER3_HI 0xd1c7
+#define mmSQ_PERFCOUNTER4_HI 0xd1c9
+#define mmSQ_PERFCOUNTER5_HI 0xd1cb
+#define mmSQ_PERFCOUNTER6_HI 0xd1cd
+#define mmSQ_PERFCOUNTER7_HI 0xd1cf
+#define mmSQ_PERFCOUNTER8_HI 0xd1d1
+#define mmSQ_PERFCOUNTER9_HI 0xd1d3
+#define mmSQ_PERFCOUNTER10_HI 0xd1d5
+#define mmSQ_PERFCOUNTER11_HI 0xd1d7
+#define mmSQ_PERFCOUNTER12_HI 0xd1d9
+#define mmSQ_PERFCOUNTER13_HI 0xd1db
+#define mmSQ_PERFCOUNTER14_HI 0xd1dd
+#define mmSQ_PERFCOUNTER15_HI 0xd1df
+#define mmSQ_PERFCOUNTER0_SELECT 0xd9c0
+#define mmSQ_PERFCOUNTER1_SELECT 0xd9c1
+#define mmSQ_PERFCOUNTER2_SELECT 0xd9c2
+#define mmSQ_PERFCOUNTER3_SELECT 0xd9c3
+#define mmSQ_PERFCOUNTER4_SELECT 0xd9c4
+#define mmSQ_PERFCOUNTER5_SELECT 0xd9c5
+#define mmSQ_PERFCOUNTER6_SELECT 0xd9c6
+#define mmSQ_PERFCOUNTER7_SELECT 0xd9c7
+#define mmSQ_PERFCOUNTER8_SELECT 0xd9c8
+#define mmSQ_PERFCOUNTER9_SELECT 0xd9c9
+#define mmSQ_PERFCOUNTER10_SELECT 0xd9ca
+#define mmSQ_PERFCOUNTER11_SELECT 0xd9cb
+#define mmSQ_PERFCOUNTER12_SELECT 0xd9cc
+#define mmSQ_PERFCOUNTER13_SELECT 0xd9cd
+#define mmSQ_PERFCOUNTER14_SELECT 0xd9ce
+#define mmSQ_PERFCOUNTER15_SELECT 0xd9cf
+#define mmCGTT_SQ_CLK_CTRL 0xf08c
+#define mmCGTT_SQG_CLK_CTRL 0xf08d
+#define mmSQ_ALU_CLK_CTRL 0xf08e
+#define mmSQ_TEX_CLK_CTRL 0xf08f
+#define mmSQ_LDS_CLK_CTRL 0xf090
+#define mmSQ_POWER_THROTTLE 0xf091
+#define mmSQ_POWER_THROTTLE2 0xf092
+#define mmSQ_TIME_HI 0x237c
+#define mmSQ_TIME_LO 0x237d
+#define mmSQ_THREAD_TRACE_BASE 0xc330
+#define mmSQ_THREAD_TRACE_BASE2 0xc337
+#define mmSQ_THREAD_TRACE_SIZE 0xc331
+#define mmSQ_THREAD_TRACE_MASK 0xc332
+#define mmSQ_THREAD_TRACE_USERDATA_0 0xc340
+#define mmSQ_THREAD_TRACE_USERDATA_1 0xc341
+#define mmSQ_THREAD_TRACE_USERDATA_2 0xc342
+#define mmSQ_THREAD_TRACE_USERDATA_3 0xc343
+#define mmSQ_THREAD_TRACE_MODE 0xc336
+#define mmSQ_THREAD_TRACE_CTRL 0xc335
+#define mmSQ_THREAD_TRACE_TOKEN_MASK 0xc333
+#define mmSQ_THREAD_TRACE_TOKEN_MASK2 0xc338
+#define mmSQ_THREAD_TRACE_PERF_MASK 0xc334
+#define mmSQ_THREAD_TRACE_WPTR 0xc339
+#define mmSQ_THREAD_TRACE_STATUS 0xc33a
+#define mmSQ_THREAD_TRACE_CNTR 0x2390
+#define mmSQ_THREAD_TRACE_HIWATER 0xc33b
+#define mmSQ_LB_CTR_CTRL 0x2398
+#define mmSQ_LB_DATA_ALU_CYCLES 0x2399
+#define mmSQ_LB_DATA_TEX_CYCLES 0x239a
+#define mmSQ_LB_DATA_ALU_STALLS 0x239b
+#define mmSQ_LB_DATA_TEX_STALLS 0x239c
+#define mmSQC_EDC_CNT 0x23a0
+#define mmSQ_EDC_SEC_CNT 0x23a1
+#define mmSQ_EDC_DED_CNT 0x23a2
+#define mmSQ_EDC_INFO 0x23a3
+#define mmSQ_BUF_RSRC_WORD0 0x23c0
+#define mmSQ_BUF_RSRC_WORD1 0x23c1
+#define mmSQ_BUF_RSRC_WORD2 0x23c2
+#define mmSQ_BUF_RSRC_WORD3 0x23c3
+#define mmSQ_IMG_RSRC_WORD0 0x23c4
+#define mmSQ_IMG_RSRC_WORD1 0x23c5
+#define mmSQ_IMG_RSRC_WORD2 0x23c6
+#define mmSQ_IMG_RSRC_WORD3 0x23c7
+#define mmSQ_IMG_RSRC_WORD4 0x23c8
+#define mmSQ_IMG_RSRC_WORD5 0x23c9
+#define mmSQ_IMG_RSRC_WORD6 0x23ca
+#define mmSQ_IMG_RSRC_WORD7 0x23cb
+#define mmSQ_IMG_SAMP_WORD0 0x23cc
+#define mmSQ_IMG_SAMP_WORD1 0x23cd
+#define mmSQ_IMG_SAMP_WORD2 0x23ce
+#define mmSQ_IMG_SAMP_WORD3 0x23cf
+#define mmSQ_FLAT_SCRATCH_WORD0 0x23d0
+#define mmSQ_FLAT_SCRATCH_WORD1 0x23d1
+#define mmSQ_M0_GPR_IDX_WORD 0x23d2
+#define mmSQ_IND_INDEX 0x2378
+#define mmSQ_CMD 0x237b
+#define mmSQ_IND_DATA 0x2379
+#define mmSQ_REG_TIMESTAMP 0x2374
+#define mmSQ_CMD_TIMESTAMP 0x2375
+#define mmSQ_HV_VMID_CTRL 0xf840
+#define ixSQ_WAVE_INST_DW0 0x1a
+#define ixSQ_WAVE_INST_DW1 0x1b
+#define ixSQ_WAVE_PC_LO 0x18
+#define ixSQ_WAVE_PC_HI 0x19
+#define ixSQ_WAVE_IB_DBG0 0x1c
+#define ixSQ_WAVE_IB_DBG1 0x1d
+#define ixSQ_WAVE_EXEC_LO 0x27e
+#define ixSQ_WAVE_EXEC_HI 0x27f
+#define ixSQ_WAVE_STATUS 0x12
+#define ixSQ_WAVE_MODE 0x11
+#define ixSQ_WAVE_TRAPSTS 0x13
+#define ixSQ_WAVE_HW_ID 0x14
+#define ixSQ_WAVE_GPR_ALLOC 0x15
+#define ixSQ_WAVE_LDS_ALLOC 0x16
+#define ixSQ_WAVE_IB_STS 0x17
+#define ixSQ_WAVE_M0 0x27c
+#define ixSQ_WAVE_TBA_LO 0x26c
+#define ixSQ_WAVE_TBA_HI 0x26d
+#define ixSQ_WAVE_TMA_LO 0x26e
+#define ixSQ_WAVE_TMA_HI 0x26f
+#define ixSQ_WAVE_TTMP0 0x270
+#define ixSQ_WAVE_TTMP1 0x271
+#define ixSQ_WAVE_TTMP2 0x272
+#define ixSQ_WAVE_TTMP3 0x273
+#define ixSQ_WAVE_TTMP4 0x274
+#define ixSQ_WAVE_TTMP5 0x275
+#define ixSQ_WAVE_TTMP6 0x276
+#define ixSQ_WAVE_TTMP7 0x277
+#define ixSQ_WAVE_TTMP8 0x278
+#define ixSQ_WAVE_TTMP9 0x279
+#define ixSQ_WAVE_TTMP10 0x27a
+#define ixSQ_WAVE_TTMP11 0x27b
+#define mmSQ_DEBUG_STS_GLOBAL 0x2309
+#define mmSQ_DEBUG_STS_GLOBAL2 0x2310
+#define mmSQ_DEBUG_STS_GLOBAL3 0x2311
+#define ixSQ_DEBUG_STS_LOCAL 0x8
+#define ixSQ_DEBUG_CTRL_LOCAL 0x9
+#define mmSH_MEM_BASES 0x230a
+#define mmSH_MEM_APE1_BASE 0x230b
+#define mmSH_MEM_APE1_LIMIT 0x230c
+#define mmSH_MEM_CONFIG 0x230d
+#define mmSQ_THREAD_TRACE_WORD_CMN 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_INST 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_INST_PC_1_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_INST_PC_2_OF_2 0x23b1
+#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2 0x23b1
+#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2 0x23b1
+#define mmSQ_THREAD_TRACE_WORD_WAVE 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_MISC 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_WAVE_START 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_REG_1_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_REG_2_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_REG_CS_1_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_REG_CS_2_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_EVENT 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_ISSUE 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_PERF_1_OF_2 0x23b0
+#define mmSQ_THREAD_TRACE_WORD_PERF_2_OF_2 0x23b1
+#define mmSQ_WREXEC_EXEC_LO 0x23b1
+#define mmSQ_WREXEC_EXEC_HI 0x23b1
+#define mmSQC_GATCL1_CNTL 0x23b2
+#define mmSQC_ATC_EDC_GATCL1_CNT 0x23b3
+#define ixSQ_INTERRUPT_WORD_CMN 0x20c0
+#define ixSQ_INTERRUPT_WORD_AUTO 0x20c0
+#define ixSQ_INTERRUPT_WORD_WAVE 0x20c0
+#define mmSQ_SOP2 0x237f
+#define mmSQ_VOP1 0x237f
+#define mmSQ_MTBUF_1 0x237f
+#define mmSQ_EXP_1 0x237f
+#define mmSQ_MUBUF_1 0x237f
+#define mmSQ_SMEM_1 0x237f
+#define mmSQ_INST 0x237f
+#define mmSQ_EXP_0 0x237f
+#define mmSQ_MUBUF_0 0x237f
+#define mmSQ_VOP_SDWA 0x237f
+#define mmSQ_VOP3_0 0x237f
+#define mmSQ_VOP2 0x237f
+#define mmSQ_MTBUF_0 0x237f
+#define mmSQ_SOPP 0x237f
+#define mmSQ_FLAT_0 0x237f
+#define mmSQ_VOP3_0_SDST_ENC 0x237f
+#define mmSQ_MIMG_1 0x237f
+#define mmSQ_SOP1 0x237f
+#define mmSQ_SOPC 0x237f
+#define mmSQ_FLAT_1 0x237f
+#define mmSQ_DS_1 0x237f
+#define mmSQ_VOP3_1 0x237f
+#define mmSQ_SMEM_0 0x237f
+#define mmSQ_MIMG_0 0x237f
+#define mmSQ_SOPK 0x237f
+#define mmSQ_DS_0 0x237f
+#define mmSQ_VOP_DPP 0x237f
+#define mmSQ_VOPC 0x237f
+#define mmSQ_VINTRP 0x237f
+#define mmCGTT_SX_CLK_CTRL0 0xf094
+#define mmCGTT_SX_CLK_CTRL1 0xf095
+#define mmCGTT_SX_CLK_CTRL2 0xf096
+#define mmCGTT_SX_CLK_CTRL3 0xf097
+#define mmCGTT_SX_CLK_CTRL4 0xf098
+#define mmSX_DEBUG_BUSY 0x2414
+#define mmSX_DEBUG_BUSY_2 0x2415
+#define mmSX_DEBUG_BUSY_3 0x2416
+#define mmSX_DEBUG_BUSY_4 0x2417
+#define mmSX_DEBUG_1 0x2418
+#define mmSX_PERFCOUNTER0_SELECT 0xda40
+#define mmSX_PERFCOUNTER1_SELECT 0xda41
+#define mmSX_PERFCOUNTER2_SELECT 0xda42
+#define mmSX_PERFCOUNTER3_SELECT 0xda43
+#define mmSX_PERFCOUNTER0_SELECT1 0xda44
+#define mmSX_PERFCOUNTER1_SELECT1 0xda45
+#define mmSX_PERFCOUNTER0_LO 0xd240
+#define mmSX_PERFCOUNTER0_HI 0xd241
+#define mmSX_PERFCOUNTER1_LO 0xd242
+#define mmSX_PERFCOUNTER1_HI 0xd243
+#define mmSX_PERFCOUNTER2_LO 0xd244
+#define mmSX_PERFCOUNTER2_HI 0xd245
+#define mmSX_PERFCOUNTER3_LO 0xd246
+#define mmSX_PERFCOUNTER3_HI 0xd247
+#define mmSX_PS_DOWNCONVERT 0xa1d5
+#define mmSX_BLEND_OPT_EPSILON 0xa1d6
+#define mmSX_BLEND_OPT_CONTROL 0xa1d7
+#define mmSX_MRT0_BLEND_OPT 0xa1d8
+#define mmSX_MRT1_BLEND_OPT 0xa1d9
+#define mmSX_MRT2_BLEND_OPT 0xa1da
+#define mmSX_MRT3_BLEND_OPT 0xa1db
+#define mmSX_MRT4_BLEND_OPT 0xa1dc
+#define mmSX_MRT5_BLEND_OPT 0xa1dd
+#define mmSX_MRT6_BLEND_OPT 0xa1de
+#define mmSX_MRT7_BLEND_OPT 0xa1df
+#define mmTCC_CTRL 0x2b80
+#define mmTCC_EDC_CNT 0x2b82
+#define mmTCC_REDUNDANCY 0x2b83
+#define mmTCC_EXE_DISABLE 0x2b84
+#define mmTCC_DSM_CNTL 0x2b85
+#define mmTCC_CGTT_SCLK_CTRL 0xf0ac
+#define mmTCA_CGTT_SCLK_CTRL 0xf0ad
+#define mmTCC_PERFCOUNTER0_SELECT 0xdb80
+#define mmTCC_PERFCOUNTER1_SELECT 0xdb82
+#define mmTCC_PERFCOUNTER0_SELECT1 0xdb81
+#define mmTCC_PERFCOUNTER1_SELECT1 0xdb83
+#define mmTCC_PERFCOUNTER2_SELECT 0xdb84
+#define mmTCC_PERFCOUNTER3_SELECT 0xdb85
+#define mmTCC_PERFCOUNTER0_LO 0xd380
+#define mmTCC_PERFCOUNTER1_LO 0xd382
+#define mmTCC_PERFCOUNTER2_LO 0xd384
+#define mmTCC_PERFCOUNTER3_LO 0xd386
+#define mmTCC_PERFCOUNTER0_HI 0xd381
+#define mmTCC_PERFCOUNTER1_HI 0xd383
+#define mmTCC_PERFCOUNTER2_HI 0xd385
+#define mmTCC_PERFCOUNTER3_HI 0xd387
+#define mmTCA_CTRL 0x2bc0
+#define mmTCA_PERFCOUNTER0_SELECT 0xdb90
+#define mmTCA_PERFCOUNTER1_SELECT 0xdb92
+#define mmTCA_PERFCOUNTER0_SELECT1 0xdb91
+#define mmTCA_PERFCOUNTER1_SELECT1 0xdb93
+#define mmTCA_PERFCOUNTER2_SELECT 0xdb94
+#define mmTCA_PERFCOUNTER3_SELECT 0xdb95
+#define mmTCA_PERFCOUNTER0_LO 0xd390
+#define mmTCA_PERFCOUNTER1_LO 0xd392
+#define mmTCA_PERFCOUNTER2_LO 0xd394
+#define mmTCA_PERFCOUNTER3_LO 0xd396
+#define mmTCA_PERFCOUNTER0_HI 0xd391
+#define mmTCA_PERFCOUNTER1_HI 0xd393
+#define mmTCA_PERFCOUNTER2_HI 0xd395
+#define mmTCA_PERFCOUNTER3_HI 0xd397
+#define mmTA_BC_BASE_ADDR 0xa020
+#define mmTA_BC_BASE_ADDR_HI 0xa021
+#define mmTD_CNTL 0x2525
+#define mmTD_STATUS 0x2526
+#define mmTD_DEBUG_INDEX 0x2528
+#define mmTD_DEBUG_DATA 0x2529
+#define mmTD_DSM_CNTL 0x252f
+#define mmTD_PERFCOUNTER0_SELECT 0xdb00
+#define mmTD_PERFCOUNTER1_SELECT 0xdb02
+#define mmTD_PERFCOUNTER0_SELECT1 0xdb01
+#define mmTD_PERFCOUNTER0_LO 0xd300
+#define mmTD_PERFCOUNTER1_LO 0xd302
+#define mmTD_PERFCOUNTER0_HI 0xd301
+#define mmTD_PERFCOUNTER1_HI 0xd303
+#define mmTD_SCRATCH 0x2533
+#define mmTA_CNTL 0x2541
+#define mmTA_CNTL_AUX 0x2542
+#define mmTA_RESERVED_010C 0x2543
+#define mmTA_CS_BC_BASE_ADDR 0xc380
+#define mmTA_CS_BC_BASE_ADDR_HI 0xc381
+#define mmTA_STATUS 0x2548
+#define mmTA_DEBUG_INDEX 0x254c
+#define mmTA_DEBUG_DATA 0x254d
+#define mmTA_PERFCOUNTER0_SELECT 0xdac0
+#define mmTA_PERFCOUNTER1_SELECT 0xdac2
+#define mmTA_PERFCOUNTER0_SELECT1 0xdac1
+#define mmTA_PERFCOUNTER0_LO 0xd2c0
+#define mmTA_PERFCOUNTER1_LO 0xd2c2
+#define mmTA_PERFCOUNTER0_HI 0xd2c1
+#define mmTA_PERFCOUNTER1_HI 0xd2c3
+#define mmTA_SCRATCH 0x2564
+#define mmSH_HIDDEN_PRIVATE_BASE_VMID 0x2580
+#define mmSH_STATIC_MEM_CONFIG 0x2581
+#define mmTCP_INVALIDATE 0x2b00
+#define mmTCP_STATUS 0x2b01
+#define mmTCP_CNTL 0x2b02
+#define mmTCP_CHAN_STEER_LO 0x2b03
+#define mmTCP_CHAN_STEER_HI 0x2b04
+#define mmTCP_ADDR_CONFIG 0x2b05
+#define mmTCP_CREDIT 0x2b06
+#define mmTCP_PERFCOUNTER0_SELECT 0xdb40
+#define mmTCP_PERFCOUNTER1_SELECT 0xdb42
+#define mmTCP_PERFCOUNTER0_SELECT1 0xdb41
+#define mmTCP_PERFCOUNTER1_SELECT1 0xdb43
+#define mmTCP_PERFCOUNTER2_SELECT 0xdb44
+#define mmTCP_PERFCOUNTER3_SELECT 0xdb45
+#define mmTCP_PERFCOUNTER0_LO 0xd340
+#define mmTCP_PERFCOUNTER1_LO 0xd342
+#define mmTCP_PERFCOUNTER2_LO 0xd344
+#define mmTCP_PERFCOUNTER3_LO 0xd346
+#define mmTCP_PERFCOUNTER0_HI 0xd341
+#define mmTCP_PERFCOUNTER1_HI 0xd343
+#define mmTCP_PERFCOUNTER2_HI 0xd345
+#define mmTCP_PERFCOUNTER3_HI 0xd347
+#define mmTCP_BUFFER_ADDR_HASH_CNTL 0x2b16
+#define mmTCP_EDC_CNT 0x2b17
+#define mmTC_CFG_L1_LOAD_POLICY0 0x2b1a
+#define mmTC_CFG_L1_LOAD_POLICY1 0x2b1b
+#define mmTC_CFG_L1_STORE_POLICY 0x2b1c
+#define mmTC_CFG_L2_LOAD_POLICY0 0x2b1d
+#define mmTC_CFG_L2_LOAD_POLICY1 0x2b1e
+#define mmTC_CFG_L2_STORE_POLICY0 0x2b1f
+#define mmTC_CFG_L2_STORE_POLICY1 0x2b20
+#define mmTC_CFG_L2_ATOMIC_POLICY 0x2b21
+#define mmTC_CFG_L1_VOLATILE 0x2b22
+#define mmTC_CFG_L2_VOLATILE 0x2b23
+#define mmTCP_WATCH0_ADDR_H 0x32a0
+#define mmTCP_WATCH1_ADDR_H 0x32a3
+#define mmTCP_WATCH2_ADDR_H 0x32a6
+#define mmTCP_WATCH3_ADDR_H 0x32a9
+#define mmTCP_WATCH0_ADDR_L 0x32a1
+#define mmTCP_WATCH1_ADDR_L 0x32a4
+#define mmTCP_WATCH2_ADDR_L 0x32a7
+#define mmTCP_WATCH3_ADDR_L 0x32aa
+#define mmTCP_WATCH0_CNTL 0x32a2
+#define mmTCP_WATCH1_CNTL 0x32a5
+#define mmTCP_WATCH2_CNTL 0x32a8
+#define mmTCP_WATCH3_CNTL 0x32ab
+#define mmTCP_GATCL1_CNTL 0x32b0
+#define mmTCP_ATC_EDC_GATCL1_CNT 0x32b1
+#define mmTCP_GATCL1_DSM_CNTL 0x32b2
+#define mmTCP_DSM_CNTL 0x32b3
+#define mmTCP_CNTL2 0x32b4
+#define mmTD_CGTT_CTRL 0xf09c
+#define mmTA_CGTT_CTRL 0xf09d
+#define mmCGTT_TCP_CLK_CTRL 0xf09e
+#define mmCGTT_TCI_CLK_CTRL 0xf09f
+#define mmTCI_STATUS 0x2b61
+#define mmTCI_CNTL_1 0x2b62
+#define mmTCI_CNTL_2 0x2b63
+#define mmGDS_CONFIG 0x25c0
+#define mmGDS_CNTL_STATUS 0x25c1
+#define mmGDS_ENHANCE2 0x25c2
+#define mmGDS_PROTECTION_FAULT 0x25c3
+#define mmGDS_VM_PROTECTION_FAULT 0x25c4
+#define mmGDS_EDC_CNT 0x25c5
+#define mmGDS_EDC_GRBM_CNT 0x25c6
+#define mmGDS_EDC_OA_DED 0x25c7
+#define mmGDS_DEBUG_CNTL 0x25c8
+#define mmGDS_DEBUG_DATA 0x25c9
+#define mmGDS_DSM_CNTL 0x25ca
+#define mmCGTT_GDS_CLK_CTRL 0xf0a0
+#define mmGDS_RD_ADDR 0xc400
+#define mmGDS_RD_DATA 0xc401
+#define mmGDS_RD_BURST_ADDR 0xc402
+#define mmGDS_RD_BURST_COUNT 0xc403
+#define mmGDS_RD_BURST_DATA 0xc404
+#define mmGDS_WR_ADDR 0xc405
+#define mmGDS_WR_DATA 0xc406
+#define mmGDS_WR_BURST_ADDR 0xc407
+#define mmGDS_WR_BURST_DATA 0xc408
+#define mmGDS_WRITE_COMPLETE 0xc409
+#define mmGDS_ATOM_CNTL 0xc40a
+#define mmGDS_ATOM_COMPLETE 0xc40b
+#define mmGDS_ATOM_BASE 0xc40c
+#define mmGDS_ATOM_SIZE 0xc40d
+#define mmGDS_ATOM_OFFSET0 0xc40e
+#define mmGDS_ATOM_OFFSET1 0xc40f
+#define mmGDS_ATOM_DST 0xc410
+#define mmGDS_ATOM_OP 0xc411
+#define mmGDS_ATOM_SRC0 0xc412
+#define mmGDS_ATOM_SRC0_U 0xc413
+#define mmGDS_ATOM_SRC1 0xc414
+#define mmGDS_ATOM_SRC1_U 0xc415
+#define mmGDS_ATOM_READ0 0xc416
+#define mmGDS_ATOM_READ0_U 0xc417
+#define mmGDS_ATOM_READ1 0xc418
+#define mmGDS_ATOM_READ1_U 0xc419
+#define mmGDS_GWS_RESOURCE_CNTL 0xc41a
+#define mmGDS_GWS_RESOURCE 0xc41b
+#define mmGDS_GWS_RESOURCE_CNT 0xc41c
+#define mmGDS_OA_CNTL 0xc41d
+#define mmGDS_OA_COUNTER 0xc41e
+#define mmGDS_OA_ADDRESS 0xc41f
+#define mmGDS_OA_INCDEC 0xc420
+#define mmGDS_OA_RING_SIZE 0xc421
+#define ixGDS_DEBUG_REG0 0x0
+#define ixGDS_DEBUG_REG1 0x1
+#define ixGDS_DEBUG_REG2 0x2
+#define ixGDS_DEBUG_REG3 0x3
+#define ixGDS_DEBUG_REG4 0x4
+#define ixGDS_DEBUG_REG5 0x5
+#define ixGDS_DEBUG_REG6 0x6
+#define mmGDS_PERFCOUNTER0_SELECT 0xda80
+#define mmGDS_PERFCOUNTER1_SELECT 0xda81
+#define mmGDS_PERFCOUNTER2_SELECT 0xda82
+#define mmGDS_PERFCOUNTER3_SELECT 0xda83
+#define mmGDS_PERFCOUNTER0_LO 0xd280
+#define mmGDS_PERFCOUNTER1_LO 0xd282
+#define mmGDS_PERFCOUNTER2_LO 0xd284
+#define mmGDS_PERFCOUNTER3_LO 0xd286
+#define mmGDS_PERFCOUNTER0_HI 0xd281
+#define mmGDS_PERFCOUNTER1_HI 0xd283
+#define mmGDS_PERFCOUNTER2_HI 0xd285
+#define mmGDS_PERFCOUNTER3_HI 0xd287
+#define mmGDS_PERFCOUNTER0_SELECT1 0xda84
+#define mmGDS_VMID0_BASE 0x3300
+#define mmGDS_VMID1_BASE 0x3302
+#define mmGDS_VMID2_BASE 0x3304
+#define mmGDS_VMID3_BASE 0x3306
+#define mmGDS_VMID4_BASE 0x3308
+#define mmGDS_VMID5_BASE 0x330a
+#define mmGDS_VMID6_BASE 0x330c
+#define mmGDS_VMID7_BASE 0x330e
+#define mmGDS_VMID8_BASE 0x3310
+#define mmGDS_VMID9_BASE 0x3312
+#define mmGDS_VMID10_BASE 0x3314
+#define mmGDS_VMID11_BASE 0x3316
+#define mmGDS_VMID12_BASE 0x3318
+#define mmGDS_VMID13_BASE 0x331a
+#define mmGDS_VMID14_BASE 0x331c
+#define mmGDS_VMID15_BASE 0x331e
+#define mmGDS_VMID0_SIZE 0x3301
+#define mmGDS_VMID1_SIZE 0x3303
+#define mmGDS_VMID2_SIZE 0x3305
+#define mmGDS_VMID3_SIZE 0x3307
+#define mmGDS_VMID4_SIZE 0x3309
+#define mmGDS_VMID5_SIZE 0x330b
+#define mmGDS_VMID6_SIZE 0x330d
+#define mmGDS_VMID7_SIZE 0x330f
+#define mmGDS_VMID8_SIZE 0x3311
+#define mmGDS_VMID9_SIZE 0x3313
+#define mmGDS_VMID10_SIZE 0x3315
+#define mmGDS_VMID11_SIZE 0x3317
+#define mmGDS_VMID12_SIZE 0x3319
+#define mmGDS_VMID13_SIZE 0x331b
+#define mmGDS_VMID14_SIZE 0x331d
+#define mmGDS_VMID15_SIZE 0x331f
+#define mmGDS_GWS_VMID0 0x3320
+#define mmGDS_GWS_VMID1 0x3321
+#define mmGDS_GWS_VMID2 0x3322
+#define mmGDS_GWS_VMID3 0x3323
+#define mmGDS_GWS_VMID4 0x3324
+#define mmGDS_GWS_VMID5 0x3325
+#define mmGDS_GWS_VMID6 0x3326
+#define mmGDS_GWS_VMID7 0x3327
+#define mmGDS_GWS_VMID8 0x3328
+#define mmGDS_GWS_VMID9 0x3329
+#define mmGDS_GWS_VMID10 0x332a
+#define mmGDS_GWS_VMID11 0x332b
+#define mmGDS_GWS_VMID12 0x332c
+#define mmGDS_GWS_VMID13 0x332d
+#define mmGDS_GWS_VMID14 0x332e
+#define mmGDS_GWS_VMID15 0x332f
+#define mmGDS_OA_VMID0 0x3330
+#define mmGDS_OA_VMID1 0x3331
+#define mmGDS_OA_VMID2 0x3332
+#define mmGDS_OA_VMID3 0x3333
+#define mmGDS_OA_VMID4 0x3334
+#define mmGDS_OA_VMID5 0x3335
+#define mmGDS_OA_VMID6 0x3336
+#define mmGDS_OA_VMID7 0x3337
+#define mmGDS_OA_VMID8 0x3338
+#define mmGDS_OA_VMID9 0x3339
+#define mmGDS_OA_VMID10 0x333a
+#define mmGDS_OA_VMID11 0x333b
+#define mmGDS_OA_VMID12 0x333c
+#define mmGDS_OA_VMID13 0x333d
+#define mmGDS_OA_VMID14 0x333e
+#define mmGDS_OA_VMID15 0x333f
+#define mmGDS_GWS_RESET0 0x3344
+#define mmGDS_GWS_RESET1 0x3345
+#define mmGDS_GWS_RESOURCE_RESET 0x3346
+#define mmGDS_COMPUTE_MAX_WAVE_ID 0x3348
+#define mmGDS_OA_RESET_MASK 0x3349
+#define mmGDS_OA_RESET 0x334a
+#define mmGDS_ENHANCE 0x334b
+#define mmGDS_OA_CGPG_RESTORE 0x334c
+#define mmGDS_CS_CTXSW_STATUS 0x334d
+#define mmGDS_CS_CTXSW_CNT0 0x334e
+#define mmGDS_CS_CTXSW_CNT1 0x334f
+#define mmGDS_CS_CTXSW_CNT2 0x3350
+#define mmGDS_CS_CTXSW_CNT3 0x3351
+#define mmGDS_GFX_CTXSW_STATUS 0x3352
+#define mmGDS_VS_CTXSW_CNT0 0x3353
+#define mmGDS_VS_CTXSW_CNT1 0x3354
+#define mmGDS_VS_CTXSW_CNT2 0x3355
+#define mmGDS_VS_CTXSW_CNT3 0x3356
+#define mmGDS_PS0_CTXSW_CNT0 0x3357
+#define mmGDS_PS1_CTXSW_CNT0 0x335b
+#define mmGDS_PS2_CTXSW_CNT0 0x335f
+#define mmGDS_PS3_CTXSW_CNT0 0x3363
+#define mmGDS_PS4_CTXSW_CNT0 0x3367
+#define mmGDS_PS5_CTXSW_CNT0 0x336b
+#define mmGDS_PS6_CTXSW_CNT0 0x336f
+#define mmGDS_PS7_CTXSW_CNT0 0x3373
+#define mmGDS_PS0_CTXSW_CNT1 0x3358
+#define mmGDS_PS1_CTXSW_CNT1 0x335c
+#define mmGDS_PS2_CTXSW_CNT1 0x3360
+#define mmGDS_PS3_CTXSW_CNT1 0x3364
+#define mmGDS_PS4_CTXSW_CNT1 0x3368
+#define mmGDS_PS5_CTXSW_CNT1 0x336c
+#define mmGDS_PS6_CTXSW_CNT1 0x3370
+#define mmGDS_PS7_CTXSW_CNT1 0x3374
+#define mmGDS_PS0_CTXSW_CNT2 0x3359
+#define mmGDS_PS1_CTXSW_CNT2 0x335d
+#define mmGDS_PS2_CTXSW_CNT2 0x3361
+#define mmGDS_PS3_CTXSW_CNT2 0x3365
+#define mmGDS_PS4_CTXSW_CNT2 0x3369
+#define mmGDS_PS5_CTXSW_CNT2 0x336d
+#define mmGDS_PS6_CTXSW_CNT2 0x3371
+#define mmGDS_PS7_CTXSW_CNT2 0x3375
+#define mmGDS_PS0_CTXSW_CNT3 0x335a
+#define mmGDS_PS1_CTXSW_CNT3 0x335e
+#define mmGDS_PS2_CTXSW_CNT3 0x3362
+#define mmGDS_PS3_CTXSW_CNT3 0x3366
+#define mmGDS_PS4_CTXSW_CNT3 0x336a
+#define mmGDS_PS5_CTXSW_CNT3 0x336e
+#define mmGDS_PS6_CTXSW_CNT3 0x3372
+#define mmGDS_PS7_CTXSW_CNT3 0x3376
+#define mmCS_COPY_STATE 0xa1f3
+#define mmGFX_COPY_STATE 0xa1f4
+#define mmVGT_DRAW_INITIATOR 0xa1fc
+#define mmVGT_EVENT_INITIATOR 0xa2a4
+#define mmVGT_EVENT_ADDRESS_REG 0xa1fe
+#define mmVGT_DMA_BASE_HI 0xa1f9
+#define mmVGT_DMA_BASE 0xa1fa
+#define mmVGT_DMA_INDEX_TYPE 0xa29f
+#define mmVGT_DMA_NUM_INSTANCES 0xa2a2
+#define mmIA_ENHANCE 0xa29c
+#define mmVGT_DMA_SIZE 0xa29d
+#define mmVGT_DMA_MAX_SIZE 0xa29e
+#define mmVGT_DMA_PRIMITIVE_TYPE 0x2271
+#define mmVGT_DMA_CONTROL 0x2272
+#define mmVGT_IMMED_DATA 0xa1fd
+#define mmVGT_INDEX_TYPE 0xc243
+#define mmVGT_NUM_INDICES 0xc24c
+#define mmVGT_NUM_INSTANCES 0xc24d
+#define mmVGT_PRIMITIVE_TYPE 0xc242
+#define mmVGT_PRIMITIVEID_EN 0xa2a1
+#define mmVGT_PRIMITIVEID_RESET 0xa2a3
+#define mmVGT_VTX_CNT_EN 0xa2ae
+#define mmVGT_REUSE_OFF 0xa2ad
+#define mmVGT_INSTANCE_STEP_RATE_0 0xa2a8
+#define mmVGT_INSTANCE_STEP_RATE_1 0xa2a9
+#define mmVGT_MAX_VTX_INDX 0xa100
+#define mmVGT_MIN_VTX_INDX 0xa101
+#define mmVGT_INDX_OFFSET 0xa102
+#define mmVGT_VERTEX_REUSE_BLOCK_CNTL 0xa316
+#define mmVGT_OUT_DEALLOC_CNTL 0xa317
+#define mmVGT_MULTI_PRIM_IB_RESET_INDX 0xa103
+#define mmVGT_MULTI_PRIM_IB_RESET_EN 0xa2a5
+#define mmVGT_ENHANCE 0xa294
+#define mmVGT_OUTPUT_PATH_CNTL 0xa284
+#define mmVGT_HOS_CNTL 0xa285
+#define mmVGT_HOS_MAX_TESS_LEVEL 0xa286
+#define mmVGT_HOS_MIN_TESS_LEVEL 0xa287
+#define mmVGT_HOS_REUSE_DEPTH 0xa288
+#define mmVGT_GROUP_PRIM_TYPE 0xa289
+#define mmVGT_GROUP_FIRST_DECR 0xa28a
+#define mmVGT_GROUP_DECR 0xa28b
+#define mmVGT_GROUP_VECT_0_CNTL 0xa28c
+#define mmVGT_GROUP_VECT_1_CNTL 0xa28d
+#define mmVGT_GROUP_VECT_0_FMT_CNTL 0xa28e
+#define mmVGT_GROUP_VECT_1_FMT_CNTL 0xa28f
+#define mmVGT_VTX_VECT_EJECT_REG 0x222c
+#define mmVGT_DMA_DATA_FIFO_DEPTH 0x222d
+#define mmVGT_DMA_REQ_FIFO_DEPTH 0x222e
+#define mmVGT_DRAW_INIT_FIFO_DEPTH 0x222f
+#define mmVGT_LAST_COPY_STATE 0x2230
+#define mmCC_GC_SHADER_ARRAY_CONFIG 0x226f
+#define mmGC_USER_SHADER_ARRAY_CONFIG 0x2270
+#define mmVGT_GS_MODE 0xa290
+#define mmVGT_GS_ONCHIP_CNTL 0xa291
+#define mmVGT_GS_OUT_PRIM_TYPE 0xa29b
+#define mmVGT_CACHE_INVALIDATION 0x2231
+#define mmVGT_RESET_DEBUG 0x2232
+#define mmVGT_STRMOUT_DELAY 0x2233
+#define mmVGT_FIFO_DEPTHS 0x2234
+#define mmVGT_GS_PER_ES 0xa295
+#define mmVGT_ES_PER_GS 0xa296
+#define mmVGT_GS_PER_VS 0xa297
+#define mmVGT_GS_VERTEX_REUSE 0x2235
+#define mmVGT_MC_LAT_CNTL 0x2236
+#define mmIA_CNTL_STATUS 0x2237
+#define mmVGT_STRMOUT_CONFIG 0xa2e5
+#define mmVGT_STRMOUT_BUFFER_SIZE_0 0xa2b4
+#define mmVGT_STRMOUT_BUFFER_SIZE_1 0xa2b8
+#define mmVGT_STRMOUT_BUFFER_SIZE_2 0xa2bc
+#define mmVGT_STRMOUT_BUFFER_SIZE_3 0xa2c0
+#define mmVGT_STRMOUT_BUFFER_OFFSET_0 0xa2b7
+#define mmVGT_STRMOUT_BUFFER_OFFSET_1 0xa2bb
+#define mmVGT_STRMOUT_BUFFER_OFFSET_2 0xa2bf
+#define mmVGT_STRMOUT_BUFFER_OFFSET_3 0xa2c3
+#define mmVGT_STRMOUT_VTX_STRIDE_0 0xa2b5
+#define mmVGT_STRMOUT_VTX_STRIDE_1 0xa2b9
+#define mmVGT_STRMOUT_VTX_STRIDE_2 0xa2bd
+#define mmVGT_STRMOUT_VTX_STRIDE_3 0xa2c1
+#define mmVGT_STRMOUT_BUFFER_CONFIG 0xa2e6
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_0 0xc244
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_1 0xc245
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_2 0xc246
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_3 0xc247
+#define mmVGT_STRMOUT_DRAW_OPAQUE_OFFSET 0xa2ca
+#define mmVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE 0xa2cb
+#define mmVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE 0xa2cc
+#define mmVGT_GS_MAX_VERT_OUT 0xa2ce
+#define mmVGT_SHADER_STAGES_EN 0xa2d5
+#define mmVGT_DISPATCH_DRAW_INDEX 0xa2dd
+#define mmVGT_LS_HS_CONFIG 0xa2d6
+#define mmVGT_DMA_LS_HS_CONFIG 0x2273
+#define mmVGT_TF_PARAM 0xa2db
+#define mmVGT_TESS_DISTRIBUTION 0xa2d4
+#define mmVGT_TF_RING_SIZE 0xc24e
+#define mmVGT_SYS_CONFIG 0x2263
+#define mmVGT_HS_OFFCHIP_PARAM 0xc24f
+#define mmVGT_TF_MEMORY_BASE 0xc250
+#define mmVGT_GS_INSTANCE_CNT 0xa2e4
+#define mmIA_MULTI_VGT_PARAM 0xa2aa
+#define mmVGT_VS_MAX_WAVE_ID 0x2268
+#define mmVGT_ESGS_RING_SIZE 0xc240
+#define mmVGT_GSVS_RING_SIZE 0xc241
+#define mmVGT_GSVS_RING_OFFSET_1 0xa298
+#define mmVGT_GSVS_RING_OFFSET_2 0xa299
+#define mmVGT_GSVS_RING_OFFSET_3 0xa29a
+#define mmVGT_ESGS_RING_ITEMSIZE 0xa2ab
+#define mmVGT_GSVS_RING_ITEMSIZE 0xa2ac
+#define mmVGT_GS_VERT_ITEMSIZE 0xa2d7
+#define mmVGT_GS_VERT_ITEMSIZE_1 0xa2d8
+#define mmVGT_GS_VERT_ITEMSIZE_2 0xa2d9
+#define mmVGT_GS_VERT_ITEMSIZE_3 0xa2da
+#define mmWD_CNTL_STATUS 0x223f
+#define mmWD_ENHANCE 0xa2a0
+#define mmGFX_PIPE_CONTROL 0x226d
+#define mmCGTT_VGT_CLK_CTRL 0xf084
+#define mmCGTT_IA_CLK_CTRL 0xf085
+#define mmCGTT_WD_CLK_CTRL 0xf086
+#define mmVGT_DEBUG_CNTL 0x2238
+#define mmVGT_DEBUG_DATA 0x2239
+#define mmIA_DEBUG_CNTL 0x223a
+#define mmIA_DEBUG_DATA 0x223b
+#define mmVGT_CNTL_STATUS 0x223c
+#define mmWD_DEBUG_CNTL 0x223d
+#define mmWD_DEBUG_DATA 0x223e
+#define mmWD_QOS 0x2242
+#define mmCC_GC_PRIM_CONFIG 0x2240
+#define mmGC_USER_PRIM_CONFIG 0x2241
+#define ixWD_DEBUG_REG0 0x0
+#define ixWD_DEBUG_REG1 0x1
+#define ixWD_DEBUG_REG2 0x2
+#define ixWD_DEBUG_REG3 0x3
+#define ixWD_DEBUG_REG4 0x4
+#define ixWD_DEBUG_REG5 0x5
+#define ixWD_DEBUG_REG6 0x6
+#define ixWD_DEBUG_REG7 0x7
+#define ixWD_DEBUG_REG8 0x8
+#define ixWD_DEBUG_REG9 0x9
+#define ixWD_DEBUG_REG10 0xa
+#define ixIA_DEBUG_REG0 0x0
+#define ixIA_DEBUG_REG1 0x1
+#define ixIA_DEBUG_REG2 0x2
+#define ixIA_DEBUG_REG3 0x3
+#define ixIA_DEBUG_REG4 0x4
+#define ixIA_DEBUG_REG5 0x5
+#define ixIA_DEBUG_REG6 0x6
+#define ixIA_DEBUG_REG7 0x7
+#define ixIA_DEBUG_REG8 0x8
+#define ixIA_DEBUG_REG9 0x9
+#define ixVGT_DEBUG_REG0 0x0
+#define ixVGT_DEBUG_REG1 0x1
+#define ixVGT_DEBUG_REG2 0x1e
+#define ixVGT_DEBUG_REG3 0x1f
+#define ixVGT_DEBUG_REG4 0x20
+#define ixVGT_DEBUG_REG5 0x21
+#define ixVGT_DEBUG_REG6 0x22
+#define ixVGT_DEBUG_REG7 0x23
+#define ixVGT_DEBUG_REG8 0x8
+#define ixVGT_DEBUG_REG9 0x9
+#define ixVGT_DEBUG_REG10 0xa
+#define ixVGT_DEBUG_REG11 0xb
+#define ixVGT_DEBUG_REG12 0xc
+#define ixVGT_DEBUG_REG13 0xd
+#define ixVGT_DEBUG_REG14 0xe
+#define ixVGT_DEBUG_REG15 0xf
+#define ixVGT_DEBUG_REG16 0x10
+#define ixVGT_DEBUG_REG17 0x11
+#define ixVGT_DEBUG_REG18 0x7
+#define ixVGT_DEBUG_REG19 0x13
+#define ixVGT_DEBUG_REG20 0x14
+#define ixVGT_DEBUG_REG21 0x15
+#define ixVGT_DEBUG_REG22 0x16
+#define ixVGT_DEBUG_REG23 0x17
+#define ixVGT_DEBUG_REG24 0x18
+#define ixVGT_DEBUG_REG25 0x19
+#define ixVGT_DEBUG_REG26 0x24
+#define ixVGT_DEBUG_REG27 0x1b
+#define ixVGT_DEBUG_REG28 0x1c
+#define ixVGT_DEBUG_REG29 0x1d
+#define ixVGT_DEBUG_REG31 0x26
+#define ixVGT_DEBUG_REG32 0x27
+#define ixVGT_DEBUG_REG33 0x28
+#define ixVGT_DEBUG_REG34 0x29
+#define ixVGT_DEBUG_REG36 0x2b
+#define mmVGT_PERFCOUNTER_SEID_MASK 0xd894
+#define mmVGT_PERFCOUNTER0_SELECT 0xd88c
+#define mmVGT_PERFCOUNTER1_SELECT 0xd88d
+#define mmVGT_PERFCOUNTER2_SELECT 0xd88e
+#define mmVGT_PERFCOUNTER3_SELECT 0xd88f
+#define mmVGT_PERFCOUNTER0_SELECT1 0xd890
+#define mmVGT_PERFCOUNTER1_SELECT1 0xd891
+#define mmVGT_PERFCOUNTER0_LO 0xd090
+#define mmVGT_PERFCOUNTER1_LO 0xd092
+#define mmVGT_PERFCOUNTER2_LO 0xd094
+#define mmVGT_PERFCOUNTER3_LO 0xd096
+#define mmVGT_PERFCOUNTER0_HI 0xd091
+#define mmVGT_PERFCOUNTER1_HI 0xd093
+#define mmVGT_PERFCOUNTER2_HI 0xd095
+#define mmVGT_PERFCOUNTER3_HI 0xd097
+#define mmIA_PERFCOUNTER0_SELECT 0xd884
+#define mmIA_PERFCOUNTER1_SELECT 0xd885
+#define mmIA_PERFCOUNTER2_SELECT 0xd886
+#define mmIA_PERFCOUNTER3_SELECT 0xd887
+#define mmIA_PERFCOUNTER0_SELECT1 0xd888
+#define mmIA_PERFCOUNTER0_LO 0xd088
+#define mmIA_PERFCOUNTER1_LO 0xd08a
+#define mmIA_PERFCOUNTER2_LO 0xd08c
+#define mmIA_PERFCOUNTER3_LO 0xd08e
+#define mmIA_PERFCOUNTER0_HI 0xd089
+#define mmIA_PERFCOUNTER1_HI 0xd08b
+#define mmIA_PERFCOUNTER2_HI 0xd08d
+#define mmIA_PERFCOUNTER3_HI 0xd08f
+#define mmWD_PERFCOUNTER0_SELECT 0xd880
+#define mmWD_PERFCOUNTER1_SELECT 0xd881
+#define mmWD_PERFCOUNTER2_SELECT 0xd882
+#define mmWD_PERFCOUNTER3_SELECT 0xd883
+#define mmWD_PERFCOUNTER0_LO 0xd080
+#define mmWD_PERFCOUNTER1_LO 0xd082
+#define mmWD_PERFCOUNTER2_LO 0xd084
+#define mmWD_PERFCOUNTER3_LO 0xd086
+#define mmWD_PERFCOUNTER0_HI 0xd081
+#define mmWD_PERFCOUNTER1_HI 0xd083
+#define mmWD_PERFCOUNTER2_HI 0xd085
+#define mmWD_PERFCOUNTER3_HI 0xd087
+#define mmDIDT_IND_INDEX 0x3280
+#define mmDIDT_IND_DATA 0x3281
+#define ixDIDT_SQ_CTRL0 0x0
+#define ixDIDT_SQ_CTRL1 0x1
+#define ixDIDT_SQ_CTRL2 0x2
+#define ixDIDT_SQ_CTRL_OCP 0x3
+#define ixDIDT_SQ_WEIGHT0_3 0x10
+#define ixDIDT_SQ_WEIGHT4_7 0x11
+#define ixDIDT_SQ_WEIGHT8_11 0x12
+#define ixDIDT_DB_CTRL0 0x20
+#define ixDIDT_DB_CTRL1 0x21
+#define ixDIDT_DB_CTRL2 0x22
+#define ixDIDT_DB_CTRL_OCP 0x23
+#define ixDIDT_DB_WEIGHT0_3 0x30
+#define ixDIDT_DB_WEIGHT4_7 0x31
+#define ixDIDT_DB_WEIGHT8_11 0x32
+#define ixDIDT_TD_CTRL0 0x40
+#define ixDIDT_TD_CTRL1 0x41
+#define ixDIDT_TD_CTRL2 0x42
+#define ixDIDT_TD_CTRL_OCP 0x43
+#define ixDIDT_TD_WEIGHT0_3 0x50
+#define ixDIDT_TD_WEIGHT4_7 0x51
+#define ixDIDT_TD_WEIGHT8_11 0x52
+#define ixDIDT_TCP_CTRL0 0x60
+#define ixDIDT_TCP_CTRL1 0x61
+#define ixDIDT_TCP_CTRL2 0x62
+#define ixDIDT_TCP_CTRL_OCP 0x63
+#define ixDIDT_TCP_WEIGHT0_3 0x70
+#define ixDIDT_TCP_WEIGHT4_7 0x71
+#define ixDIDT_TCP_WEIGHT8_11 0x72
+#define ixDIDT_DBR_CTRL0 0x80
+#define ixDIDT_DBR_CTRL1 0x81
+#define ixDIDT_DBR_CTRL2 0x82
+#define ixDIDT_DBR_CTRL_OCP 0x83
+#define ixDIDT_DBR_WEIGHT0_3 0x90
+#define ixDIDT_DBR_WEIGHT4_7 0x91
+#define ixDIDT_DBR_WEIGHT8_11 0x92
+
+#endif /* GFX_8_1_D_H */
--- /dev/null
+/*
+ * GFX_8_1 Register documentation
+ *
+ * Copyright (C) 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GFX_8_1_ENUM_H
+#define GFX_8_1_ENUM_H
+
+typedef enum SurfaceNumber {
+ NUMBER_UNORM = 0x0,
+ NUMBER_SNORM = 0x1,
+ NUMBER_USCALED = 0x2,
+ NUMBER_SSCALED = 0x3,
+ NUMBER_UINT = 0x4,
+ NUMBER_SINT = 0x5,
+ NUMBER_SRGB = 0x6,
+ NUMBER_FLOAT = 0x7,
+} SurfaceNumber;
+typedef enum SurfaceSwap {
+ SWAP_STD = 0x0,
+ SWAP_ALT = 0x1,
+ SWAP_STD_REV = 0x2,
+ SWAP_ALT_REV = 0x3,
+} SurfaceSwap;
+typedef enum CBMode {
+ CB_DISABLE = 0x0,
+ CB_NORMAL = 0x1,
+ CB_ELIMINATE_FAST_CLEAR = 0x2,
+ CB_RESOLVE = 0x3,
+ CB_DECOMPRESS = 0x4,
+ CB_FMASK_DECOMPRESS = 0x5,
+ CB_DCC_DECOMPRESS = 0x6,
+} CBMode;
+typedef enum RoundMode {
+ ROUND_BY_HALF = 0x0,
+ ROUND_TRUNCATE = 0x1,
+} RoundMode;
+typedef enum SourceFormat {
+ EXPORT_4C_32BPC = 0x0,
+ EXPORT_4C_16BPC = 0x1,
+ EXPORT_2C_32BPC_GR = 0x2,
+ EXPORT_2C_32BPC_AR = 0x3,
+} SourceFormat;
+typedef enum BlendOp {
+ BLEND_ZERO = 0x0,
+ BLEND_ONE = 0x1,
+ BLEND_SRC_COLOR = 0x2,
+ BLEND_ONE_MINUS_SRC_COLOR = 0x3,
+ BLEND_SRC_ALPHA = 0x4,
+ BLEND_ONE_MINUS_SRC_ALPHA = 0x5,
+ BLEND_DST_ALPHA = 0x6,
+ BLEND_ONE_MINUS_DST_ALPHA = 0x7,
+ BLEND_DST_COLOR = 0x8,
+ BLEND_ONE_MINUS_DST_COLOR = 0x9,
+ BLEND_SRC_ALPHA_SATURATE = 0xa,
+ BLEND_BOTH_SRC_ALPHA = 0xb,
+ BLEND_BOTH_INV_SRC_ALPHA = 0xc,
+ BLEND_CONSTANT_COLOR = 0xd,
+ BLEND_ONE_MINUS_CONSTANT_COLOR = 0xe,
+ BLEND_SRC1_COLOR = 0xf,
+ BLEND_INV_SRC1_COLOR = 0x10,
+ BLEND_SRC1_ALPHA = 0x11,
+ BLEND_INV_SRC1_ALPHA = 0x12,
+ BLEND_CONSTANT_ALPHA = 0x13,
+ BLEND_ONE_MINUS_CONSTANT_ALPHA = 0x14,
+} BlendOp;
+typedef enum CombFunc {
+ COMB_DST_PLUS_SRC = 0x0,
+ COMB_SRC_MINUS_DST = 0x1,
+ COMB_MIN_DST_SRC = 0x2,
+ COMB_MAX_DST_SRC = 0x3,
+ COMB_DST_MINUS_SRC = 0x4,
+} CombFunc;
+typedef enum BlendOpt {
+ FORCE_OPT_AUTO = 0x0,
+ FORCE_OPT_DISABLE = 0x1,
+ FORCE_OPT_ENABLE_IF_SRC_A_0 = 0x2,
+ FORCE_OPT_ENABLE_IF_SRC_RGB_0 = 0x3,
+ FORCE_OPT_ENABLE_IF_SRC_ARGB_0 = 0x4,
+ FORCE_OPT_ENABLE_IF_SRC_A_1 = 0x5,
+ FORCE_OPT_ENABLE_IF_SRC_RGB_1 = 0x6,
+ FORCE_OPT_ENABLE_IF_SRC_ARGB_1 = 0x7,
+} BlendOpt;
+typedef enum CmaskCode {
+ CMASK_CLR00_F0 = 0x0,
+ CMASK_CLR00_F1 = 0x1,
+ CMASK_CLR00_F2 = 0x2,
+ CMASK_CLR00_FX = 0x3,
+ CMASK_CLR01_F0 = 0x4,
+ CMASK_CLR01_F1 = 0x5,
+ CMASK_CLR01_F2 = 0x6,
+ CMASK_CLR01_FX = 0x7,
+ CMASK_CLR10_F0 = 0x8,
+ CMASK_CLR10_F1 = 0x9,
+ CMASK_CLR10_F2 = 0xa,
+ CMASK_CLR10_FX = 0xb,
+ CMASK_CLR11_F0 = 0xc,
+ CMASK_CLR11_F1 = 0xd,
+ CMASK_CLR11_F2 = 0xe,
+ CMASK_CLR11_FX = 0xf,
+} CmaskCode;
+typedef enum CmaskAddr {
+ CMASK_ADDR_TILED = 0x0,
+ CMASK_ADDR_LINEAR = 0x1,
+ CMASK_ADDR_COMPATIBLE = 0x2,
+} CmaskAddr;
+typedef enum CBPerfSel {
+ CB_PERF_SEL_NONE = 0x0,
+ CB_PERF_SEL_BUSY = 0x1,
+ CB_PERF_SEL_CORE_SCLK_VLD = 0x2,
+ CB_PERF_SEL_REG_SCLK0_VLD = 0x3,
+ CB_PERF_SEL_REG_SCLK1_VLD = 0x4,
+ CB_PERF_SEL_DRAWN_QUAD = 0x5,
+ CB_PERF_SEL_DRAWN_PIXEL = 0x6,
+ CB_PERF_SEL_DRAWN_QUAD_FRAGMENT = 0x7,
+ CB_PERF_SEL_DRAWN_TILE = 0x8,
+ CB_PERF_SEL_DB_CB_TILE_VALID_READY = 0x9,
+ CB_PERF_SEL_DB_CB_TILE_VALID_READYB = 0xa,
+ CB_PERF_SEL_DB_CB_TILE_VALIDB_READY = 0xb,
+ CB_PERF_SEL_DB_CB_TILE_VALIDB_READYB = 0xc,
+ CB_PERF_SEL_CM_FC_TILE_VALID_READY = 0xd,
+ CB_PERF_SEL_CM_FC_TILE_VALID_READYB = 0xe,
+ CB_PERF_SEL_CM_FC_TILE_VALIDB_READY = 0xf,
+ CB_PERF_SEL_CM_FC_TILE_VALIDB_READYB = 0x10,
+ CB_PERF_SEL_MERGE_TILE_ONLY_VALID_READY = 0x11,
+ CB_PERF_SEL_MERGE_TILE_ONLY_VALID_READYB = 0x12,
+ CB_PERF_SEL_DB_CB_LQUAD_VALID_READY = 0x13,
+ CB_PERF_SEL_DB_CB_LQUAD_VALID_READYB = 0x14,
+ CB_PERF_SEL_DB_CB_LQUAD_VALIDB_READY = 0x15,
+ CB_PERF_SEL_DB_CB_LQUAD_VALIDB_READYB = 0x16,
+ CB_PERF_SEL_LQUAD_NO_TILE = 0x17,
+ CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_R = 0x18,
+ CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_AR = 0x19,
+ CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_GR = 0x1a,
+ CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_ABGR = 0x1b,
+ CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_FP16_ABGR = 0x1c,
+ CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_SIGNED16_ABGR = 0x1d,
+ CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_UNSIGNED16_ABGR= 0x1e,
+ CB_PERF_SEL_QUAD_KILLED_BY_EXTRA_PIXEL_EXPORT = 0x1f,
+ CB_PERF_SEL_QUAD_KILLED_BY_COLOR_INVALID = 0x20,
+ CB_PERF_SEL_QUAD_KILLED_BY_NULL_TARGET_SHADER_MASK= 0x21,
+ CB_PERF_SEL_QUAD_KILLED_BY_NULL_SAMPLE_MASK = 0x22,
+ CB_PERF_SEL_QUAD_KILLED_BY_DISCARD_PIXEL = 0x23,
+ CB_PERF_SEL_FC_CLEAR_QUAD_VALID_READY = 0x24,
+ CB_PERF_SEL_FC_CLEAR_QUAD_VALID_READYB = 0x25,
+ CB_PERF_SEL_FC_CLEAR_QUAD_VALIDB_READY = 0x26,
+ CB_PERF_SEL_FC_CLEAR_QUAD_VALIDB_READYB = 0x27,
+ CB_PERF_SEL_FOP_IN_VALID_READY = 0x28,
+ CB_PERF_SEL_FOP_IN_VALID_READYB = 0x29,
+ CB_PERF_SEL_FOP_IN_VALIDB_READY = 0x2a,
+ CB_PERF_SEL_FOP_IN_VALIDB_READYB = 0x2b,
+ CB_PERF_SEL_FC_CC_QUADFRAG_VALID_READY = 0x2c,
+ CB_PERF_SEL_FC_CC_QUADFRAG_VALID_READYB = 0x2d,
+ CB_PERF_SEL_FC_CC_QUADFRAG_VALIDB_READY = 0x2e,
+ CB_PERF_SEL_FC_CC_QUADFRAG_VALIDB_READYB = 0x2f,
+ CB_PERF_SEL_CC_IB_SR_FRAG_VALID_READY = 0x30,
+ CB_PERF_SEL_CC_IB_SR_FRAG_VALID_READYB = 0x31,
+ CB_PERF_SEL_CC_IB_SR_FRAG_VALIDB_READY = 0x32,
+ CB_PERF_SEL_CC_IB_SR_FRAG_VALIDB_READYB = 0x33,
+ CB_PERF_SEL_CC_IB_TB_FRAG_VALID_READY = 0x34,
+ CB_PERF_SEL_CC_IB_TB_FRAG_VALID_READYB = 0x35,
+ CB_PERF_SEL_CC_IB_TB_FRAG_VALIDB_READY = 0x36,
+ CB_PERF_SEL_CC_IB_TB_FRAG_VALIDB_READYB = 0x37,
+ CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALID_READY = 0x38,
+ CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALID_READYB = 0x39,
+ CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALIDB_READY = 0x3a,
+ CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALIDB_READYB = 0x3b,
+ CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALID_READY = 0x3c,
+ CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALID_READYB = 0x3d,
+ CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALIDB_READY = 0x3e,
+ CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALIDB_READYB = 0x3f,
+ CB_PERF_SEL_CC_BC_CS_FRAG_VALID = 0x40,
+ CB_PERF_SEL_CM_CACHE_HIT = 0x41,
+ CB_PERF_SEL_CM_CACHE_TAG_MISS = 0x42,
+ CB_PERF_SEL_CM_CACHE_SECTOR_MISS = 0x43,
+ CB_PERF_SEL_CM_CACHE_REEVICTION_STALL = 0x44,
+ CB_PERF_SEL_CM_CACHE_EVICT_NONZERO_INFLIGHT_STALL= 0x45,
+ CB_PERF_SEL_CM_CACHE_REPLACE_PENDING_EVICT_STALL = 0x46,
+ CB_PERF_SEL_CM_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL= 0x47,
+ CB_PERF_SEL_CM_CACHE_READ_OUTPUT_STALL = 0x48,
+ CB_PERF_SEL_CM_CACHE_WRITE_OUTPUT_STALL = 0x49,
+ CB_PERF_SEL_CM_CACHE_ACK_OUTPUT_STALL = 0x4a,
+ CB_PERF_SEL_CM_CACHE_STALL = 0x4b,
+ CB_PERF_SEL_CM_CACHE_FLUSH = 0x4c,
+ CB_PERF_SEL_CM_CACHE_TAGS_FLUSHED = 0x4d,
+ CB_PERF_SEL_CM_CACHE_SECTORS_FLUSHED = 0x4e,
+ CB_PERF_SEL_CM_CACHE_DIRTY_SECTORS_FLUSHED = 0x4f,
+ CB_PERF_SEL_FC_CACHE_HIT = 0x50,
+ CB_PERF_SEL_FC_CACHE_TAG_MISS = 0x51,
+ CB_PERF_SEL_FC_CACHE_SECTOR_MISS = 0x52,
+ CB_PERF_SEL_FC_CACHE_REEVICTION_STALL = 0x53,
+ CB_PERF_SEL_FC_CACHE_EVICT_NONZERO_INFLIGHT_STALL= 0x54,
+ CB_PERF_SEL_FC_CACHE_REPLACE_PENDING_EVICT_STALL = 0x55,
+ CB_PERF_SEL_FC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL= 0x56,
+ CB_PERF_SEL_FC_CACHE_READ_OUTPUT_STALL = 0x57,
+ CB_PERF_SEL_FC_CACHE_WRITE_OUTPUT_STALL = 0x58,
+ CB_PERF_SEL_FC_CACHE_ACK_OUTPUT_STALL = 0x59,
+ CB_PERF_SEL_FC_CACHE_STALL = 0x5a,
+ CB_PERF_SEL_FC_CACHE_FLUSH = 0x5b,
+ CB_PERF_SEL_FC_CACHE_TAGS_FLUSHED = 0x5c,
+ CB_PERF_SEL_FC_CACHE_SECTORS_FLUSHED = 0x5d,
+ CB_PERF_SEL_FC_CACHE_DIRTY_SECTORS_FLUSHED = 0x5e,
+ CB_PERF_SEL_CC_CACHE_HIT = 0x5f,
+ CB_PERF_SEL_CC_CACHE_TAG_MISS = 0x60,
+ CB_PERF_SEL_CC_CACHE_SECTOR_MISS = 0x61,
+ CB_PERF_SEL_CC_CACHE_REEVICTION_STALL = 0x62,
+ CB_PERF_SEL_CC_CACHE_EVICT_NONZERO_INFLIGHT_STALL= 0x63,
+ CB_PERF_SEL_CC_CACHE_REPLACE_PENDING_EVICT_STALL = 0x64,
+ CB_PERF_SEL_CC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL= 0x65,
+ CB_PERF_SEL_CC_CACHE_READ_OUTPUT_STALL = 0x66,
+ CB_PERF_SEL_CC_CACHE_WRITE_OUTPUT_STALL = 0x67,
+ CB_PERF_SEL_CC_CACHE_ACK_OUTPUT_STALL = 0x68,
+ CB_PERF_SEL_CC_CACHE_STALL = 0x69,
+ CB_PERF_SEL_CC_CACHE_FLUSH = 0x6a,
+ CB_PERF_SEL_CC_CACHE_TAGS_FLUSHED = 0x6b,
+ CB_PERF_SEL_CC_CACHE_SECTORS_FLUSHED = 0x6c,
+ CB_PERF_SEL_CC_CACHE_DIRTY_SECTORS_FLUSHED = 0x6d,
+ CB_PERF_SEL_CC_CACHE_WA_TO_RMW_CONVERSION = 0x6e,
+ CB_PERF_SEL_CC_CACHE_READS_SAVED_DUE_TO_DCC = 0x6f,
+ CB_PERF_SEL_CB_TAP_WRREQ_VALID_READY = 0x70,
+ CB_PERF_SEL_CB_TAP_WRREQ_VALID_READYB = 0x71,
+ CB_PERF_SEL_CB_TAP_WRREQ_VALIDB_READY = 0x72,
+ CB_PERF_SEL_CB_TAP_WRREQ_VALIDB_READYB = 0x73,
+ CB_PERF_SEL_CM_MC_WRITE_REQUEST = 0x74,
+ CB_PERF_SEL_FC_MC_WRITE_REQUEST = 0x75,
+ CB_PERF_SEL_CC_MC_WRITE_REQUEST = 0x76,
+ CB_PERF_SEL_CM_MC_WRITE_REQUESTS_IN_FLIGHT = 0x77,
+ CB_PERF_SEL_FC_MC_WRITE_REQUESTS_IN_FLIGHT = 0x78,
+ CB_PERF_SEL_CC_MC_WRITE_REQUESTS_IN_FLIGHT = 0x79,
+ CB_PERF_SEL_CB_TAP_RDREQ_VALID_READY = 0x7a,
+ CB_PERF_SEL_CB_TAP_RDREQ_VALID_READYB = 0x7b,
+ CB_PERF_SEL_CB_TAP_RDREQ_VALIDB_READY = 0x7c,
+ CB_PERF_SEL_CB_TAP_RDREQ_VALIDB_READYB = 0x7d,
+ CB_PERF_SEL_CM_MC_READ_REQUEST = 0x7e,
+ CB_PERF_SEL_FC_MC_READ_REQUEST = 0x7f,
+ CB_PERF_SEL_CC_MC_READ_REQUEST = 0x80,
+ CB_PERF_SEL_CM_MC_READ_REQUESTS_IN_FLIGHT = 0x81,
+ CB_PERF_SEL_FC_MC_READ_REQUESTS_IN_FLIGHT = 0x82,
+ CB_PERF_SEL_CC_MC_READ_REQUESTS_IN_FLIGHT = 0x83,
+ CB_PERF_SEL_CM_TQ_FULL = 0x84,
+ CB_PERF_SEL_CM_TQ_FIFO_TILE_RESIDENCY_STALL = 0x85,
+ CB_PERF_SEL_FC_QUAD_RDLAT_FIFO_FULL = 0x86,
+ CB_PERF_SEL_FC_TILE_RDLAT_FIFO_FULL = 0x87,
+ CB_PERF_SEL_FC_RDLAT_FIFO_QUAD_RESIDENCY_STALL = 0x88,
+ CB_PERF_SEL_FOP_FMASK_RAW_STALL = 0x89,
+ CB_PERF_SEL_FOP_FMASK_BYPASS_STALL = 0x8a,
+ CB_PERF_SEL_CC_SF_FULL = 0x8b,
+ CB_PERF_SEL_CC_RB_FULL = 0x8c,
+ CB_PERF_SEL_CC_EVENFIFO_QUAD_RESIDENCY_STALL = 0x8d,
+ CB_PERF_SEL_CC_ODDFIFO_QUAD_RESIDENCY_STALL = 0x8e,
+ CB_PERF_SEL_BLENDER_RAW_HAZARD_STALL = 0x8f,
+ CB_PERF_SEL_EVENT = 0x90,
+ CB_PERF_SEL_EVENT_CACHE_FLUSH_TS = 0x91,
+ CB_PERF_SEL_EVENT_CONTEXT_DONE = 0x92,
+ CB_PERF_SEL_EVENT_CACHE_FLUSH = 0x93,
+ CB_PERF_SEL_EVENT_CACHE_FLUSH_AND_INV_TS_EVENT = 0x94,
+ CB_PERF_SEL_EVENT_CACHE_FLUSH_AND_INV_EVENT = 0x95,
+ CB_PERF_SEL_EVENT_FLUSH_AND_INV_CB_DATA_TS = 0x96,
+ CB_PERF_SEL_EVENT_FLUSH_AND_INV_CB_META = 0x97,
+ CB_PERF_SEL_CC_SURFACE_SYNC = 0x98,
+ CB_PERF_SEL_CMASK_READ_DATA_0xC = 0x99,
+ CB_PERF_SEL_CMASK_READ_DATA_0xD = 0x9a,
+ CB_PERF_SEL_CMASK_READ_DATA_0xE = 0x9b,
+ CB_PERF_SEL_CMASK_READ_DATA_0xF = 0x9c,
+ CB_PERF_SEL_CMASK_WRITE_DATA_0xC = 0x9d,
+ CB_PERF_SEL_CMASK_WRITE_DATA_0xD = 0x9e,
+ CB_PERF_SEL_CMASK_WRITE_DATA_0xE = 0x9f,
+ CB_PERF_SEL_CMASK_WRITE_DATA_0xF = 0xa0,
+ CB_PERF_SEL_TWO_PROBE_QUAD_FRAGMENT = 0xa1,
+ CB_PERF_SEL_EXPORT_32_ABGR_QUAD_FRAGMENT = 0xa2,
+ CB_PERF_SEL_DUAL_SOURCE_COLOR_QUAD_FRAGMENT = 0xa3,
+ CB_PERF_SEL_QUAD_HAS_1_FRAGMENT_BEFORE_UPDATE = 0xa4,
+ CB_PERF_SEL_QUAD_HAS_2_FRAGMENTS_BEFORE_UPDATE = 0xa5,
+ CB_PERF_SEL_QUAD_HAS_3_FRAGMENTS_BEFORE_UPDATE = 0xa6,
+ CB_PERF_SEL_QUAD_HAS_4_FRAGMENTS_BEFORE_UPDATE = 0xa7,
+ CB_PERF_SEL_QUAD_HAS_5_FRAGMENTS_BEFORE_UPDATE = 0xa8,
+ CB_PERF_SEL_QUAD_HAS_6_FRAGMENTS_BEFORE_UPDATE = 0xa9,
+ CB_PERF_SEL_QUAD_HAS_7_FRAGMENTS_BEFORE_UPDATE = 0xaa,
+ CB_PERF_SEL_QUAD_HAS_8_FRAGMENTS_BEFORE_UPDATE = 0xab,
+ CB_PERF_SEL_QUAD_HAS_1_FRAGMENT_AFTER_UPDATE = 0xac,
+ CB_PERF_SEL_QUAD_HAS_2_FRAGMENTS_AFTER_UPDATE = 0xad,
+ CB_PERF_SEL_QUAD_HAS_3_FRAGMENTS_AFTER_UPDATE = 0xae,
+ CB_PERF_SEL_QUAD_HAS_4_FRAGMENTS_AFTER_UPDATE = 0xaf,
+ CB_PERF_SEL_QUAD_HAS_5_FRAGMENTS_AFTER_UPDATE = 0xb0,
+ CB_PERF_SEL_QUAD_HAS_6_FRAGMENTS_AFTER_UPDATE = 0xb1,
+ CB_PERF_SEL_QUAD_HAS_7_FRAGMENTS_AFTER_UPDATE = 0xb2,
+ CB_PERF_SEL_QUAD_HAS_8_FRAGMENTS_AFTER_UPDATE = 0xb3,
+ CB_PERF_SEL_QUAD_ADDED_1_FRAGMENT = 0xb4,
+ CB_PERF_SEL_QUAD_ADDED_2_FRAGMENTS = 0xb5,
+ CB_PERF_SEL_QUAD_ADDED_3_FRAGMENTS = 0xb6,
+ CB_PERF_SEL_QUAD_ADDED_4_FRAGMENTS = 0xb7,
+ CB_PERF_SEL_QUAD_ADDED_5_FRAGMENTS = 0xb8,
+ CB_PERF_SEL_QUAD_ADDED_6_FRAGMENTS = 0xb9,
+ CB_PERF_SEL_QUAD_ADDED_7_FRAGMENTS = 0xba,
+ CB_PERF_SEL_QUAD_REMOVED_1_FRAGMENT = 0xbb,
+ CB_PERF_SEL_QUAD_REMOVED_2_FRAGMENTS = 0xbc,
+ CB_PERF_SEL_QUAD_REMOVED_3_FRAGMENTS = 0xbd,
+ CB_PERF_SEL_QUAD_REMOVED_4_FRAGMENTS = 0xbe,
+ CB_PERF_SEL_QUAD_REMOVED_5_FRAGMENTS = 0xbf,
+ CB_PERF_SEL_QUAD_REMOVED_6_FRAGMENTS = 0xc0,
+ CB_PERF_SEL_QUAD_REMOVED_7_FRAGMENTS = 0xc1,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_0 = 0xc2,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_1 = 0xc3,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_2 = 0xc4,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_3 = 0xc5,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_4 = 0xc6,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_5 = 0xc7,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_6 = 0xc8,
+ CB_PERF_SEL_QUAD_READS_FRAGMENT_7 = 0xc9,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_0 = 0xca,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_1 = 0xcb,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_2 = 0xcc,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_3 = 0xcd,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_4 = 0xce,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_5 = 0xcf,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_6 = 0xd0,
+ CB_PERF_SEL_QUAD_WRITES_FRAGMENT_7 = 0xd1,
+ CB_PERF_SEL_QUAD_BLEND_OPT_DONT_READ_DST = 0xd2,
+ CB_PERF_SEL_QUAD_BLEND_OPT_BLEND_BYPASS = 0xd3,
+ CB_PERF_SEL_QUAD_BLEND_OPT_DISCARD_PIXELS = 0xd4,
+ CB_PERF_SEL_QUAD_DST_READ_COULD_HAVE_BEEN_OPTIMIZED= 0xd5,
+ CB_PERF_SEL_QUAD_BLENDING_COULD_HAVE_BEEN_BYPASSED= 0xd6,
+ CB_PERF_SEL_QUAD_COULD_HAVE_BEEN_DISCARDED = 0xd7,
+ CB_PERF_SEL_BLEND_OPT_PIXELS_RESULT_EQ_DEST = 0xd8,
+ CB_PERF_SEL_DRAWN_BUSY = 0xd9,
+ CB_PERF_SEL_TILE_TO_CMR_REGION_BUSY = 0xda,
+ CB_PERF_SEL_CMR_TO_FCR_REGION_BUSY = 0xdb,
+ CB_PERF_SEL_FCR_TO_CCR_REGION_BUSY = 0xdc,
+ CB_PERF_SEL_CCR_TO_CCW_REGION_BUSY = 0xdd,
+ CB_PERF_SEL_FC_PF_SLOW_MODE_QUAD_EMPTY_HALF_DROPPED= 0xde,
+ CB_PERF_SEL_FC_SEQUENCER_CLEAR = 0xdf,
+ CB_PERF_SEL_FC_SEQUENCER_ELIMINATE_FAST_CLEAR = 0xe0,
+ CB_PERF_SEL_FC_SEQUENCER_FMASK_DECOMPRESS = 0xe1,
+ CB_PERF_SEL_FC_SEQUENCER_FMASK_COMPRESSION_DISABLE= 0xe2,
+ CB_PERF_SEL_FC_KEYID_RDLAT_FIFO_FULL = 0xe3,
+ CB_PERF_SEL_FC_DOC_IS_STALLED = 0xe4,
+ CB_PERF_SEL_FC_DOC_MRTS_NOT_COMBINED = 0xe5,
+ CB_PERF_SEL_FC_DOC_MRTS_COMBINED = 0xe6,
+ CB_PERF_SEL_FC_DOC_QTILE_CAM_MISS = 0xe7,
+ CB_PERF_SEL_FC_DOC_QTILE_CAM_HIT = 0xe8,
+ CB_PERF_SEL_FC_DOC_CLINE_CAM_MISS = 0xe9,
+ CB_PERF_SEL_FC_DOC_CLINE_CAM_HIT = 0xea,
+ CB_PERF_SEL_FC_DOC_QUAD_PTR_FIFO_IS_FULL = 0xeb,
+ CB_PERF_SEL_FC_DOC_OVERWROTE_1_SECTOR = 0xec,
+ CB_PERF_SEL_FC_DOC_OVERWROTE_2_SECTORS = 0xed,
+ CB_PERF_SEL_FC_DOC_OVERWROTE_3_SECTORS = 0xee,
+ CB_PERF_SEL_FC_DOC_OVERWROTE_4_SECTORS = 0xef,
+ CB_PERF_SEL_FC_DOC_TOTAL_OVERWRITTEN_SECTORS = 0xf0,
+ CB_PERF_SEL_FC_DCC_CACHE_HIT = 0xf1,
+ CB_PERF_SEL_FC_DCC_CACHE_TAG_MISS = 0xf2,
+ CB_PERF_SEL_FC_DCC_CACHE_SECTOR_MISS = 0xf3,
+ CB_PERF_SEL_FC_DCC_CACHE_REEVICTION_STALL = 0xf4,
+ CB_PERF_SEL_FC_DCC_CACHE_EVICT_NONZERO_INFLIGHT_STALL= 0xf5,
+ CB_PERF_SEL_FC_DCC_CACHE_REPLACE_PENDING_EVICT_STALL= 0xf6,
+ CB_PERF_SEL_FC_DCC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL= 0xf7,
+ CB_PERF_SEL_FC_DCC_CACHE_READ_OUTPUT_STALL = 0xf8,
+ CB_PERF_SEL_FC_DCC_CACHE_WRITE_OUTPUT_STALL = 0xf9,
+ CB_PERF_SEL_FC_DCC_CACHE_ACK_OUTPUT_STALL = 0xfa,
+ CB_PERF_SEL_FC_DCC_CACHE_STALL = 0xfb,
+ CB_PERF_SEL_FC_DCC_CACHE_FLUSH = 0xfc,
+ CB_PERF_SEL_FC_DCC_CACHE_TAGS_FLUSHED = 0xfd,
+ CB_PERF_SEL_FC_DCC_CACHE_SECTORS_FLUSHED = 0xfe,
+ CB_PERF_SEL_FC_DCC_CACHE_DIRTY_SECTORS_FLUSHED = 0xff,
+ CB_PERF_SEL_CC_DCC_BEYOND_TILE_SPLIT = 0x100,
+ CB_PERF_SEL_FC_MC_DCC_WRITE_REQUEST = 0x101,
+ CB_PERF_SEL_FC_MC_DCC_WRITE_REQUESTS_IN_FLIGHT = 0x102,
+ CB_PERF_SEL_FC_MC_DCC_READ_REQUEST = 0x103,
+ CB_PERF_SEL_FC_MC_DCC_READ_REQUESTS_IN_FLIGHT = 0x104,
+ CB_PERF_SEL_CC_DCC_RDREQ_STALL = 0x105,
+ CB_PERF_SEL_CC_DCC_DECOMPRESS_TIDS_IN = 0x106,
+ CB_PERF_SEL_CC_DCC_DECOMPRESS_TIDS_OUT = 0x107,
+ CB_PERF_SEL_CC_DCC_COMPRESS_TIDS_IN = 0x108,
+ CB_PERF_SEL_CC_DCC_COMPRESS_TIDS_OUT = 0x109,
+ CB_PERF_SEL_FC_DCC_KEY_VALUE__CLEAR = 0x10a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__4_BLOCKS__2TO1 = 0x10b,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__3BLOCKS_2TO1__1BLOCK_2TO2= 0x10c,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO1__1BLOCK_2TO2__1BLOCK_2TO1= 0x10d,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_2TO2__2BLOCKS_2TO1= 0x10e,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__3BLOCKS_2TO1= 0x10f,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO1__2BLOCKS_2TO2= 0x110,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__2BLOCKS_2TO2__1BLOCK_2TO1= 0x111,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_2TO2__1BLOCK_2TO1__1BLOCK_2TO2= 0x112,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_2TO1__1BLOCK_2TO2__1BLOCK_2TO1= 0x113,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO2__2BLOCKS_2TO1= 0x114,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__2BLOCKS_2TO1__1BLOCK_2TO2= 0x115,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__3BLOCKS_2TO2= 0x116,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_2TO1__2BLOCKS_2TO2= 0x117,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO2__1BLOCK_2TO1__1BLOCK_2TO2= 0x118,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__3BLOCKS_2TO2__1BLOCK_2TO1= 0x119,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_4TO1 = 0x11a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO1__1BLOCK_4TO2= 0x11b,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO1__1BLOCK_4TO3= 0x11c,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO1__1BLOCK_4TO4= 0x11d,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO2__1BLOCK_4TO1= 0x11e,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_4TO2 = 0x11f,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO2__1BLOCK_4TO3= 0x120,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO2__1BLOCK_4TO4= 0x121,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO3__1BLOCK_4TO1= 0x122,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO3__1BLOCK_4TO2= 0x123,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_4TO3 = 0x124,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO3__1BLOCK_4TO4= 0x125,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO4__1BLOCK_4TO1= 0x126,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO4__1BLOCK_4TO2= 0x127,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO4__1BLOCK_4TO3= 0x128,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO1__1BLOCK_4TO1= 0x129,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO1__1BLOCK_4TO2= 0x12a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO1__1BLOCK_4TO3= 0x12b,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO1__1BLOCK_4TO4= 0x12c,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_2TO2__1BLOCK_4TO1= 0x12d,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_2TO2__1BLOCK_4TO2= 0x12e,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_2TO2__1BLOCK_4TO3= 0x12f,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_2TO2__1BLOCK_4TO4= 0x130,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_2TO1__1BLOCK_4TO1= 0x131,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_2TO1__1BLOCK_4TO2= 0x132,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_2TO1__1BLOCK_4TO3= 0x133,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_2TO1__1BLOCK_4TO4= 0x134,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO2__1BLOCK_4TO1= 0x135,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO2__1BLOCK_4TO2= 0x136,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__2BLOCKS_2TO2__1BLOCK_4TO3= 0x137,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO1__1BLOCK_2TO1= 0x138,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO2__1BLOCK_2TO1= 0x139,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO3__1BLOCK_2TO1= 0x13a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO4__1BLOCK_2TO1= 0x13b,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_4TO1__1BLOCK_2TO1= 0x13c,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_4TO2__1BLOCK_2TO1= 0x13d,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_4TO3__1BLOCK_2TO1= 0x13e,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_4TO4__1BLOCK_2TO1= 0x13f,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO1__1BLOCK_2TO2= 0x140,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO2__1BLOCK_2TO2= 0x141,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO3__1BLOCK_2TO2= 0x142,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_4TO4__1BLOCK_2TO2= 0x143,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_4TO1__1BLOCK_2TO2= 0x144,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_4TO2__1BLOCK_2TO2= 0x145,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_4TO3__1BLOCK_2TO2= 0x146,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO1__2BLOCKS_2TO1= 0x147,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO2__2BLOCKS_2TO1= 0x148,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO3__2BLOCKS_2TO1= 0x149,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO4__2BLOCKS_2TO1= 0x14a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO1__2BLOCKS_2TO2= 0x14b,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO2__2BLOCKS_2TO2= 0x14c,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO3__2BLOCKS_2TO2= 0x14d,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO1__1BLOCK_2TO1__1BLOCK_2TO2= 0x14e,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO2__1BLOCK_2TO1__1BLOCK_2TO2= 0x14f,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO3__1BLOCK_2TO1__1BLOCK_2TO2= 0x150,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO4__1BLOCK_2TO1__1BLOCK_2TO2= 0x151,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO1__1BLOCK_2TO2__1BLOCK_2TO1= 0x152,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO2__1BLOCK_2TO2__1BLOCK_2TO1= 0x153,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO3__1BLOCK_2TO2__1BLOCK_2TO1= 0x154,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_4TO4__1BLOCK_2TO2__1BLOCK_2TO1= 0x155,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_6TO1= 0x156,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_6TO2= 0x157,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_6TO3= 0x158,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_6TO4= 0x159,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_6TO5= 0x15a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__1BLOCK_6TO6= 0x15b,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__INV0 = 0x15c,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO1__INV1 = 0x15d,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_6TO1= 0x15e,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_6TO2= 0x15f,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_6TO3= 0x160,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_6TO4= 0x161,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__1BLOCK_6TO5= 0x162,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__INV0 = 0x163,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_2TO2__INV1 = 0x164,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO1__1BLOCK_2TO1= 0x165,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO2__1BLOCK_2TO1= 0x166,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO3__1BLOCK_2TO1= 0x167,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO4__1BLOCK_2TO1= 0x168,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO5__1BLOCK_2TO1= 0x169,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO6__1BLOCK_2TO1= 0x16a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__INV0__1BLOCK_2TO1 = 0x16b,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__INV1__1BLOCK_2TO1 = 0x16c,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO1__1BLOCK_2TO2= 0x16d,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO2__1BLOCK_2TO2= 0x16e,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO3__1BLOCK_2TO2= 0x16f,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO4__1BLOCK_2TO2= 0x170,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_6TO5__1BLOCK_2TO2= 0x171,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__INV0__1BLOCK_2TO2 = 0x172,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__INV1__1BLOCK_2TO2 = 0x173,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_8TO1 = 0x174,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_8TO2 = 0x175,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_8TO3 = 0x176,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_8TO4 = 0x177,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_8TO5 = 0x178,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_8TO6 = 0x179,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__1BLOCK_8TO7 = 0x17a,
+ CB_PERF_SEL_CC_DCC_KEY_VALUE__UNCOMPRESSED = 0x17b,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_2TO1 = 0x17c,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_4TO1 = 0x17d,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_4TO2 = 0x17e,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_4TO3 = 0x17f,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_6TO1 = 0x180,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_6TO2 = 0x181,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_6TO3 = 0x182,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_6TO4 = 0x183,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_6TO5 = 0x184,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_8TO1 = 0x185,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_8TO2 = 0x186,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_8TO3 = 0x187,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_8TO4 = 0x188,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_8TO5 = 0x189,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_8TO6 = 0x18a,
+ CB_PERF_SEL_CC_DCC_COMPRESS_RATIO_8TO7 = 0x18b,
+ CB_PERF_SEL_RBP_EXPORT_8PIX_LIT_BOTH = 0x18c,
+ CB_PERF_SEL_RBP_EXPORT_8PIX_LIT_LEFT = 0x18d,
+ CB_PERF_SEL_RBP_EXPORT_8PIX_LIT_RIGHT = 0x18e,
+ CB_PERF_SEL_RBP_SPLIT_MICROTILE = 0x18f,
+ CB_PERF_SEL_RBP_SPLIT_AA_SAMPLE_MASK = 0x190,
+ CB_PERF_SEL_RBP_SPLIT_PARTIAL_TARGET_MASK = 0x191,
+ CB_PERF_SEL_RBP_SPLIT_LINEAR_ADDRESSING = 0x192,
+ CB_PERF_SEL_RBP_SPLIT_AA_NO_FMASK_COMPRESS = 0x193,
+ CB_PERF_SEL_RBP_INSERT_MISSING_LAST_QUAD = 0x194,
+} CBPerfSel;
+typedef enum CBPerfOpFilterSel {
+ CB_PERF_OP_FILTER_SEL_WRITE_ONLY = 0x0,
+ CB_PERF_OP_FILTER_SEL_NEEDS_DESTINATION = 0x1,
+ CB_PERF_OP_FILTER_SEL_RESOLVE = 0x2,
+ CB_PERF_OP_FILTER_SEL_DECOMPRESS = 0x3,
+ CB_PERF_OP_FILTER_SEL_FMASK_DECOMPRESS = 0x4,
+ CB_PERF_OP_FILTER_SEL_ELIMINATE_FAST_CLEAR = 0x5,
+} CBPerfOpFilterSel;
+typedef enum CBPerfClearFilterSel {
+ CB_PERF_CLEAR_FILTER_SEL_NONCLEAR = 0x0,
+ CB_PERF_CLEAR_FILTER_SEL_CLEAR = 0x1,
+} CBPerfClearFilterSel;
+typedef enum CP_RING_ID {
+ RINGID0 = 0x0,
+ RINGID1 = 0x1,
+ RINGID2 = 0x2,
+ RINGID3 = 0x3,
+} CP_RING_ID;
+typedef enum CP_PIPE_ID {
+ PIPE_ID0 = 0x0,
+ PIPE_ID1 = 0x1,
+ PIPE_ID2 = 0x2,
+ PIPE_ID3 = 0x3,
+} CP_PIPE_ID;
+typedef enum CP_ME_ID {
+ ME_ID0 = 0x0,
+ ME_ID1 = 0x1,
+ ME_ID2 = 0x2,
+ ME_ID3 = 0x3,
+} CP_ME_ID;
+typedef enum SPM_PERFMON_STATE {
+ STRM_PERFMON_STATE_DISABLE_AND_RESET = 0x0,
+ STRM_PERFMON_STATE_START_COUNTING = 0x1,
+ STRM_PERFMON_STATE_STOP_COUNTING = 0x2,
+ STRM_PERFMON_STATE_RESERVED_3 = 0x3,
+ STRM_PERFMON_STATE_DISABLE_AND_RESET_PHANTOM = 0x4,
+ STRM_PERFMON_STATE_COUNT_AND_DUMP_PHANTOM = 0x5,
+} SPM_PERFMON_STATE;
+typedef enum CP_PERFMON_STATE {
+ CP_PERFMON_STATE_DISABLE_AND_RESET = 0x0,
+ CP_PERFMON_STATE_START_COUNTING = 0x1,
+ CP_PERFMON_STATE_STOP_COUNTING = 0x2,
+ CP_PERFMON_STATE_RESERVED_3 = 0x3,
+ CP_PERFMON_STATE_DISABLE_AND_RESET_PHANTOM = 0x4,
+ CP_PERFMON_STATE_COUNT_AND_DUMP_PHANTOM = 0x5,
+} CP_PERFMON_STATE;
+typedef enum CP_PERFMON_ENABLE_MODE {
+ CP_PERFMON_ENABLE_MODE_ALWAYS_COUNT = 0x0,
+ CP_PERFMON_ENABLE_MODE_RESERVED_1 = 0x1,
+ CP_PERFMON_ENABLE_MODE_COUNT_CONTEXT_TRUE = 0x2,
+ CP_PERFMON_ENABLE_MODE_COUNT_CONTEXT_FALSE = 0x3,
+} CP_PERFMON_ENABLE_MODE;
+typedef enum CPG_PERFCOUNT_SEL {
+ CPG_PERF_SEL_ALWAYS_COUNT = 0x0,
+ CPG_PERF_SEL_RBIU_FIFO_FULL = 0x1,
+ CPG_PERF_SEL_CSF_RTS_BUT_MIU_NOT_RTR = 0x2,
+ CPG_PERF_SEL_CSF_ST_BASE_SIZE_FIFO_FULL = 0x3,
+ CPG_PERF_SEL_CP_GRBM_DWORDS_SENT = 0x4,
+ CPG_PERF_SEL_ME_PARSER_BUSY = 0x5,
+ CPG_PERF_SEL_COUNT_TYPE0_PACKETS = 0x6,
+ CPG_PERF_SEL_COUNT_TYPE3_PACKETS = 0x7,
+ CPG_PERF_SEL_CSF_FETCHING_CMD_BUFFERS = 0x8,
+ CPG_PERF_SEL_CP_GRBM_OUT_OF_CREDITS = 0x9,
+ CPG_PERF_SEL_CP_PFP_GRBM_OUT_OF_CREDITS = 0xa,
+ CPG_PERF_SEL_CP_GDS_GRBM_OUT_OF_CREDITS = 0xb,
+ CPG_PERF_SEL_RCIU_STALLED_ON_ME_READ = 0xc,
+ CPG_PERF_SEL_RCIU_STALLED_ON_DMA_READ = 0xd,
+ CPG_PERF_SEL_SSU_STALLED_ON_ACTIVE_CNTX = 0xe,
+ CPG_PERF_SEL_SSU_STALLED_ON_CLEAN_SIGNALS = 0xf,
+ CPG_PERF_SEL_QU_STALLED_ON_EOP_DONE_PULSE = 0x10,
+ CPG_PERF_SEL_QU_STALLED_ON_EOP_DONE_WR_CONFIRM = 0x11,
+ CPG_PERF_SEL_PFP_STALLED_ON_CSF_READY = 0x12,
+ CPG_PERF_SEL_PFP_STALLED_ON_MEQ_READY = 0x13,
+ CPG_PERF_SEL_PFP_STALLED_ON_RCIU_READY = 0x14,
+ CPG_PERF_SEL_PFP_STALLED_FOR_DATA_FROM_ROQ = 0x15,
+ CPG_PERF_SEL_ME_STALLED_FOR_DATA_FROM_PFP = 0x16,
+ CPG_PERF_SEL_ME_STALLED_FOR_DATA_FROM_STQ = 0x17,
+ CPG_PERF_SEL_ME_STALLED_ON_NO_AVAIL_GFX_CNTX = 0x18,
+ CPG_PERF_SEL_ME_STALLED_WRITING_TO_RCIU = 0x19,
+ CPG_PERF_SEL_ME_STALLED_WRITING_CONSTANTS = 0x1a,
+ CPG_PERF_SEL_ME_STALLED_ON_PARTIAL_FLUSH = 0x1b,
+ CPG_PERF_SEL_ME_WAIT_ON_CE_COUNTER = 0x1c,
+ CPG_PERF_SEL_ME_WAIT_ON_AVAIL_BUFFER = 0x1d,
+ CPG_PERF_SEL_SEMAPHORE_BUSY_POLLING_FOR_PASS = 0x1e,
+ CPG_PERF_SEL_LOAD_STALLED_ON_SET_COHERENCY = 0x1f,
+ CPG_PERF_SEL_DYNAMIC_CLK_VALID = 0x20,
+ CPG_PERF_SEL_REGISTER_CLK_VALID = 0x21,
+ CPG_PERF_SEL_MIU_WRITE_REQUEST_SENT = 0x22,
+ CPG_PERF_SEL_MIU_READ_REQUEST_SENT = 0x23,
+ CPG_PERF_SEL_CE_STALL_RAM_DUMP = 0x24,
+ CPG_PERF_SEL_CE_STALL_RAM_WRITE = 0x25,
+ CPG_PERF_SEL_CE_STALL_ON_INC_FIFO = 0x26,
+ CPG_PERF_SEL_CE_STALL_ON_WR_RAM_FIFO = 0x27,
+ CPG_PERF_SEL_CE_STALL_ON_DATA_FROM_MIU = 0x28,
+ CPG_PERF_SEL_CE_STALL_ON_DATA_FROM_ROQ = 0x29,
+ CPG_PERF_SEL_CE_STALL_ON_CE_BUFFER_FLAG = 0x2a,
+ CPG_PERF_SEL_CE_STALL_ON_DE_COUNTER = 0x2b,
+ CPG_PERF_SEL_TCIU_STALL_WAIT_ON_FREE = 0x2c,
+ CPG_PERF_SEL_TCIU_STALL_WAIT_ON_TAGS = 0x2d,
+ CPG_PERF_SEL_ATCL2IU_STALL_WAIT_ON_FREE = 0x2e,
+ CPG_PERF_SEL_ATCL2IU_STALL_WAIT_ON_TAGS = 0x2f,
+ CPG_PERF_SEL_ATCL1_STALL_ON_TRANSLATION = 0x30,
+} CPG_PERFCOUNT_SEL;
+typedef enum CPF_PERFCOUNT_SEL {
+ CPF_PERF_SEL_ALWAYS_COUNT = 0x0,
+ CPF_PERF_SEL_MIU_STALLED_WAITING_RDREQ_FREE = 0x1,
+ CPF_PERF_SEL_TCIU_STALLED_WAITING_ON_FREE = 0x2,
+ CPF_PERF_SEL_TCIU_STALLED_WAITING_ON_TAGS = 0x3,
+ CPF_PERF_SEL_CSF_BUSY_FOR_FETCHING_RING = 0x4,
+ CPF_PERF_SEL_CSF_BUSY_FOR_FETCHING_IB1 = 0x5,
+ CPF_PERF_SEL_CSF_BUSY_FOR_FETCHING_IB2 = 0x6,
+ CPF_PERF_SEL_CSF_BUSY_FOR_FECTHINC_STATE = 0x7,
+ CPF_PERF_SEL_MIU_BUSY_FOR_OUTSTANDING_TAGS = 0x8,
+ CPF_PERF_SEL_CSF_RTS_MIU_NOT_RTR = 0x9,
+ CPF_PERF_SEL_CSF_STATE_FIFO_NOT_RTR = 0xa,
+ CPF_PERF_SEL_CSF_FETCHING_CMD_BUFFERS = 0xb,
+ CPF_PERF_SEL_GRBM_DWORDS_SENT = 0xc,
+ CPF_PERF_SEL_DYNAMIC_CLOCK_VALID = 0xd,
+ CPF_PERF_SEL_REGISTER_CLOCK_VALID = 0xe,
+ CPF_PERF_SEL_MIU_WRITE_REQUEST_SEND = 0xf,
+ CPF_PERF_SEL_MIU_READ_REQUEST_SEND = 0x10,
+ CPF_PERF_SEL_ATCL2IU_STALL_WAIT_ON_FREE = 0x11,
+ CPF_PERF_SEL_ATCL2IU_STALL_WAIT_ON_TAGS = 0x12,
+ CPF_PERF_SEL_ATCL1_STALL_ON_TRANSLATION = 0x13,
+} CPF_PERFCOUNT_SEL;
+typedef enum CPC_PERFCOUNT_SEL {
+ CPC_PERF_SEL_ALWAYS_COUNT = 0x0,
+ CPC_PERF_SEL_RCIU_STALL_WAIT_ON_FREE = 0x1,
+ CPC_PERF_SEL_RCIU_STALL_PRIV_VIOLATION = 0x2,
+ CPC_PERF_SEL_MIU_STALL_ON_RDREQ_FREE = 0x3,
+ CPC_PERF_SEL_MIU_STALL_ON_WRREQ_FREE = 0x4,
+ CPC_PERF_SEL_TCIU_STALL_WAIT_ON_FREE = 0x5,
+ CPC_PERF_SEL_ME1_STALL_WAIT_ON_RCIU_READY = 0x6,
+ CPC_PERF_SEL_ME1_STALL_WAIT_ON_RCIU_READY_PERF = 0x7,
+ CPC_PERF_SEL_ME1_STALL_WAIT_ON_RCIU_READ = 0x8,
+ CPC_PERF_SEL_ME1_STALL_WAIT_ON_MIU_READ = 0x9,
+ CPC_PERF_SEL_ME1_STALL_WAIT_ON_MIU_WRITE = 0xa,
+ CPC_PERF_SEL_ME1_STALL_ON_DATA_FROM_ROQ = 0xb,
+ CPC_PERF_SEL_ME1_STALL_ON_DATA_FROM_ROQ_PERF = 0xc,
+ CPC_PERF_SEL_ME1_BUSY_FOR_PACKET_DECODE = 0xd,
+ CPC_PERF_SEL_ME2_STALL_WAIT_ON_RCIU_READY = 0xe,
+ CPC_PERF_SEL_ME2_STALL_WAIT_ON_RCIU_READY_PERF = 0xf,
+ CPC_PERF_SEL_ME2_STALL_WAIT_ON_RCIU_READ = 0x10,
+ CPC_PERF_SEL_ME2_STALL_WAIT_ON_MIU_READ = 0x11,
+ CPC_PERF_SEL_ME2_STALL_WAIT_ON_MIU_WRITE = 0x12,
+ CPC_PERF_SEL_ME2_STALL_ON_DATA_FROM_ROQ = 0x13,
+ CPC_PERF_SEL_ME2_STALL_ON_DATA_FROM_ROQ_PERF = 0x14,
+ CPC_PERF_SEL_ME2_BUSY_FOR_PACKET_DECODE = 0x15,
+ CPC_PERF_SEL_ATCL2IU_STALL_WAIT_ON_FREE = 0x16,
+ CPC_PERF_SEL_ATCL2IU_STALL_WAIT_ON_TAGS = 0x17,
+ CPC_PERF_SEL_ATCL1_STALL_ON_TRANSLATION = 0x18,
+} CPC_PERFCOUNT_SEL;
+typedef enum CP_ALPHA_TAG_RAM_SEL {
+ CPG_TAG_RAM = 0x0,
+ CPC_TAG_RAM = 0x1,
+ CPF_TAG_RAM = 0x2,
+ RSV_TAG_RAM = 0x3,
+} CP_ALPHA_TAG_RAM_SEL;
+#define SEM_ECC_ERROR 0x0
+#define SEM_RESERVED 0x1
+#define SEM_FAILED 0x2
+#define SEM_PASSED 0x3
+#define IQ_QUEUE_SLEEP 0x0
+#define IQ_OFFLOAD_RETRY 0x1
+#define IQ_SCH_WAVE_MSG 0x2
+#define IQ_SEM_REARM 0x3
+#define IQ_DEQUEUE_RETRY 0x4
+#define IQ_INTR_TYPE_PQ 0x0
+#define IQ_INTR_TYPE_IB 0x1
+#define IQ_INTR_TYPE_MQD 0x2
+#define VMID_SZ 0x4
+#define CONFIG_SPACE_START 0x2000
+#define CONFIG_SPACE_END 0x9fff
+#define CONFIG_SPACE1_START 0x2000
+#define CONFIG_SPACE1_END 0x2bff
+#define CONFIG_SPACE2_START 0x3000
+#define CONFIG_SPACE2_END 0x9fff
+#define UCONFIG_SPACE_START 0xc000
+#define UCONFIG_SPACE_END 0xffff
+#define PERSISTENT_SPACE_START 0x2c00
+#define PERSISTENT_SPACE_END 0x2fff
+#define CONTEXT_SPACE_START 0xa000
+#define CONTEXT_SPACE_END 0xbfff
+typedef enum ForceControl {
+ FORCE_OFF = 0x0,
+ FORCE_ENABLE = 0x1,
+ FORCE_DISABLE = 0x2,
+ FORCE_RESERVED = 0x3,
+} ForceControl;
+typedef enum ZSamplePosition {
+ Z_SAMPLE_CENTER = 0x0,
+ Z_SAMPLE_CENTROID = 0x1,
+} ZSamplePosition;
+typedef enum ZOrder {
+ LATE_Z = 0x0,
+ EARLY_Z_THEN_LATE_Z = 0x1,
+ RE_Z = 0x2,
+ EARLY_Z_THEN_RE_Z = 0x3,
+} ZOrder;
+typedef enum ZpassControl {
+ ZPASS_DISABLE = 0x0,
+ ZPASS_SAMPLES = 0x1,
+ ZPASS_PIXELS = 0x2,
+} ZpassControl;
+typedef enum ZModeForce {
+ NO_FORCE = 0x0,
+ FORCE_EARLY_Z = 0x1,
+ FORCE_LATE_Z = 0x2,
+ FORCE_RE_Z = 0x3,
+} ZModeForce;
+typedef enum ZLimitSumm {
+ FORCE_SUMM_OFF = 0x0,
+ FORCE_SUMM_MINZ = 0x1,
+ FORCE_SUMM_MAXZ = 0x2,
+ FORCE_SUMM_BOTH = 0x3,
+} ZLimitSumm;
+typedef enum CompareFrag {
+ FRAG_NEVER = 0x0,
+ FRAG_LESS = 0x1,
+ FRAG_EQUAL = 0x2,
+ FRAG_LEQUAL = 0x3,
+ FRAG_GREATER = 0x4,
+ FRAG_NOTEQUAL = 0x5,
+ FRAG_GEQUAL = 0x6,
+ FRAG_ALWAYS = 0x7,
+} CompareFrag;
+typedef enum StencilOp {
+ STENCIL_KEEP = 0x0,
+ STENCIL_ZERO = 0x1,
+ STENCIL_ONES = 0x2,
+ STENCIL_REPLACE_TEST = 0x3,
+ STENCIL_REPLACE_OP = 0x4,
+ STENCIL_ADD_CLAMP = 0x5,
+ STENCIL_SUB_CLAMP = 0x6,
+ STENCIL_INVERT = 0x7,
+ STENCIL_ADD_WRAP = 0x8,
+ STENCIL_SUB_WRAP = 0x9,
+ STENCIL_AND = 0xa,
+ STENCIL_OR = 0xb,
+ STENCIL_XOR = 0xc,
+ STENCIL_NAND = 0xd,
+ STENCIL_NOR = 0xe,
+ STENCIL_XNOR = 0xf,
+} StencilOp;
+typedef enum ConservativeZExport {
+ EXPORT_ANY_Z = 0x0,
+ EXPORT_LESS_THAN_Z = 0x1,
+ EXPORT_GREATER_THAN_Z = 0x2,
+ EXPORT_RESERVED = 0x3,
+} ConservativeZExport;
+typedef enum DbPSLControl {
+ PSLC_AUTO = 0x0,
+ PSLC_ON_HANG_ONLY = 0x1,
+ PSLC_ASAP = 0x2,
+ PSLC_COUNTDOWN = 0x3,
+} DbPSLControl;
+typedef enum PerfCounter_Vals {
+ DB_PERF_SEL_SC_DB_tile_sends = 0x0,
+ DB_PERF_SEL_SC_DB_tile_busy = 0x1,
+ DB_PERF_SEL_SC_DB_tile_stalls = 0x2,
+ DB_PERF_SEL_SC_DB_tile_events = 0x3,
+ DB_PERF_SEL_SC_DB_tile_tiles = 0x4,
+ DB_PERF_SEL_SC_DB_tile_covered = 0x5,
+ DB_PERF_SEL_hiz_tc_read_starved = 0x6,
+ DB_PERF_SEL_hiz_tc_write_stall = 0x7,
+ DB_PERF_SEL_hiz_qtiles_culled = 0x8,
+ DB_PERF_SEL_his_qtiles_culled = 0x9,
+ DB_PERF_SEL_DB_SC_tile_sends = 0xa,
+ DB_PERF_SEL_DB_SC_tile_busy = 0xb,
+ DB_PERF_SEL_DB_SC_tile_stalls = 0xc,
+ DB_PERF_SEL_DB_SC_tile_df_stalls = 0xd,
+ DB_PERF_SEL_DB_SC_tile_tiles = 0xe,
+ DB_PERF_SEL_DB_SC_tile_culled = 0xf,
+ DB_PERF_SEL_DB_SC_tile_hier_kill = 0x10,
+ DB_PERF_SEL_DB_SC_tile_fast_ops = 0x11,
+ DB_PERF_SEL_DB_SC_tile_no_ops = 0x12,
+ DB_PERF_SEL_DB_SC_tile_tile_rate = 0x13,
+ DB_PERF_SEL_DB_SC_tile_ssaa_kill = 0x14,
+ DB_PERF_SEL_DB_SC_tile_fast_z_ops = 0x15,
+ DB_PERF_SEL_DB_SC_tile_fast_stencil_ops = 0x16,
+ DB_PERF_SEL_SC_DB_quad_sends = 0x17,
+ DB_PERF_SEL_SC_DB_quad_busy = 0x18,
+ DB_PERF_SEL_SC_DB_quad_squads = 0x19,
+ DB_PERF_SEL_SC_DB_quad_tiles = 0x1a,
+ DB_PERF_SEL_SC_DB_quad_pixels = 0x1b,
+ DB_PERF_SEL_SC_DB_quad_killed_tiles = 0x1c,
+ DB_PERF_SEL_DB_SC_quad_sends = 0x1d,
+ DB_PERF_SEL_DB_SC_quad_busy = 0x1e,
+ DB_PERF_SEL_DB_SC_quad_stalls = 0x1f,
+ DB_PERF_SEL_DB_SC_quad_tiles = 0x20,
+ DB_PERF_SEL_DB_SC_quad_lit_quad = 0x21,
+ DB_PERF_SEL_DB_CB_tile_sends = 0x22,
+ DB_PERF_SEL_DB_CB_tile_busy = 0x23,
+ DB_PERF_SEL_DB_CB_tile_stalls = 0x24,
+ DB_PERF_SEL_SX_DB_quad_sends = 0x25,
+ DB_PERF_SEL_SX_DB_quad_busy = 0x26,
+ DB_PERF_SEL_SX_DB_quad_stalls = 0x27,
+ DB_PERF_SEL_SX_DB_quad_quads = 0x28,
+ DB_PERF_SEL_SX_DB_quad_pixels = 0x29,
+ DB_PERF_SEL_SX_DB_quad_exports = 0x2a,
+ DB_PERF_SEL_SH_quads_outstanding_sum = 0x2b,
+ DB_PERF_SEL_DB_CB_lquad_sends = 0x2c,
+ DB_PERF_SEL_DB_CB_lquad_busy = 0x2d,
+ DB_PERF_SEL_DB_CB_lquad_stalls = 0x2e,
+ DB_PERF_SEL_DB_CB_lquad_quads = 0x2f,
+ DB_PERF_SEL_tile_rd_sends = 0x30,
+ DB_PERF_SEL_mi_tile_rd_outstanding_sum = 0x31,
+ DB_PERF_SEL_quad_rd_sends = 0x32,
+ DB_PERF_SEL_quad_rd_busy = 0x33,
+ DB_PERF_SEL_quad_rd_mi_stall = 0x34,
+ DB_PERF_SEL_quad_rd_rw_collision = 0x35,
+ DB_PERF_SEL_quad_rd_tag_stall = 0x36,
+ DB_PERF_SEL_quad_rd_32byte_reqs = 0x37,
+ DB_PERF_SEL_quad_rd_panic = 0x38,
+ DB_PERF_SEL_mi_quad_rd_outstanding_sum = 0x39,
+ DB_PERF_SEL_quad_rdret_sends = 0x3a,
+ DB_PERF_SEL_quad_rdret_busy = 0x3b,
+ DB_PERF_SEL_tile_wr_sends = 0x3c,
+ DB_PERF_SEL_tile_wr_acks = 0x3d,
+ DB_PERF_SEL_mi_tile_wr_outstanding_sum = 0x3e,
+ DB_PERF_SEL_quad_wr_sends = 0x3f,
+ DB_PERF_SEL_quad_wr_busy = 0x40,
+ DB_PERF_SEL_quad_wr_mi_stall = 0x41,
+ DB_PERF_SEL_quad_wr_coherency_stall = 0x42,
+ DB_PERF_SEL_quad_wr_acks = 0x43,
+ DB_PERF_SEL_mi_quad_wr_outstanding_sum = 0x44,
+ DB_PERF_SEL_Tile_Cache_misses = 0x45,
+ DB_PERF_SEL_Tile_Cache_hits = 0x46,
+ DB_PERF_SEL_Tile_Cache_flushes = 0x47,
+ DB_PERF_SEL_Tile_Cache_surface_stall = 0x48,
+ DB_PERF_SEL_Tile_Cache_starves = 0x49,
+ DB_PERF_SEL_Tile_Cache_mem_return_starve = 0x4a,
+ DB_PERF_SEL_tcp_dispatcher_reads = 0x4b,
+ DB_PERF_SEL_tcp_prefetcher_reads = 0x4c,
+ DB_PERF_SEL_tcp_preloader_reads = 0x4d,
+ DB_PERF_SEL_tcp_dispatcher_flushes = 0x4e,
+ DB_PERF_SEL_tcp_prefetcher_flushes = 0x4f,
+ DB_PERF_SEL_tcp_preloader_flushes = 0x50,
+ DB_PERF_SEL_Depth_Tile_Cache_sends = 0x51,
+ DB_PERF_SEL_Depth_Tile_Cache_busy = 0x52,
+ DB_PERF_SEL_Depth_Tile_Cache_starves = 0x53,
+ DB_PERF_SEL_Depth_Tile_Cache_dtile_locked = 0x54,
+ DB_PERF_SEL_Depth_Tile_Cache_alloc_stall = 0x55,
+ DB_PERF_SEL_Depth_Tile_Cache_misses = 0x56,
+ DB_PERF_SEL_Depth_Tile_Cache_hits = 0x57,
+ DB_PERF_SEL_Depth_Tile_Cache_flushes = 0x58,
+ DB_PERF_SEL_Depth_Tile_Cache_noop_tile = 0x59,
+ DB_PERF_SEL_Depth_Tile_Cache_detailed_noop = 0x5a,
+ DB_PERF_SEL_Depth_Tile_Cache_event = 0x5b,
+ DB_PERF_SEL_Depth_Tile_Cache_tile_frees = 0x5c,
+ DB_PERF_SEL_Depth_Tile_Cache_data_frees = 0x5d,
+ DB_PERF_SEL_Depth_Tile_Cache_mem_return_starve = 0x5e,
+ DB_PERF_SEL_Stencil_Cache_misses = 0x5f,
+ DB_PERF_SEL_Stencil_Cache_hits = 0x60,
+ DB_PERF_SEL_Stencil_Cache_flushes = 0x61,
+ DB_PERF_SEL_Stencil_Cache_starves = 0x62,
+ DB_PERF_SEL_Stencil_Cache_frees = 0x63,
+ DB_PERF_SEL_Z_Cache_separate_Z_misses = 0x64,
+ DB_PERF_SEL_Z_Cache_separate_Z_hits = 0x65,
+ DB_PERF_SEL_Z_Cache_separate_Z_flushes = 0x66,
+ DB_PERF_SEL_Z_Cache_separate_Z_starves = 0x67,
+ DB_PERF_SEL_Z_Cache_pmask_misses = 0x68,
+ DB_PERF_SEL_Z_Cache_pmask_hits = 0x69,
+ DB_PERF_SEL_Z_Cache_pmask_flushes = 0x6a,
+ DB_PERF_SEL_Z_Cache_pmask_starves = 0x6b,
+ DB_PERF_SEL_Z_Cache_frees = 0x6c,
+ DB_PERF_SEL_Plane_Cache_misses = 0x6d,
+ DB_PERF_SEL_Plane_Cache_hits = 0x6e,
+ DB_PERF_SEL_Plane_Cache_flushes = 0x6f,
+ DB_PERF_SEL_Plane_Cache_starves = 0x70,
+ DB_PERF_SEL_Plane_Cache_frees = 0x71,
+ DB_PERF_SEL_flush_expanded_stencil = 0x72,
+ DB_PERF_SEL_flush_compressed_stencil = 0x73,
+ DB_PERF_SEL_flush_single_stencil = 0x74,
+ DB_PERF_SEL_planes_flushed = 0x75,
+ DB_PERF_SEL_flush_1plane = 0x76,
+ DB_PERF_SEL_flush_2plane = 0x77,
+ DB_PERF_SEL_flush_3plane = 0x78,
+ DB_PERF_SEL_flush_4plane = 0x79,
+ DB_PERF_SEL_flush_5plane = 0x7a,
+ DB_PERF_SEL_flush_6plane = 0x7b,
+ DB_PERF_SEL_flush_7plane = 0x7c,
+ DB_PERF_SEL_flush_8plane = 0x7d,
+ DB_PERF_SEL_flush_9plane = 0x7e,
+ DB_PERF_SEL_flush_10plane = 0x7f,
+ DB_PERF_SEL_flush_11plane = 0x80,
+ DB_PERF_SEL_flush_12plane = 0x81,
+ DB_PERF_SEL_flush_13plane = 0x82,
+ DB_PERF_SEL_flush_14plane = 0x83,
+ DB_PERF_SEL_flush_15plane = 0x84,
+ DB_PERF_SEL_flush_16plane = 0x85,
+ DB_PERF_SEL_flush_expanded_z = 0x86,
+ DB_PERF_SEL_earlyZ_waiting_for_postZ_done = 0x87,
+ DB_PERF_SEL_reZ_waiting_for_postZ_done = 0x88,
+ DB_PERF_SEL_dk_tile_sends = 0x89,
+ DB_PERF_SEL_dk_tile_busy = 0x8a,
+ DB_PERF_SEL_dk_tile_quad_starves = 0x8b,
+ DB_PERF_SEL_dk_tile_stalls = 0x8c,
+ DB_PERF_SEL_dk_squad_sends = 0x8d,
+ DB_PERF_SEL_dk_squad_busy = 0x8e,
+ DB_PERF_SEL_dk_squad_stalls = 0x8f,
+ DB_PERF_SEL_Op_Pipe_Busy = 0x90,
+ DB_PERF_SEL_Op_Pipe_MC_Read_stall = 0x91,
+ DB_PERF_SEL_qc_busy = 0x92,
+ DB_PERF_SEL_qc_xfc = 0x93,
+ DB_PERF_SEL_qc_conflicts = 0x94,
+ DB_PERF_SEL_qc_full_stall = 0x95,
+ DB_PERF_SEL_qc_in_preZ_tile_stalls_postZ = 0x96,
+ DB_PERF_SEL_qc_in_postZ_tile_stalls_preZ = 0x97,
+ DB_PERF_SEL_tsc_insert_summarize_stall = 0x98,
+ DB_PERF_SEL_tl_busy = 0x99,
+ DB_PERF_SEL_tl_dtc_read_starved = 0x9a,
+ DB_PERF_SEL_tl_z_fetch_stall = 0x9b,
+ DB_PERF_SEL_tl_stencil_stall = 0x9c,
+ DB_PERF_SEL_tl_z_decompress_stall = 0x9d,
+ DB_PERF_SEL_tl_stencil_locked_stall = 0x9e,
+ DB_PERF_SEL_tl_events = 0x9f,
+ DB_PERF_SEL_tl_summarize_squads = 0xa0,
+ DB_PERF_SEL_tl_flush_expand_squads = 0xa1,
+ DB_PERF_SEL_tl_expand_squads = 0xa2,
+ DB_PERF_SEL_tl_preZ_squads = 0xa3,
+ DB_PERF_SEL_tl_postZ_squads = 0xa4,
+ DB_PERF_SEL_tl_preZ_noop_squads = 0xa5,
+ DB_PERF_SEL_tl_postZ_noop_squads = 0xa6,
+ DB_PERF_SEL_tl_tile_ops = 0xa7,
+ DB_PERF_SEL_tl_in_xfc = 0xa8,
+ DB_PERF_SEL_tl_in_single_stencil_expand_stall = 0xa9,
+ DB_PERF_SEL_tl_in_fast_z_stall = 0xaa,
+ DB_PERF_SEL_tl_out_xfc = 0xab,
+ DB_PERF_SEL_tl_out_squads = 0xac,
+ DB_PERF_SEL_zf_plane_multicycle = 0xad,
+ DB_PERF_SEL_PostZ_Samples_passing_Z = 0xae,
+ DB_PERF_SEL_PostZ_Samples_failing_Z = 0xaf,
+ DB_PERF_SEL_PostZ_Samples_failing_S = 0xb0,
+ DB_PERF_SEL_PreZ_Samples_passing_Z = 0xb1,
+ DB_PERF_SEL_PreZ_Samples_failing_Z = 0xb2,
+ DB_PERF_SEL_PreZ_Samples_failing_S = 0xb3,
+ DB_PERF_SEL_ts_tc_update_stall = 0xb4,
+ DB_PERF_SEL_sc_kick_start = 0xb5,
+ DB_PERF_SEL_sc_kick_end = 0xb6,
+ DB_PERF_SEL_clock_reg_active = 0xb7,
+ DB_PERF_SEL_clock_main_active = 0xb8,
+ DB_PERF_SEL_clock_mem_export_active = 0xb9,
+ DB_PERF_SEL_esr_ps_out_busy = 0xba,
+ DB_PERF_SEL_esr_ps_lqf_busy = 0xbb,
+ DB_PERF_SEL_esr_ps_lqf_stall = 0xbc,
+ DB_PERF_SEL_etr_out_send = 0xbd,
+ DB_PERF_SEL_etr_out_busy = 0xbe,
+ DB_PERF_SEL_etr_out_ltile_probe_fifo_full_stall = 0xbf,
+ DB_PERF_SEL_etr_out_cb_tile_stall = 0xc0,
+ DB_PERF_SEL_etr_out_esr_stall = 0xc1,
+ DB_PERF_SEL_esr_ps_sqq_busy = 0xc2,
+ DB_PERF_SEL_esr_ps_sqq_stall = 0xc3,
+ DB_PERF_SEL_esr_eot_fwd_busy = 0xc4,
+ DB_PERF_SEL_esr_eot_fwd_holding_squad = 0xc5,
+ DB_PERF_SEL_esr_eot_fwd_forward = 0xc6,
+ DB_PERF_SEL_esr_sqq_zi_busy = 0xc7,
+ DB_PERF_SEL_esr_sqq_zi_stall = 0xc8,
+ DB_PERF_SEL_postzl_sq_pt_busy = 0xc9,
+ DB_PERF_SEL_postzl_sq_pt_stall = 0xca,
+ DB_PERF_SEL_postzl_se_busy = 0xcb,
+ DB_PERF_SEL_postzl_se_stall = 0xcc,
+ DB_PERF_SEL_postzl_partial_launch = 0xcd,
+ DB_PERF_SEL_postzl_full_launch = 0xce,
+ DB_PERF_SEL_postzl_partial_waiting = 0xcf,
+ DB_PERF_SEL_postzl_tile_mem_stall = 0xd0,
+ DB_PERF_SEL_postzl_tile_init_stall = 0xd1,
+ DB_PEFF_SEL_prezl_tile_mem_stall = 0xd2,
+ DB_PERF_SEL_prezl_tile_init_stall = 0xd3,
+ DB_PERF_SEL_dtt_sm_clash_stall = 0xd4,
+ DB_PERF_SEL_dtt_sm_slot_stall = 0xd5,
+ DB_PERF_SEL_dtt_sm_miss_stall = 0xd6,
+ DB_PERF_SEL_mi_rdreq_busy = 0xd7,
+ DB_PERF_SEL_mi_rdreq_stall = 0xd8,
+ DB_PERF_SEL_mi_wrreq_busy = 0xd9,
+ DB_PERF_SEL_mi_wrreq_stall = 0xda,
+ DB_PERF_SEL_recomp_tile_to_1zplane_no_fastop = 0xdb,
+ DB_PERF_SEL_dkg_tile_rate_tile = 0xdc,
+ DB_PERF_SEL_prezl_src_in_sends = 0xdd,
+ DB_PERF_SEL_prezl_src_in_stall = 0xde,
+ DB_PERF_SEL_prezl_src_in_squads = 0xdf,
+ DB_PERF_SEL_prezl_src_in_squads_unrolled = 0xe0,
+ DB_PERF_SEL_prezl_src_in_tile_rate = 0xe1,
+ DB_PERF_SEL_prezl_src_in_tile_rate_unrolled = 0xe2,
+ DB_PERF_SEL_prezl_src_out_stall = 0xe3,
+ DB_PERF_SEL_postzl_src_in_sends = 0xe4,
+ DB_PERF_SEL_postzl_src_in_stall = 0xe5,
+ DB_PERF_SEL_postzl_src_in_squads = 0xe6,
+ DB_PERF_SEL_postzl_src_in_squads_unrolled = 0xe7,
+ DB_PERF_SEL_postzl_src_in_tile_rate = 0xe8,
+ DB_PERF_SEL_postzl_src_in_tile_rate_unrolled = 0xe9,
+ DB_PERF_SEL_postzl_src_out_stall = 0xea,
+ DB_PERF_SEL_esr_ps_src_in_sends = 0xeb,
+ DB_PERF_SEL_esr_ps_src_in_stall = 0xec,
+ DB_PERF_SEL_esr_ps_src_in_squads = 0xed,
+ DB_PERF_SEL_esr_ps_src_in_squads_unrolled = 0xee,
+ DB_PERF_SEL_esr_ps_src_in_tile_rate = 0xef,
+ DB_PERF_SEL_esr_ps_src_in_tile_rate_unrolled = 0xf0,
+ DB_PERF_SEL_esr_ps_src_in_tile_rate_unrolled_to_pixel_rate= 0xf1,
+ DB_PERF_SEL_esr_ps_src_out_stall = 0xf2,
+ DB_PERF_SEL_depth_bounds_qtiles_culled = 0xf3,
+ DB_PERF_SEL_PreZ_Samples_failing_DB = 0xf4,
+ DB_PERF_SEL_PostZ_Samples_failing_DB = 0xf5,
+ DB_PERF_SEL_flush_compressed = 0xf6,
+ DB_PERF_SEL_flush_plane_le4 = 0xf7,
+ DB_PERF_SEL_tiles_z_fully_summarized = 0xf8,
+ DB_PERF_SEL_tiles_stencil_fully_summarized = 0xf9,
+ DB_PERF_SEL_tiles_z_clear_on_expclear = 0xfa,
+ DB_PERF_SEL_tiles_s_clear_on_expclear = 0xfb,
+ DB_PERF_SEL_tiles_decomp_on_expclear = 0xfc,
+ DB_PERF_SEL_tiles_compressed_to_decompressed = 0xfd,
+ DB_PERF_SEL_Op_Pipe_Prez_Busy = 0xfe,
+ DB_PERF_SEL_Op_Pipe_Postz_Busy = 0xff,
+ DB_PERF_SEL_di_dt_stall = 0x100,
+ DB_PERF_SEL_DB_SC_quad_double_quad = 0x101,
+ DB_PERF_SEL_SX_DB_quad_export_quads = 0x102,
+ DB_PERF_SEL_SX_DB_quad_double_format = 0x103,
+ DB_PERF_SEL_SX_DB_quad_fast_format = 0x104,
+ DB_PERF_SEL_SX_DB_quad_slow_format = 0x105,
+ DB_PERF_SEL_DB_CB_lquad_export_quads = 0x106,
+ DB_PERF_SEL_DB_CB_lquad_double_format = 0x107,
+ DB_PERF_SEL_DB_CB_lquad_fast_format = 0x108,
+ DB_PERF_SEL_DB_CB_lquad_slow_format = 0x109,
+} PerfCounter_Vals;
+typedef enum RingCounterControl {
+ COUNTER_RING_SPLIT = 0x0,
+ COUNTER_RING_0 = 0x1,
+ COUNTER_RING_1 = 0x2,
+} RingCounterControl;
+typedef enum PixelPipeCounterId {
+ PIXEL_PIPE_OCCLUSION_COUNT_0 = 0x0,
+ PIXEL_PIPE_OCCLUSION_COUNT_1 = 0x1,
+ PIXEL_PIPE_OCCLUSION_COUNT_2 = 0x2,
+ PIXEL_PIPE_OCCLUSION_COUNT_3 = 0x3,
+ PIXEL_PIPE_SCREEN_MIN_EXTENTS_0 = 0x4,
+ PIXEL_PIPE_SCREEN_MAX_EXTENTS_0 = 0x5,
+ PIXEL_PIPE_SCREEN_MIN_EXTENTS_1 = 0x6,
+ PIXEL_PIPE_SCREEN_MAX_EXTENTS_1 = 0x7,
+} PixelPipeCounterId;
+typedef enum PixelPipeStride {
+ PIXEL_PIPE_STRIDE_32_BITS = 0x0,
+ PIXEL_PIPE_STRIDE_64_BITS = 0x1,
+ PIXEL_PIPE_STRIDE_128_BITS = 0x2,
+ PIXEL_PIPE_STRIDE_256_BITS = 0x3,
+} PixelPipeStride;
+typedef enum GB_EDC_DED_MODE {
+ GB_EDC_DED_MODE_LOG = 0x0,
+ GB_EDC_DED_MODE_HALT = 0x1,
+ GB_EDC_DED_MODE_INT_HALT = 0x2,
+} GB_EDC_DED_MODE;
+#define GB_TILING_CONFIG_TABLE_SIZE 0x20
+#define GB_TILING_CONFIG_MACROTABLE_SIZE 0x10
+typedef enum GRBM_PERF_SEL {
+ GRBM_PERF_SEL_COUNT = 0x0,
+ GRBM_PERF_SEL_USER_DEFINED = 0x1,
+ GRBM_PERF_SEL_GUI_ACTIVE = 0x2,
+ GRBM_PERF_SEL_CP_BUSY = 0x3,
+ GRBM_PERF_SEL_CP_COHER_BUSY = 0x4,
+ GRBM_PERF_SEL_CP_DMA_BUSY = 0x5,
+ GRBM_PERF_SEL_CB_BUSY = 0x6,
+ GRBM_PERF_SEL_DB_BUSY = 0x7,
+ GRBM_PERF_SEL_PA_BUSY = 0x8,
+ GRBM_PERF_SEL_SC_BUSY = 0x9,
+ GRBM_PERF_SEL_RESERVED_6 = 0xa,
+ GRBM_PERF_SEL_SPI_BUSY = 0xb,
+ GRBM_PERF_SEL_SX_BUSY = 0xc,
+ GRBM_PERF_SEL_TA_BUSY = 0xd,
+ GRBM_PERF_SEL_CB_CLEAN = 0xe,
+ GRBM_PERF_SEL_DB_CLEAN = 0xf,
+ GRBM_PERF_SEL_RESERVED_5 = 0x10,
+ GRBM_PERF_SEL_VGT_BUSY = 0x11,
+ GRBM_PERF_SEL_RESERVED_4 = 0x12,
+ GRBM_PERF_SEL_RESERVED_3 = 0x13,
+ GRBM_PERF_SEL_RESERVED_2 = 0x14,
+ GRBM_PERF_SEL_RESERVED_1 = 0x15,
+ GRBM_PERF_SEL_RESERVED_0 = 0x16,
+ GRBM_PERF_SEL_IA_BUSY = 0x17,
+ GRBM_PERF_SEL_IA_NO_DMA_BUSY = 0x18,
+ GRBM_PERF_SEL_GDS_BUSY = 0x19,
+ GRBM_PERF_SEL_BCI_BUSY = 0x1a,
+ GRBM_PERF_SEL_RLC_BUSY = 0x1b,
+ GRBM_PERF_SEL_TC_BUSY = 0x1c,
+ GRBM_PERF_SEL_CPG_BUSY = 0x1d,
+ GRBM_PERF_SEL_CPC_BUSY = 0x1e,
+ GRBM_PERF_SEL_CPF_BUSY = 0x1f,
+ GRBM_PERF_SEL_WD_BUSY = 0x20,
+ GRBM_PERF_SEL_WD_NO_DMA_BUSY = 0x21,
+} GRBM_PERF_SEL;
+typedef enum GRBM_SE0_PERF_SEL {
+ GRBM_SE0_PERF_SEL_COUNT = 0x0,
+ GRBM_SE0_PERF_SEL_USER_DEFINED = 0x1,
+ GRBM_SE0_PERF_SEL_CB_BUSY = 0x2,
+ GRBM_SE0_PERF_SEL_DB_BUSY = 0x3,
+ GRBM_SE0_PERF_SEL_SC_BUSY = 0x4,
+ GRBM_SE0_PERF_SEL_RESERVED_1 = 0x5,
+ GRBM_SE0_PERF_SEL_SPI_BUSY = 0x6,
+ GRBM_SE0_PERF_SEL_SX_BUSY = 0x7,
+ GRBM_SE0_PERF_SEL_TA_BUSY = 0x8,
+ GRBM_SE0_PERF_SEL_CB_CLEAN = 0x9,
+ GRBM_SE0_PERF_SEL_DB_CLEAN = 0xa,
+ GRBM_SE0_PERF_SEL_RESERVED_0 = 0xb,
+ GRBM_SE0_PERF_SEL_PA_BUSY = 0xc,
+ GRBM_SE0_PERF_SEL_VGT_BUSY = 0xd,
+ GRBM_SE0_PERF_SEL_BCI_BUSY = 0xe,
+} GRBM_SE0_PERF_SEL;
+typedef enum GRBM_SE1_PERF_SEL {
+ GRBM_SE1_PERF_SEL_COUNT = 0x0,
+ GRBM_SE1_PERF_SEL_USER_DEFINED = 0x1,
+ GRBM_SE1_PERF_SEL_CB_BUSY = 0x2,
+ GRBM_SE1_PERF_SEL_DB_BUSY = 0x3,
+ GRBM_SE1_PERF_SEL_SC_BUSY = 0x4,
+ GRBM_SE1_PERF_SEL_RESERVED_1 = 0x5,
+ GRBM_SE1_PERF_SEL_SPI_BUSY = 0x6,
+ GRBM_SE1_PERF_SEL_SX_BUSY = 0x7,
+ GRBM_SE1_PERF_SEL_TA_BUSY = 0x8,
+ GRBM_SE1_PERF_SEL_CB_CLEAN = 0x9,
+ GRBM_SE1_PERF_SEL_DB_CLEAN = 0xa,
+ GRBM_SE1_PERF_SEL_RESERVED_0 = 0xb,
+ GRBM_SE1_PERF_SEL_PA_BUSY = 0xc,
+ GRBM_SE1_PERF_SEL_VGT_BUSY = 0xd,
+ GRBM_SE1_PERF_SEL_BCI_BUSY = 0xe,
+} GRBM_SE1_PERF_SEL;
+typedef enum GRBM_SE2_PERF_SEL {
+ GRBM_SE2_PERF_SEL_COUNT = 0x0,
+ GRBM_SE2_PERF_SEL_USER_DEFINED = 0x1,
+ GRBM_SE2_PERF_SEL_CB_BUSY = 0x2,
+ GRBM_SE2_PERF_SEL_DB_BUSY = 0x3,
+ GRBM_SE2_PERF_SEL_SC_BUSY = 0x4,
+ GRBM_SE2_PERF_SEL_RESERVED_1 = 0x5,
+ GRBM_SE2_PERF_SEL_SPI_BUSY = 0x6,
+ GRBM_SE2_PERF_SEL_SX_BUSY = 0x7,
+ GRBM_SE2_PERF_SEL_TA_BUSY = 0x8,
+ GRBM_SE2_PERF_SEL_CB_CLEAN = 0x9,
+ GRBM_SE2_PERF_SEL_DB_CLEAN = 0xa,
+ GRBM_SE2_PERF_SEL_RESERVED_0 = 0xb,
+ GRBM_SE2_PERF_SEL_PA_BUSY = 0xc,
+ GRBM_SE2_PERF_SEL_VGT_BUSY = 0xd,
+ GRBM_SE2_PERF_SEL_BCI_BUSY = 0xe,
+} GRBM_SE2_PERF_SEL;
+typedef enum GRBM_SE3_PERF_SEL {
+ GRBM_SE3_PERF_SEL_COUNT = 0x0,
+ GRBM_SE3_PERF_SEL_USER_DEFINED = 0x1,
+ GRBM_SE3_PERF_SEL_CB_BUSY = 0x2,
+ GRBM_SE3_PERF_SEL_DB_BUSY = 0x3,
+ GRBM_SE3_PERF_SEL_SC_BUSY = 0x4,
+ GRBM_SE3_PERF_SEL_RESERVED_1 = 0x5,
+ GRBM_SE3_PERF_SEL_SPI_BUSY = 0x6,
+ GRBM_SE3_PERF_SEL_SX_BUSY = 0x7,
+ GRBM_SE3_PERF_SEL_TA_BUSY = 0x8,
+ GRBM_SE3_PERF_SEL_CB_CLEAN = 0x9,
+ GRBM_SE3_PERF_SEL_DB_CLEAN = 0xa,
+ GRBM_SE3_PERF_SEL_RESERVED_0 = 0xb,
+ GRBM_SE3_PERF_SEL_PA_BUSY = 0xc,
+ GRBM_SE3_PERF_SEL_VGT_BUSY = 0xd,
+ GRBM_SE3_PERF_SEL_BCI_BUSY = 0xe,
+} GRBM_SE3_PERF_SEL;
+typedef enum SU_PERFCNT_SEL {
+ PERF_PAPC_PASX_REQ = 0x0,
+ PERF_PAPC_PASX_DISABLE_PIPE = 0x1,
+ PERF_PAPC_PASX_FIRST_VECTOR = 0x2,
+ PERF_PAPC_PASX_SECOND_VECTOR = 0x3,
+ PERF_PAPC_PASX_FIRST_DEAD = 0x4,
+ PERF_PAPC_PASX_SECOND_DEAD = 0x5,
+ PERF_PAPC_PASX_VTX_KILL_DISCARD = 0x6,
+ PERF_PAPC_PASX_VTX_NAN_DISCARD = 0x7,
+ PERF_PAPC_PA_INPUT_PRIM = 0x8,
+ PERF_PAPC_PA_INPUT_NULL_PRIM = 0x9,
+ PERF_PAPC_PA_INPUT_EVENT_FLAG = 0xa,
+ PERF_PAPC_PA_INPUT_FIRST_PRIM_SLOT = 0xb,
+ PERF_PAPC_PA_INPUT_END_OF_PACKET = 0xc,
+ PERF_PAPC_PA_INPUT_EXTENDED_EVENT = 0xd,
+ PERF_PAPC_CLPR_CULL_PRIM = 0xe,
+ PERF_PAPC_CLPR_VVUCP_CULL_PRIM = 0xf,
+ PERF_PAPC_CLPR_VV_CULL_PRIM = 0x10,
+ PERF_PAPC_CLPR_UCP_CULL_PRIM = 0x11,
+ PERF_PAPC_CLPR_VTX_KILL_CULL_PRIM = 0x12,
+ PERF_PAPC_CLPR_VTX_NAN_CULL_PRIM = 0x13,
+ PERF_PAPC_CLPR_CULL_TO_NULL_PRIM = 0x14,
+ PERF_PAPC_CLPR_VVUCP_CLIP_PRIM = 0x15,
+ PERF_PAPC_CLPR_VV_CLIP_PRIM = 0x16,
+ PERF_PAPC_CLPR_UCP_CLIP_PRIM = 0x17,
+ PERF_PAPC_CLPR_POINT_CLIP_CANDIDATE = 0x18,
+ PERF_PAPC_CLPR_CLIP_PLANE_CNT_1 = 0x19,
+ PERF_PAPC_CLPR_CLIP_PLANE_CNT_2 = 0x1a,
+ PERF_PAPC_CLPR_CLIP_PLANE_CNT_3 = 0x1b,
+ PERF_PAPC_CLPR_CLIP_PLANE_CNT_4 = 0x1c,
+ PERF_PAPC_CLPR_CLIP_PLANE_CNT_5_8 = 0x1d,
+ PERF_PAPC_CLPR_CLIP_PLANE_CNT_9_12 = 0x1e,
+ PERF_PAPC_CLPR_CLIP_PLANE_NEAR = 0x1f,
+ PERF_PAPC_CLPR_CLIP_PLANE_FAR = 0x20,
+ PERF_PAPC_CLPR_CLIP_PLANE_LEFT = 0x21,
+ PERF_PAPC_CLPR_CLIP_PLANE_RIGHT = 0x22,
+ PERF_PAPC_CLPR_CLIP_PLANE_TOP = 0x23,
+ PERF_PAPC_CLPR_CLIP_PLANE_BOTTOM = 0x24,
+ PERF_PAPC_CLPR_GSC_KILL_CULL_PRIM = 0x25,
+ PERF_PAPC_CLPR_RASTER_KILL_CULL_PRIM = 0x26,
+ PERF_PAPC_CLSM_NULL_PRIM = 0x27,
+ PERF_PAPC_CLSM_TOTALLY_VISIBLE_PRIM = 0x28,
+ PERF_PAPC_CLSM_CULL_TO_NULL_PRIM = 0x29,
+ PERF_PAPC_CLSM_OUT_PRIM_CNT_1 = 0x2a,
+ PERF_PAPC_CLSM_OUT_PRIM_CNT_2 = 0x2b,
+ PERF_PAPC_CLSM_OUT_PRIM_CNT_3 = 0x2c,
+ PERF_PAPC_CLSM_OUT_PRIM_CNT_4 = 0x2d,
+ PERF_PAPC_CLSM_OUT_PRIM_CNT_5_8 = 0x2e,
+ PERF_PAPC_CLSM_OUT_PRIM_CNT_9_13 = 0x2f,
+ PERF_PAPC_CLIPGA_VTE_KILL_PRIM = 0x30,
+ PERF_PAPC_SU_INPUT_PRIM = 0x31,
+ PERF_PAPC_SU_INPUT_CLIP_PRIM = 0x32,
+ PERF_PAPC_SU_INPUT_NULL_PRIM = 0x33,
+ PERF_PAPC_SU_INPUT_PRIM_DUAL = 0x34,
+ PERF_PAPC_SU_INPUT_CLIP_PRIM_DUAL = 0x35,
+ PERF_PAPC_SU_ZERO_AREA_CULL_PRIM = 0x36,
+ PERF_PAPC_SU_BACK_FACE_CULL_PRIM = 0x37,
+ PERF_PAPC_SU_FRONT_FACE_CULL_PRIM = 0x38,
+ PERF_PAPC_SU_POLYMODE_FACE_CULL = 0x39,
+ PERF_PAPC_SU_POLYMODE_BACK_CULL = 0x3a,
+ PERF_PAPC_SU_POLYMODE_FRONT_CULL = 0x3b,
+ PERF_PAPC_SU_POLYMODE_INVALID_FILL = 0x3c,
+ PERF_PAPC_SU_OUTPUT_PRIM = 0x3d,
+ PERF_PAPC_SU_OUTPUT_CLIP_PRIM = 0x3e,
+ PERF_PAPC_SU_OUTPUT_NULL_PRIM = 0x3f,
+ PERF_PAPC_SU_OUTPUT_EVENT_FLAG = 0x40,
+ PERF_PAPC_SU_OUTPUT_FIRST_PRIM_SLOT = 0x41,
+ PERF_PAPC_SU_OUTPUT_END_OF_PACKET = 0x42,
+ PERF_PAPC_SU_OUTPUT_POLYMODE_FACE = 0x43,
+ PERF_PAPC_SU_OUTPUT_POLYMODE_BACK = 0x44,
+ PERF_PAPC_SU_OUTPUT_POLYMODE_FRONT = 0x45,
+ PERF_PAPC_SU_OUT_CLIP_POLYMODE_FACE = 0x46,
+ PERF_PAPC_SU_OUT_CLIP_POLYMODE_BACK = 0x47,
+ PERF_PAPC_SU_OUT_CLIP_POLYMODE_FRONT = 0x48,
+ PERF_PAPC_SU_OUTPUT_PRIM_DUAL = 0x49,
+ PERF_PAPC_SU_OUTPUT_CLIP_PRIM_DUAL = 0x4a,
+ PERF_PAPC_SU_OUTPUT_POLYMODE_DUAL = 0x4b,
+ PERF_PAPC_SU_OUTPUT_CLIP_POLYMODE_DUAL = 0x4c,
+ PERF_PAPC_PASX_REQ_IDLE = 0x4d,
+ PERF_PAPC_PASX_REQ_BUSY = 0x4e,
+ PERF_PAPC_PASX_REQ_STALLED = 0x4f,
+ PERF_PAPC_PASX_REC_IDLE = 0x50,
+ PERF_PAPC_PASX_REC_BUSY = 0x51,
+ PERF_PAPC_PASX_REC_STARVED_SX = 0x52,
+ PERF_PAPC_PASX_REC_STALLED = 0x53,
+ PERF_PAPC_PASX_REC_STALLED_POS_MEM = 0x54,
+ PERF_PAPC_PASX_REC_STALLED_CCGSM_IN = 0x55,
+ PERF_PAPC_CCGSM_IDLE = 0x56,
+ PERF_PAPC_CCGSM_BUSY = 0x57,
+ PERF_PAPC_CCGSM_STALLED = 0x58,
+ PERF_PAPC_CLPRIM_IDLE = 0x59,
+ PERF_PAPC_CLPRIM_BUSY = 0x5a,
+ PERF_PAPC_CLPRIM_STALLED = 0x5b,
+ PERF_PAPC_CLPRIM_STARVED_CCGSM = 0x5c,
+ PERF_PAPC_CLIPSM_IDLE = 0x5d,
+ PERF_PAPC_CLIPSM_BUSY = 0x5e,
+ PERF_PAPC_CLIPSM_WAIT_CLIP_VERT_ENGH = 0x5f,
+ PERF_PAPC_CLIPSM_WAIT_HIGH_PRI_SEQ = 0x60,
+ PERF_PAPC_CLIPSM_WAIT_CLIPGA = 0x61,
+ PERF_PAPC_CLIPSM_WAIT_AVAIL_VTE_CLIP = 0x62,
+ PERF_PAPC_CLIPSM_WAIT_CLIP_OUTSM = 0x63,
+ PERF_PAPC_CLIPGA_IDLE = 0x64,
+ PERF_PAPC_CLIPGA_BUSY = 0x65,
+ PERF_PAPC_CLIPGA_STARVED_VTE_CLIP = 0x66,
+ PERF_PAPC_CLIPGA_STALLED = 0x67,
+ PERF_PAPC_CLIP_IDLE = 0x68,
+ PERF_PAPC_CLIP_BUSY = 0x69,
+ PERF_PAPC_SU_IDLE = 0x6a,
+ PERF_PAPC_SU_BUSY = 0x6b,
+ PERF_PAPC_SU_STARVED_CLIP = 0x6c,
+ PERF_PAPC_SU_STALLED_SC = 0x6d,
+ PERF_PAPC_CL_DYN_SCLK_VLD = 0x6e,
+ PERF_PAPC_SU_DYN_SCLK_VLD = 0x6f,
+ PERF_PAPC_PA_REG_SCLK_VLD = 0x70,
+ PERF_PAPC_SU_MULTI_GPU_PRIM_FILTER_CULL = 0x71,
+ PERF_PAPC_PASX_SE0_REQ = 0x72,
+ PERF_PAPC_PASX_SE1_REQ = 0x73,
+ PERF_PAPC_PASX_SE0_FIRST_VECTOR = 0x74,
+ PERF_PAPC_PASX_SE0_SECOND_VECTOR = 0x75,
+ PERF_PAPC_PASX_SE1_FIRST_VECTOR = 0x76,
+ PERF_PAPC_PASX_SE1_SECOND_VECTOR = 0x77,
+ PERF_PAPC_SU_SE0_PRIM_FILTER_CULL = 0x78,
+ PERF_PAPC_SU_SE1_PRIM_FILTER_CULL = 0x79,
+ PERF_PAPC_SU_SE01_PRIM_FILTER_CULL = 0x7a,
+ PERF_PAPC_SU_SE0_OUTPUT_PRIM = 0x7b,
+ PERF_PAPC_SU_SE1_OUTPUT_PRIM = 0x7c,
+ PERF_PAPC_SU_SE01_OUTPUT_PRIM = 0x7d,
+ PERF_PAPC_SU_SE0_OUTPUT_NULL_PRIM = 0x7e,
+ PERF_PAPC_SU_SE1_OUTPUT_NULL_PRIM = 0x7f,
+ PERF_PAPC_SU_SE01_OUTPUT_NULL_PRIM = 0x80,
+ PERF_PAPC_SU_SE0_OUTPUT_FIRST_PRIM_SLOT = 0x81,
+ PERF_PAPC_SU_SE1_OUTPUT_FIRST_PRIM_SLOT = 0x82,
+ PERF_PAPC_SU_SE0_STALLED_SC = 0x83,
+ PERF_PAPC_SU_SE1_STALLED_SC = 0x84,
+ PERF_PAPC_SU_SE01_STALLED_SC = 0x85,
+ PERF_PAPC_CLSM_CLIPPING_PRIM = 0x86,
+ PERF_PAPC_SU_CULLED_PRIM = 0x87,
+ PERF_PAPC_SU_OUTPUT_EOPG = 0x88,
+ PERF_PAPC_SU_SE2_PRIM_FILTER_CULL = 0x89,
+ PERF_PAPC_SU_SE3_PRIM_FILTER_CULL = 0x8a,
+ PERF_PAPC_SU_SE2_OUTPUT_PRIM = 0x8b,
+ PERF_PAPC_SU_SE3_OUTPUT_PRIM = 0x8c,
+ PERF_PAPC_SU_SE2_OUTPUT_NULL_PRIM = 0x8d,
+ PERF_PAPC_SU_SE3_OUTPUT_NULL_PRIM = 0x8e,
+ PERF_PAPC_SU_SE0_OUTPUT_END_OF_PACKET = 0x8f,
+ PERF_PAPC_SU_SE1_OUTPUT_END_OF_PACKET = 0x90,
+ PERF_PAPC_SU_SE2_OUTPUT_END_OF_PACKET = 0x91,
+ PERF_PAPC_SU_SE3_OUTPUT_END_OF_PACKET = 0x92,
+ PERF_PAPC_SU_SE0_OUTPUT_EOPG = 0x93,
+ PERF_PAPC_SU_SE1_OUTPUT_EOPG = 0x94,
+ PERF_PAPC_SU_SE2_OUTPUT_EOPG = 0x95,
+ PERF_PAPC_SU_SE3_OUTPUT_EOPG = 0x96,
+ PERF_PAPC_SU_SE2_STALLED_SC = 0x97,
+ PERF_PAPC_SU_SE3_STALLED_SC = 0x98,
+} SU_PERFCNT_SEL;
+typedef enum SC_PERFCNT_SEL {
+ SC_SRPS_WINDOW_VALID = 0x0,
+ SC_PSSW_WINDOW_VALID = 0x1,
+ SC_TPQZ_WINDOW_VALID = 0x2,
+ SC_QZQP_WINDOW_VALID = 0x3,
+ SC_TRPK_WINDOW_VALID = 0x4,
+ SC_SRPS_WINDOW_VALID_BUSY = 0x5,
+ SC_PSSW_WINDOW_VALID_BUSY = 0x6,
+ SC_TPQZ_WINDOW_VALID_BUSY = 0x7,
+ SC_QZQP_WINDOW_VALID_BUSY = 0x8,
+ SC_TRPK_WINDOW_VALID_BUSY = 0x9,
+ SC_STARVED_BY_PA = 0xa,
+ SC_STALLED_BY_PRIMFIFO = 0xb,
+ SC_STALLED_BY_DB_TILE = 0xc,
+ SC_STARVED_BY_DB_TILE = 0xd,
+ SC_STALLED_BY_TILEORDERFIFO = 0xe,
+ SC_STALLED_BY_TILEFIFO = 0xf,
+ SC_STALLED_BY_DB_QUAD = 0x10,
+ SC_STARVED_BY_DB_QUAD = 0x11,
+ SC_STALLED_BY_QUADFIFO = 0x12,
+ SC_STALLED_BY_BCI = 0x13,
+ SC_STALLED_BY_SPI = 0x14,
+ SC_SCISSOR_DISCARD = 0x15,
+ SC_BB_DISCARD = 0x16,
+ SC_SUPERTILE_COUNT = 0x17,
+ SC_SUPERTILE_PER_PRIM_H0 = 0x18,
+ SC_SUPERTILE_PER_PRIM_H1 = 0x19,
+ SC_SUPERTILE_PER_PRIM_H2 = 0x1a,
+ SC_SUPERTILE_PER_PRIM_H3 = 0x1b,
+ SC_SUPERTILE_PER_PRIM_H4 = 0x1c,
+ SC_SUPERTILE_PER_PRIM_H5 = 0x1d,
+ SC_SUPERTILE_PER_PRIM_H6 = 0x1e,
+ SC_SUPERTILE_PER_PRIM_H7 = 0x1f,
+ SC_SUPERTILE_PER_PRIM_H8 = 0x20,
+ SC_SUPERTILE_PER_PRIM_H9 = 0x21,
+ SC_SUPERTILE_PER_PRIM_H10 = 0x22,
+ SC_SUPERTILE_PER_PRIM_H11 = 0x23,
+ SC_SUPERTILE_PER_PRIM_H12 = 0x24,
+ SC_SUPERTILE_PER_PRIM_H13 = 0x25,
+ SC_SUPERTILE_PER_PRIM_H14 = 0x26,
+ SC_SUPERTILE_PER_PRIM_H15 = 0x27,
+ SC_SUPERTILE_PER_PRIM_H16 = 0x28,
+ SC_TILE_PER_PRIM_H0 = 0x29,
+ SC_TILE_PER_PRIM_H1 = 0x2a,
+ SC_TILE_PER_PRIM_H2 = 0x2b,
+ SC_TILE_PER_PRIM_H3 = 0x2c,
+ SC_TILE_PER_PRIM_H4 = 0x2d,
+ SC_TILE_PER_PRIM_H5 = 0x2e,
+ SC_TILE_PER_PRIM_H6 = 0x2f,
+ SC_TILE_PER_PRIM_H7 = 0x30,
+ SC_TILE_PER_PRIM_H8 = 0x31,
+ SC_TILE_PER_PRIM_H9 = 0x32,
+ SC_TILE_PER_PRIM_H10 = 0x33,
+ SC_TILE_PER_PRIM_H11 = 0x34,
+ SC_TILE_PER_PRIM_H12 = 0x35,
+ SC_TILE_PER_PRIM_H13 = 0x36,
+ SC_TILE_PER_PRIM_H14 = 0x37,
+ SC_TILE_PER_PRIM_H15 = 0x38,
+ SC_TILE_PER_PRIM_H16 = 0x39,
+ SC_TILE_PER_SUPERTILE_H0 = 0x3a,
+ SC_TILE_PER_SUPERTILE_H1 = 0x3b,
+ SC_TILE_PER_SUPERTILE_H2 = 0x3c,
+ SC_TILE_PER_SUPERTILE_H3 = 0x3d,
+ SC_TILE_PER_SUPERTILE_H4 = 0x3e,
+ SC_TILE_PER_SUPERTILE_H5 = 0x3f,
+ SC_TILE_PER_SUPERTILE_H6 = 0x40,
+ SC_TILE_PER_SUPERTILE_H7 = 0x41,
+ SC_TILE_PER_SUPERTILE_H8 = 0x42,
+ SC_TILE_PER_SUPERTILE_H9 = 0x43,
+ SC_TILE_PER_SUPERTILE_H10 = 0x44,
+ SC_TILE_PER_SUPERTILE_H11 = 0x45,
+ SC_TILE_PER_SUPERTILE_H12 = 0x46,
+ SC_TILE_PER_SUPERTILE_H13 = 0x47,
+ SC_TILE_PER_SUPERTILE_H14 = 0x48,
+ SC_TILE_PER_SUPERTILE_H15 = 0x49,
+ SC_TILE_PER_SUPERTILE_H16 = 0x4a,
+ SC_TILE_PICKED_H1 = 0x4b,
+ SC_TILE_PICKED_H2 = 0x4c,
+ SC_TILE_PICKED_H3 = 0x4d,
+ SC_TILE_PICKED_H4 = 0x4e,
+ SC_QZ0_MULTI_GPU_TILE_DISCARD = 0x4f,
+ SC_QZ1_MULTI_GPU_TILE_DISCARD = 0x50,
+ SC_QZ2_MULTI_GPU_TILE_DISCARD = 0x51,
+ SC_QZ3_MULTI_GPU_TILE_DISCARD = 0x52,
+ SC_QZ0_TILE_COUNT = 0x53,
+ SC_QZ1_TILE_COUNT = 0x54,
+ SC_QZ2_TILE_COUNT = 0x55,
+ SC_QZ3_TILE_COUNT = 0x56,
+ SC_QZ0_TILE_COVERED_COUNT = 0x57,
+ SC_QZ1_TILE_COVERED_COUNT = 0x58,
+ SC_QZ2_TILE_COVERED_COUNT = 0x59,
+ SC_QZ3_TILE_COVERED_COUNT = 0x5a,
+ SC_QZ0_TILE_NOT_COVERED_COUNT = 0x5b,
+ SC_QZ1_TILE_NOT_COVERED_COUNT = 0x5c,
+ SC_QZ2_TILE_NOT_COVERED_COUNT = 0x5d,
+ SC_QZ3_TILE_NOT_COVERED_COUNT = 0x5e,
+ SC_QZ0_QUAD_PER_TILE_H0 = 0x5f,
+ SC_QZ0_QUAD_PER_TILE_H1 = 0x60,
+ SC_QZ0_QUAD_PER_TILE_H2 = 0x61,
+ SC_QZ0_QUAD_PER_TILE_H3 = 0x62,
+ SC_QZ0_QUAD_PER_TILE_H4 = 0x63,
+ SC_QZ0_QUAD_PER_TILE_H5 = 0x64,
+ SC_QZ0_QUAD_PER_TILE_H6 = 0x65,
+ SC_QZ0_QUAD_PER_TILE_H7 = 0x66,
+ SC_QZ0_QUAD_PER_TILE_H8 = 0x67,
+ SC_QZ0_QUAD_PER_TILE_H9 = 0x68,
+ SC_QZ0_QUAD_PER_TILE_H10 = 0x69,
+ SC_QZ0_QUAD_PER_TILE_H11 = 0x6a,
+ SC_QZ0_QUAD_PER_TILE_H12 = 0x6b,
+ SC_QZ0_QUAD_PER_TILE_H13 = 0x6c,
+ SC_QZ0_QUAD_PER_TILE_H14 = 0x6d,
+ SC_QZ0_QUAD_PER_TILE_H15 = 0x6e,
+ SC_QZ0_QUAD_PER_TILE_H16 = 0x6f,
+ SC_QZ1_QUAD_PER_TILE_H0 = 0x70,
+ SC_QZ1_QUAD_PER_TILE_H1 = 0x71,
+ SC_QZ1_QUAD_PER_TILE_H2 = 0x72,
+ SC_QZ1_QUAD_PER_TILE_H3 = 0x73,
+ SC_QZ1_QUAD_PER_TILE_H4 = 0x74,
+ SC_QZ1_QUAD_PER_TILE_H5 = 0x75,
+ SC_QZ1_QUAD_PER_TILE_H6 = 0x76,
+ SC_QZ1_QUAD_PER_TILE_H7 = 0x77,
+ SC_QZ1_QUAD_PER_TILE_H8 = 0x78,
+ SC_QZ1_QUAD_PER_TILE_H9 = 0x79,
+ SC_QZ1_QUAD_PER_TILE_H10 = 0x7a,
+ SC_QZ1_QUAD_PER_TILE_H11 = 0x7b,
+ SC_QZ1_QUAD_PER_TILE_H12 = 0x7c,
+ SC_QZ1_QUAD_PER_TILE_H13 = 0x7d,
+ SC_QZ1_QUAD_PER_TILE_H14 = 0x7e,
+ SC_QZ1_QUAD_PER_TILE_H15 = 0x7f,
+ SC_QZ1_QUAD_PER_TILE_H16 = 0x80,
+ SC_QZ2_QUAD_PER_TILE_H0 = 0x81,
+ SC_QZ2_QUAD_PER_TILE_H1 = 0x82,
+ SC_QZ2_QUAD_PER_TILE_H2 = 0x83,
+ SC_QZ2_QUAD_PER_TILE_H3 = 0x84,
+ SC_QZ2_QUAD_PER_TILE_H4 = 0x85,
+ SC_QZ2_QUAD_PER_TILE_H5 = 0x86,
+ SC_QZ2_QUAD_PER_TILE_H6 = 0x87,
+ SC_QZ2_QUAD_PER_TILE_H7 = 0x88,
+ SC_QZ2_QUAD_PER_TILE_H8 = 0x89,
+ SC_QZ2_QUAD_PER_TILE_H9 = 0x8a,
+ SC_QZ2_QUAD_PER_TILE_H10 = 0x8b,
+ SC_QZ2_QUAD_PER_TILE_H11 = 0x8c,
+ SC_QZ2_QUAD_PER_TILE_H12 = 0x8d,
+ SC_QZ2_QUAD_PER_TILE_H13 = 0x8e,
+ SC_QZ2_QUAD_PER_TILE_H14 = 0x8f,
+ SC_QZ2_QUAD_PER_TILE_H15 = 0x90,
+ SC_QZ2_QUAD_PER_TILE_H16 = 0x91,
+ SC_QZ3_QUAD_PER_TILE_H0 = 0x92,
+ SC_QZ3_QUAD_PER_TILE_H1 = 0x93,
+ SC_QZ3_QUAD_PER_TILE_H2 = 0x94,
+ SC_QZ3_QUAD_PER_TILE_H3 = 0x95,
+ SC_QZ3_QUAD_PER_TILE_H4 = 0x96,
+ SC_QZ3_QUAD_PER_TILE_H5 = 0x97,
+ SC_QZ3_QUAD_PER_TILE_H6 = 0x98,
+ SC_QZ3_QUAD_PER_TILE_H7 = 0x99,
+ SC_QZ3_QUAD_PER_TILE_H8 = 0x9a,
+ SC_QZ3_QUAD_PER_TILE_H9 = 0x9b,
+ SC_QZ3_QUAD_PER_TILE_H10 = 0x9c,
+ SC_QZ3_QUAD_PER_TILE_H11 = 0x9d,
+ SC_QZ3_QUAD_PER_TILE_H12 = 0x9e,
+ SC_QZ3_QUAD_PER_TILE_H13 = 0x9f,
+ SC_QZ3_QUAD_PER_TILE_H14 = 0xa0,
+ SC_QZ3_QUAD_PER_TILE_H15 = 0xa1,
+ SC_QZ3_QUAD_PER_TILE_H16 = 0xa2,
+ SC_QZ0_QUAD_COUNT = 0xa3,
+ SC_QZ1_QUAD_COUNT = 0xa4,
+ SC_QZ2_QUAD_COUNT = 0xa5,
+ SC_QZ3_QUAD_COUNT = 0xa6,
+ SC_P0_HIZ_TILE_COUNT = 0xa7,
+ SC_P1_HIZ_TILE_COUNT = 0xa8,
+ SC_P2_HIZ_TILE_COUNT = 0xa9,
+ SC_P3_HIZ_TILE_COUNT = 0xaa,
+ SC_P0_HIZ_QUAD_PER_TILE_H0 = 0xab,
+ SC_P0_HIZ_QUAD_PER_TILE_H1 = 0xac,
+ SC_P0_HIZ_QUAD_PER_TILE_H2 = 0xad,
+ SC_P0_HIZ_QUAD_PER_TILE_H3 = 0xae,
+ SC_P0_HIZ_QUAD_PER_TILE_H4 = 0xaf,
+ SC_P0_HIZ_QUAD_PER_TILE_H5 = 0xb0,
+ SC_P0_HIZ_QUAD_PER_TILE_H6 = 0xb1,
+ SC_P0_HIZ_QUAD_PER_TILE_H7 = 0xb2,
+ SC_P0_HIZ_QUAD_PER_TILE_H8 = 0xb3,
+ SC_P0_HIZ_QUAD_PER_TILE_H9 = 0xb4,
+ SC_P0_HIZ_QUAD_PER_TILE_H10 = 0xb5,
+ SC_P0_HIZ_QUAD_PER_TILE_H11 = 0xb6,
+ SC_P0_HIZ_QUAD_PER_TILE_H12 = 0xb7,
+ SC_P0_HIZ_QUAD_PER_TILE_H13 = 0xb8,
+ SC_P0_HIZ_QUAD_PER_TILE_H14 = 0xb9,
+ SC_P0_HIZ_QUAD_PER_TILE_H15 = 0xba,
+ SC_P0_HIZ_QUAD_PER_TILE_H16 = 0xbb,
+ SC_P1_HIZ_QUAD_PER_TILE_H0 = 0xbc,
+ SC_P1_HIZ_QUAD_PER_TILE_H1 = 0xbd,
+ SC_P1_HIZ_QUAD_PER_TILE_H2 = 0xbe,
+ SC_P1_HIZ_QUAD_PER_TILE_H3 = 0xbf,
+ SC_P1_HIZ_QUAD_PER_TILE_H4 = 0xc0,
+ SC_P1_HIZ_QUAD_PER_TILE_H5 = 0xc1,
+ SC_P1_HIZ_QUAD_PER_TILE_H6 = 0xc2,
+ SC_P1_HIZ_QUAD_PER_TILE_H7 = 0xc3,
+ SC_P1_HIZ_QUAD_PER_TILE_H8 = 0xc4,
+ SC_P1_HIZ_QUAD_PER_TILE_H9 = 0xc5,
+ SC_P1_HIZ_QUAD_PER_TILE_H10 = 0xc6,
+ SC_P1_HIZ_QUAD_PER_TILE_H11 = 0xc7,
+ SC_P1_HIZ_QUAD_PER_TILE_H12 = 0xc8,
+ SC_P1_HIZ_QUAD_PER_TILE_H13 = 0xc9,
+ SC_P1_HIZ_QUAD_PER_TILE_H14 = 0xca,
+ SC_P1_HIZ_QUAD_PER_TILE_H15 = 0xcb,
+ SC_P1_HIZ_QUAD_PER_TILE_H16 = 0xcc,
+ SC_P2_HIZ_QUAD_PER_TILE_H0 = 0xcd,
+ SC_P2_HIZ_QUAD_PER_TILE_H1 = 0xce,
+ SC_P2_HIZ_QUAD_PER_TILE_H2 = 0xcf,
+ SC_P2_HIZ_QUAD_PER_TILE_H3 = 0xd0,
+ SC_P2_HIZ_QUAD_PER_TILE_H4 = 0xd1,
+ SC_P2_HIZ_QUAD_PER_TILE_H5 = 0xd2,
+ SC_P2_HIZ_QUAD_PER_TILE_H6 = 0xd3,
+ SC_P2_HIZ_QUAD_PER_TILE_H7 = 0xd4,
+ SC_P2_HIZ_QUAD_PER_TILE_H8 = 0xd5,
+ SC_P2_HIZ_QUAD_PER_TILE_H9 = 0xd6,
+ SC_P2_HIZ_QUAD_PER_TILE_H10 = 0xd7,
+ SC_P2_HIZ_QUAD_PER_TILE_H11 = 0xd8,
+ SC_P2_HIZ_QUAD_PER_TILE_H12 = 0xd9,
+ SC_P2_HIZ_QUAD_PER_TILE_H13 = 0xda,
+ SC_P2_HIZ_QUAD_PER_TILE_H14 = 0xdb,
+ SC_P2_HIZ_QUAD_PER_TILE_H15 = 0xdc,
+ SC_P2_HIZ_QUAD_PER_TILE_H16 = 0xdd,
+ SC_P3_HIZ_QUAD_PER_TILE_H0 = 0xde,
+ SC_P3_HIZ_QUAD_PER_TILE_H1 = 0xdf,
+ SC_P3_HIZ_QUAD_PER_TILE_H2 = 0xe0,
+ SC_P3_HIZ_QUAD_PER_TILE_H3 = 0xe1,
+ SC_P3_HIZ_QUAD_PER_TILE_H4 = 0xe2,
+ SC_P3_HIZ_QUAD_PER_TILE_H5 = 0xe3,
+ SC_P3_HIZ_QUAD_PER_TILE_H6 = 0xe4,
+ SC_P3_HIZ_QUAD_PER_TILE_H7 = 0xe5,
+ SC_P3_HIZ_QUAD_PER_TILE_H8 = 0xe6,
+ SC_P3_HIZ_QUAD_PER_TILE_H9 = 0xe7,
+ SC_P3_HIZ_QUAD_PER_TILE_H10 = 0xe8,
+ SC_P3_HIZ_QUAD_PER_TILE_H11 = 0xe9,
+ SC_P3_HIZ_QUAD_PER_TILE_H12 = 0xea,
+ SC_P3_HIZ_QUAD_PER_TILE_H13 = 0xeb,
+ SC_P3_HIZ_QUAD_PER_TILE_H14 = 0xec,
+ SC_P3_HIZ_QUAD_PER_TILE_H15 = 0xed,
+ SC_P3_HIZ_QUAD_PER_TILE_H16 = 0xee,
+ SC_P0_HIZ_QUAD_COUNT = 0xef,
+ SC_P1_HIZ_QUAD_COUNT = 0xf0,
+ SC_P2_HIZ_QUAD_COUNT = 0xf1,
+ SC_P3_HIZ_QUAD_COUNT = 0xf2,
+ SC_P0_DETAIL_QUAD_COUNT = 0xf3,
+ SC_P1_DETAIL_QUAD_COUNT = 0xf4,
+ SC_P2_DETAIL_QUAD_COUNT = 0xf5,
+ SC_P3_DETAIL_QUAD_COUNT = 0xf6,
+ SC_P0_DETAIL_QUAD_WITH_1_PIX = 0xf7,
+ SC_P0_DETAIL_QUAD_WITH_2_PIX = 0xf8,
+ SC_P0_DETAIL_QUAD_WITH_3_PIX = 0xf9,
+ SC_P0_DETAIL_QUAD_WITH_4_PIX = 0xfa,
+ SC_P1_DETAIL_QUAD_WITH_1_PIX = 0xfb,
+ SC_P1_DETAIL_QUAD_WITH_2_PIX = 0xfc,
+ SC_P1_DETAIL_QUAD_WITH_3_PIX = 0xfd,
+ SC_P1_DETAIL_QUAD_WITH_4_PIX = 0xfe,
+ SC_P2_DETAIL_QUAD_WITH_1_PIX = 0xff,
+ SC_P2_DETAIL_QUAD_WITH_2_PIX = 0x100,
+ SC_P2_DETAIL_QUAD_WITH_3_PIX = 0x101,
+ SC_P2_DETAIL_QUAD_WITH_4_PIX = 0x102,
+ SC_P3_DETAIL_QUAD_WITH_1_PIX = 0x103,
+ SC_P3_DETAIL_QUAD_WITH_2_PIX = 0x104,
+ SC_P3_DETAIL_QUAD_WITH_3_PIX = 0x105,
+ SC_P3_DETAIL_QUAD_WITH_4_PIX = 0x106,
+ SC_EARLYZ_QUAD_COUNT = 0x107,
+ SC_EARLYZ_QUAD_WITH_1_PIX = 0x108,
+ SC_EARLYZ_QUAD_WITH_2_PIX = 0x109,
+ SC_EARLYZ_QUAD_WITH_3_PIX = 0x10a,
+ SC_EARLYZ_QUAD_WITH_4_PIX = 0x10b,
+ SC_PKR_QUAD_PER_ROW_H1 = 0x10c,
+ SC_PKR_QUAD_PER_ROW_H2 = 0x10d,
+ SC_PKR_4X2_QUAD_SPLIT = 0x10e,
+ SC_PKR_4X2_FILL_QUAD = 0x10f,
+ SC_PKR_END_OF_VECTOR = 0x110,
+ SC_PKR_CONTROL_XFER = 0x111,
+ SC_PKR_DBHANG_FORCE_EOV = 0x112,
+ SC_REG_SCLK_BUSY = 0x113,
+ SC_GRP0_DYN_SCLK_BUSY = 0x114,
+ SC_GRP1_DYN_SCLK_BUSY = 0x115,
+ SC_GRP2_DYN_SCLK_BUSY = 0x116,
+ SC_GRP3_DYN_SCLK_BUSY = 0x117,
+ SC_GRP4_DYN_SCLK_BUSY = 0x118,
+ SC_PA0_SC_DATA_FIFO_RD = 0x119,
+ SC_PA0_SC_DATA_FIFO_WE = 0x11a,
+ SC_PA1_SC_DATA_FIFO_RD = 0x11b,
+ SC_PA1_SC_DATA_FIFO_WE = 0x11c,
+ SC_PS_ARB_XFC_ALL_EVENT_OR_PRIM_CYCLES = 0x11d,
+ SC_PS_ARB_XFC_ONLY_PRIM_CYCLES = 0x11e,
+ SC_PS_ARB_XFC_ONLY_ONE_INC_PER_PRIM = 0x11f,
+ SC_PS_ARB_STALLED_FROM_BELOW = 0x120,
+ SC_PS_ARB_STARVED_FROM_ABOVE = 0x121,
+ SC_PS_ARB_SC_BUSY = 0x122,
+ SC_PS_ARB_PA_SC_BUSY = 0x123,
+ SC_PA2_SC_DATA_FIFO_RD = 0x124,
+ SC_PA2_SC_DATA_FIFO_WE = 0x125,
+ SC_PA3_SC_DATA_FIFO_RD = 0x126,
+ SC_PA3_SC_DATA_FIFO_WE = 0x127,
+ SC_PA_SC_DEALLOC_0_0_WE = 0x128,
+ SC_PA_SC_DEALLOC_0_1_WE = 0x129,
+ SC_PA_SC_DEALLOC_1_0_WE = 0x12a,
+ SC_PA_SC_DEALLOC_1_1_WE = 0x12b,
+ SC_PA_SC_DEALLOC_2_0_WE = 0x12c,
+ SC_PA_SC_DEALLOC_2_1_WE = 0x12d,
+ SC_PA_SC_DEALLOC_3_0_WE = 0x12e,
+ SC_PA_SC_DEALLOC_3_1_WE = 0x12f,
+ SC_PA0_SC_EOP_WE = 0x130,
+ SC_PA0_SC_EOPG_WE = 0x131,
+ SC_PA0_SC_EVENT_WE = 0x132,
+ SC_PA1_SC_EOP_WE = 0x133,
+ SC_PA1_SC_EOPG_WE = 0x134,
+ SC_PA1_SC_EVENT_WE = 0x135,
+ SC_PA2_SC_EOP_WE = 0x136,
+ SC_PA2_SC_EOPG_WE = 0x137,
+ SC_PA2_SC_EVENT_WE = 0x138,
+ SC_PA3_SC_EOP_WE = 0x139,
+ SC_PA3_SC_EOPG_WE = 0x13a,
+ SC_PA3_SC_EVENT_WE = 0x13b,
+ SC_PS_ARB_OOO_THRESHOLD_SWITCH_TO_DESIRED_FIFO = 0x13c,
+ SC_PS_ARB_OOO_FIFO_EMPTY_SWITCH = 0x13d,
+ SC_PS_ARB_NULL_PRIM_BUBBLE_POP = 0x13e,
+ SC_PS_ARB_EOP_POP_SYNC_POP = 0x13f,
+ SC_PS_ARB_EVENT_SYNC_POP = 0x140,
+ SC_SC_PS_ENG_MULTICYCLE_BUBBLE = 0x141,
+ SC_PA0_SC_FPOV_WE = 0x142,
+ SC_PA1_SC_FPOV_WE = 0x143,
+ SC_PA2_SC_FPOV_WE = 0x144,
+ SC_PA3_SC_FPOV_WE = 0x145,
+ SC_PA0_SC_LPOV_WE = 0x146,
+ SC_PA1_SC_LPOV_WE = 0x147,
+ SC_PA2_SC_LPOV_WE = 0x148,
+ SC_PA3_SC_LPOV_WE = 0x149,
+ SC_SC_SPI_DEALLOC_0_0 = 0x14a,
+ SC_SC_SPI_DEALLOC_0_1 = 0x14b,
+ SC_SC_SPI_DEALLOC_0_2 = 0x14c,
+ SC_SC_SPI_DEALLOC_1_0 = 0x14d,
+ SC_SC_SPI_DEALLOC_1_1 = 0x14e,
+ SC_SC_SPI_DEALLOC_1_2 = 0x14f,
+ SC_SC_SPI_DEALLOC_2_0 = 0x150,
+ SC_SC_SPI_DEALLOC_2_1 = 0x151,
+ SC_SC_SPI_DEALLOC_2_2 = 0x152,
+ SC_SC_SPI_DEALLOC_3_0 = 0x153,
+ SC_SC_SPI_DEALLOC_3_1 = 0x154,
+ SC_SC_SPI_DEALLOC_3_2 = 0x155,
+ SC_SC_SPI_FPOV_0 = 0x156,
+ SC_SC_SPI_FPOV_1 = 0x157,
+ SC_SC_SPI_FPOV_2 = 0x158,
+ SC_SC_SPI_FPOV_3 = 0x159,
+ SC_SC_SPI_EVENT = 0x15a,
+ SC_PS_TS_EVENT_FIFO_PUSH = 0x15b,
+ SC_PS_TS_EVENT_FIFO_POP = 0x15c,
+ SC_PS_CTX_DONE_FIFO_PUSH = 0x15d,
+ SC_PS_CTX_DONE_FIFO_POP = 0x15e,
+ SC_MULTICYCLE_BUBBLE_FREEZE = 0x15f,
+ SC_EOP_SYNC_WINDOW = 0x160,
+ SC_PA0_SC_NULL_WE = 0x161,
+ SC_PA0_SC_NULL_DEALLOC_WE = 0x162,
+ SC_PA0_SC_DATA_FIFO_EOPG_RD = 0x163,
+ SC_PA0_SC_DATA_FIFO_EOP_RD = 0x164,
+ SC_PA0_SC_DEALLOC_0_RD = 0x165,
+ SC_PA0_SC_DEALLOC_1_RD = 0x166,
+ SC_PA1_SC_DATA_FIFO_EOPG_RD = 0x167,
+ SC_PA1_SC_DATA_FIFO_EOP_RD = 0x168,
+ SC_PA1_SC_DEALLOC_0_RD = 0x169,
+ SC_PA1_SC_DEALLOC_1_RD = 0x16a,
+ SC_PA1_SC_NULL_WE = 0x16b,
+ SC_PA1_SC_NULL_DEALLOC_WE = 0x16c,
+ SC_PA2_SC_DATA_FIFO_EOPG_RD = 0x16d,
+ SC_PA2_SC_DATA_FIFO_EOP_RD = 0x16e,
+ SC_PA2_SC_DEALLOC_0_RD = 0x16f,
+ SC_PA2_SC_DEALLOC_1_RD = 0x170,
+ SC_PA2_SC_NULL_WE = 0x171,
+ SC_PA2_SC_NULL_DEALLOC_WE = 0x172,
+ SC_PA3_SC_DATA_FIFO_EOPG_RD = 0x173,
+ SC_PA3_SC_DATA_FIFO_EOP_RD = 0x174,
+ SC_PA3_SC_DEALLOC_0_RD = 0x175,
+ SC_PA3_SC_DEALLOC_1_RD = 0x176,
+ SC_PA3_SC_NULL_WE = 0x177,
+ SC_PA3_SC_NULL_DEALLOC_WE = 0x178,
+ SC_PS_PA0_SC_FIFO_EMPTY = 0x179,
+ SC_PS_PA0_SC_FIFO_FULL = 0x17a,
+ SC_PA0_PS_DATA_SEND = 0x17b,
+ SC_PS_PA1_SC_FIFO_EMPTY = 0x17c,
+ SC_PS_PA1_SC_FIFO_FULL = 0x17d,
+ SC_PA1_PS_DATA_SEND = 0x17e,
+ SC_PS_PA2_SC_FIFO_EMPTY = 0x17f,
+ SC_PS_PA2_SC_FIFO_FULL = 0x180,
+ SC_PA2_PS_DATA_SEND = 0x181,
+ SC_PS_PA3_SC_FIFO_EMPTY = 0x182,
+ SC_PS_PA3_SC_FIFO_FULL = 0x183,
+ SC_PA3_PS_DATA_SEND = 0x184,
+ SC_BUSY_PROCESSING_MULTICYCLE_PRIM = 0x185,
+ SC_BUSY_CNT_NOT_ZERO = 0x186,
+ SC_BM_BUSY = 0x187,
+ SC_BACKEND_BUSY = 0x188,
+ SC_SCF_SCB_INTERFACE_BUSY = 0x189,
+ SC_SCB_BUSY = 0x18a,
+ SC_STARVED_BY_PA_WITH_UNSELECTED_PA_NOT_EMPTY = 0x18b,
+ SC_STARVED_BY_PA_WITH_UNSELECTED_PA_FULL = 0x18c,
+} SC_PERFCNT_SEL;
+typedef enum SePairXsel {
+ RASTER_CONFIG_SE_PAIR_XSEL_8_WIDE_TILE = 0x0,
+ RASTER_CONFIG_SE_PAIR_XSEL_16_WIDE_TILE = 0x1,
+ RASTER_CONFIG_SE_PAIR_XSEL_32_WIDE_TILE = 0x2,
+ RASTER_CONFIG_SE_PAIR_XSEL_64_WIDE_TILE = 0x3,
+} SePairXsel;
+typedef enum SePairYsel {
+ RASTER_CONFIG_SE_PAIR_YSEL_8_WIDE_TILE = 0x0,
+ RASTER_CONFIG_SE_PAIR_YSEL_16_WIDE_TILE = 0x1,
+ RASTER_CONFIG_SE_PAIR_YSEL_32_WIDE_TILE = 0x2,
+ RASTER_CONFIG_SE_PAIR_YSEL_64_WIDE_TILE = 0x3,
+} SePairYsel;
+typedef enum SePairMap {
+ RASTER_CONFIG_SE_PAIR_MAP_0 = 0x0,
+ RASTER_CONFIG_SE_PAIR_MAP_1 = 0x1,
+ RASTER_CONFIG_SE_PAIR_MAP_2 = 0x2,
+ RASTER_CONFIG_SE_PAIR_MAP_3 = 0x3,
+} SePairMap;
+typedef enum SeXsel {
+ RASTER_CONFIG_SE_XSEL_8_WIDE_TILE = 0x0,
+ RASTER_CONFIG_SE_XSEL_16_WIDE_TILE = 0x1,
+ RASTER_CONFIG_SE_XSEL_32_WIDE_TILE = 0x2,
+ RASTER_CONFIG_SE_XSEL_64_WIDE_TILE = 0x3,
+} SeXsel;
+typedef enum SeYsel {
+ RASTER_CONFIG_SE_YSEL_8_WIDE_TILE = 0x0,
+ RASTER_CONFIG_SE_YSEL_16_WIDE_TILE = 0x1,
+ RASTER_CONFIG_SE_YSEL_32_WIDE_TILE = 0x2,
+ RASTER_CONFIG_SE_YSEL_64_WIDE_TILE = 0x3,
+} SeYsel;
+typedef enum SeMap {
+ RASTER_CONFIG_SE_MAP_0 = 0x0,
+ RASTER_CONFIG_SE_MAP_1 = 0x1,
+ RASTER_CONFIG_SE_MAP_2 = 0x2,
+ RASTER_CONFIG_SE_MAP_3 = 0x3,
+} SeMap;
+typedef enum ScXsel {
+ RASTER_CONFIG_SC_XSEL_8_WIDE_TILE = 0x0,
+ RASTER_CONFIG_SC_XSEL_16_WIDE_TILE = 0x1,
+ RASTER_CONFIG_SC_XSEL_32_WIDE_TILE = 0x2,
+ RASTER_CONFIG_SC_XSEL_64_WIDE_TILE = 0x3,
+} ScXsel;
+typedef enum ScYsel {
+ RASTER_CONFIG_SC_YSEL_8_WIDE_TILE = 0x0,
+ RASTER_CONFIG_SC_YSEL_16_WIDE_TILE = 0x1,
+ RASTER_CONFIG_SC_YSEL_32_WIDE_TILE = 0x2,
+ RASTER_CONFIG_SC_YSEL_64_WIDE_TILE = 0x3,
+} ScYsel;
+typedef enum ScMap {
+ RASTER_CONFIG_SC_MAP_0 = 0x0,
+ RASTER_CONFIG_SC_MAP_1 = 0x1,
+ RASTER_CONFIG_SC_MAP_2 = 0x2,
+ RASTER_CONFIG_SC_MAP_3 = 0x3,
+} ScMap;
+typedef enum PkrXsel2 {
+ RASTER_CONFIG_PKR_XSEL2_0 = 0x0,
+ RASTER_CONFIG_PKR_XSEL2_1 = 0x1,
+ RASTER_CONFIG_PKR_XSEL2_2 = 0x2,
+ RASTER_CONFIG_PKR_XSEL2_3 = 0x3,
+} PkrXsel2;
+typedef enum PkrXsel {
+ RASTER_CONFIG_PKR_XSEL_0 = 0x0,
+ RASTER_CONFIG_PKR_XSEL_1 = 0x1,
+ RASTER_CONFIG_PKR_XSEL_2 = 0x2,
+ RASTER_CONFIG_PKR_XSEL_3 = 0x3,
+} PkrXsel;
+typedef enum PkrYsel {
+ RASTER_CONFIG_PKR_YSEL_0 = 0x0,
+ RASTER_CONFIG_PKR_YSEL_1 = 0x1,
+ RASTER_CONFIG_PKR_YSEL_2 = 0x2,
+ RASTER_CONFIG_PKR_YSEL_3 = 0x3,
+} PkrYsel;
+typedef enum PkrMap {
+ RASTER_CONFIG_PKR_MAP_0 = 0x0,
+ RASTER_CONFIG_PKR_MAP_1 = 0x1,
+ RASTER_CONFIG_PKR_MAP_2 = 0x2,
+ RASTER_CONFIG_PKR_MAP_3 = 0x3,
+} PkrMap;
+typedef enum RbXsel {
+ RASTER_CONFIG_RB_XSEL_0 = 0x0,
+ RASTER_CONFIG_RB_XSEL_1 = 0x1,
+} RbXsel;
+typedef enum RbYsel {
+ RASTER_CONFIG_RB_YSEL_0 = 0x0,
+ RASTER_CONFIG_RB_YSEL_1 = 0x1,
+} RbYsel;
+typedef enum RbXsel2 {
+ RASTER_CONFIG_RB_XSEL2_0 = 0x0,
+ RASTER_CONFIG_RB_XSEL2_1 = 0x1,
+ RASTER_CONFIG_RB_XSEL2_2 = 0x2,
+ RASTER_CONFIG_RB_XSEL2_3 = 0x3,
+} RbXsel2;
+typedef enum RbMap {
+ RASTER_CONFIG_RB_MAP_0 = 0x0,
+ RASTER_CONFIG_RB_MAP_1 = 0x1,
+ RASTER_CONFIG_RB_MAP_2 = 0x2,
+ RASTER_CONFIG_RB_MAP_3 = 0x3,
+} RbMap;
+typedef enum CSDATA_TYPE {
+ CSDATA_TYPE_TG = 0x0,
+ CSDATA_TYPE_STATE = 0x1,
+ CSDATA_TYPE_EVENT = 0x2,
+ CSDATA_TYPE_PRIVATE = 0x3,
+} CSDATA_TYPE;
+#define CSDATA_TYPE_WIDTH 0x2
+#define CSDATA_ADDR_WIDTH 0x7
+#define CSDATA_DATA_WIDTH 0x20
+typedef enum SPI_SAMPLE_CNTL {
+ CENTROIDS_ONLY = 0x0,
+ CENTERS_ONLY = 0x1,
+ CENTROIDS_AND_CENTERS = 0x2,
+ UNDEF = 0x3,
+} SPI_SAMPLE_CNTL;
+typedef enum SPI_FOG_MODE {
+ SPI_FOG_NONE = 0x0,
+ SPI_FOG_EXP = 0x1,
+ SPI_FOG_EXP2 = 0x2,
+ SPI_FOG_LINEAR = 0x3,
+} SPI_FOG_MODE;
+typedef enum SPI_PNT_SPRITE_OVERRIDE {
+ SPI_PNT_SPRITE_SEL_0 = 0x0,
+ SPI_PNT_SPRITE_SEL_1 = 0x1,
+ SPI_PNT_SPRITE_SEL_S = 0x2,
+ SPI_PNT_SPRITE_SEL_T = 0x3,
+ SPI_PNT_SPRITE_SEL_NONE = 0x4,
+} SPI_PNT_SPRITE_OVERRIDE;
+typedef enum SPI_PERFCNT_SEL {
+ SPI_PERF_VS_WINDOW_VALID = 0x0,
+ SPI_PERF_VS_BUSY = 0x1,
+ SPI_PERF_VS_FIRST_WAVE = 0x2,
+ SPI_PERF_VS_LAST_WAVE = 0x3,
+ SPI_PERF_VS_LSHS_DEALLOC = 0x4,
+ SPI_PERF_VS_PC_STALL = 0x5,
+ SPI_PERF_VS_POS0_STALL = 0x6,
+ SPI_PERF_VS_POS1_STALL = 0x7,
+ SPI_PERF_VS_CRAWLER_STALL = 0x8,
+ SPI_PERF_VS_EVENT_WAVE = 0x9,
+ SPI_PERF_VS_WAVE = 0xa,
+ SPI_PERF_VS_PERS_UPD_FULL0 = 0xb,
+ SPI_PERF_VS_PERS_UPD_FULL1 = 0xc,
+ SPI_PERF_VS_LATE_ALLOC_FULL = 0xd,
+ SPI_PERF_VS_FIRST_SUBGRP = 0xe,
+ SPI_PERF_VS_LAST_SUBGRP = 0xf,
+ SPI_PERF_GS_WINDOW_VALID = 0x10,
+ SPI_PERF_GS_BUSY = 0x11,
+ SPI_PERF_GS_CRAWLER_STALL = 0x12,
+ SPI_PERF_GS_EVENT_WAVE = 0x13,
+ SPI_PERF_GS_WAVE = 0x14,
+ SPI_PERF_GS_PERS_UPD_FULL0 = 0x15,
+ SPI_PERF_GS_PERS_UPD_FULL1 = 0x16,
+ SPI_PERF_GS_FIRST_SUBGRP = 0x17,
+ SPI_PERF_GS_LAST_SUBGRP = 0x18,
+ SPI_PERF_ES_WINDOW_VALID = 0x19,
+ SPI_PERF_ES_BUSY = 0x1a,
+ SPI_PERF_ES_CRAWLER_STALL = 0x1b,
+ SPI_PERF_ES_FIRST_WAVE = 0x1c,
+ SPI_PERF_ES_LAST_WAVE = 0x1d,
+ SPI_PERF_ES_LSHS_DEALLOC = 0x1e,
+ SPI_PERF_ES_EVENT_WAVE = 0x1f,
+ SPI_PERF_ES_WAVE = 0x20,
+ SPI_PERF_ES_PERS_UPD_FULL0 = 0x21,
+ SPI_PERF_ES_PERS_UPD_FULL1 = 0x22,
+ SPI_PERF_ES_FIRST_SUBGRP = 0x23,
+ SPI_PERF_ES_LAST_SUBGRP = 0x24,
+ SPI_PERF_HS_WINDOW_VALID = 0x25,
+ SPI_PERF_HS_BUSY = 0x26,
+ SPI_PERF_HS_CRAWLER_STALL = 0x27,
+ SPI_PERF_HS_FIRST_WAVE = 0x28,
+ SPI_PERF_HS_LAST_WAVE = 0x29,
+ SPI_PERF_HS_LSHS_DEALLOC = 0x2a,
+ SPI_PERF_HS_EVENT_WAVE = 0x2b,
+ SPI_PERF_HS_WAVE = 0x2c,
+ SPI_PERF_HS_PERS_UPD_FULL0 = 0x2d,
+ SPI_PERF_HS_PERS_UPD_FULL1 = 0x2e,
+ SPI_PERF_LS_WINDOW_VALID = 0x2f,
+ SPI_PERF_LS_BUSY = 0x30,
+ SPI_PERF_LS_CRAWLER_STALL = 0x31,
+ SPI_PERF_LS_FIRST_WAVE = 0x32,
+ SPI_PERF_LS_LAST_WAVE = 0x33,
+ SPI_PERF_OFFCHIP_LDS_STALL_LS = 0x34,
+ SPI_PERF_LS_EVENT_WAVE = 0x35,
+ SPI_PERF_LS_WAVE = 0x36,
+ SPI_PERF_LS_PERS_UPD_FULL0 = 0x37,
+ SPI_PERF_LS_PERS_UPD_FULL1 = 0x38,
+ SPI_PERF_CSG_WINDOW_VALID = 0x39,
+ SPI_PERF_CSG_BUSY = 0x3a,
+ SPI_PERF_CSG_NUM_THREADGROUPS = 0x3b,
+ SPI_PERF_CSG_CRAWLER_STALL = 0x3c,
+ SPI_PERF_CSG_EVENT_WAVE = 0x3d,
+ SPI_PERF_CSG_WAVE = 0x3e,
+ SPI_PERF_CSN_WINDOW_VALID = 0x3f,
+ SPI_PERF_CSN_BUSY = 0x40,
+ SPI_PERF_CSN_NUM_THREADGROUPS = 0x41,
+ SPI_PERF_CSN_CRAWLER_STALL = 0x42,
+ SPI_PERF_CSN_EVENT_WAVE = 0x43,
+ SPI_PERF_CSN_WAVE = 0x44,
+ SPI_PERF_PS_CTL_WINDOW_VALID = 0x45,
+ SPI_PERF_PS_CTL_BUSY = 0x46,
+ SPI_PERF_PS_CTL_ACTIVE = 0x47,
+ SPI_PERF_PS_CTL_DEALLOC_BIN0 = 0x48,
+ SPI_PERF_PS_CTL_FPOS_BIN1_STALL = 0x49,
+ SPI_PERF_PS_CTL_EVENT_WAVE = 0x4a,
+ SPI_PERF_PS_CTL_WAVE = 0x4b,
+ SPI_PERF_PS_CTL_OPT_WAVE = 0x4c,
+ SPI_PERF_PS_CTL_PASS_BIN0 = 0x4d,
+ SPI_PERF_PS_CTL_PASS_BIN1 = 0x4e,
+ SPI_PERF_PS_CTL_FPOS_BIN2 = 0x4f,
+ SPI_PERF_PS_CTL_PRIM_BIN0 = 0x50,
+ SPI_PERF_PS_CTL_PRIM_BIN1 = 0x51,
+ SPI_PERF_PS_CTL_CNF_BIN2 = 0x52,
+ SPI_PERF_PS_CTL_CNF_BIN3 = 0x53,
+ SPI_PERF_PS_CTL_CRAWLER_STALL = 0x54,
+ SPI_PERF_PS_CTL_LDS_RES_FULL = 0x55,
+ SPI_PERF_PS_PERS_UPD_FULL0 = 0x56,
+ SPI_PERF_PS_PERS_UPD_FULL1 = 0x57,
+ SPI_PERF_PIX_ALLOC_PEND_CNT = 0x58,
+ SPI_PERF_PIX_ALLOC_SCB_STALL = 0x59,
+ SPI_PERF_PIX_ALLOC_DB0_STALL = 0x5a,
+ SPI_PERF_PIX_ALLOC_DB1_STALL = 0x5b,
+ SPI_PERF_PIX_ALLOC_DB2_STALL = 0x5c,
+ SPI_PERF_PIX_ALLOC_DB3_STALL = 0x5d,
+ SPI_PERF_LDS0_PC_VALID = 0x5e,
+ SPI_PERF_LDS1_PC_VALID = 0x5f,
+ SPI_PERF_RA_PIPE_REQ_BIN2 = 0x60,
+ SPI_PERF_RA_TASK_REQ_BIN3 = 0x61,
+ SPI_PERF_RA_WR_CTL_FULL = 0x62,
+ SPI_PERF_RA_REQ_NO_ALLOC = 0x63,
+ SPI_PERF_RA_REQ_NO_ALLOC_PS = 0x64,
+ SPI_PERF_RA_REQ_NO_ALLOC_VS = 0x65,
+ SPI_PERF_RA_REQ_NO_ALLOC_GS = 0x66,
+ SPI_PERF_RA_REQ_NO_ALLOC_ES = 0x67,
+ SPI_PERF_RA_REQ_NO_ALLOC_HS = 0x68,
+ SPI_PERF_RA_REQ_NO_ALLOC_LS = 0x69,
+ SPI_PERF_RA_REQ_NO_ALLOC_CSG = 0x6a,
+ SPI_PERF_RA_REQ_NO_ALLOC_CSN = 0x6b,
+ SPI_PERF_RA_RES_STALL_PS = 0x6c,
+ SPI_PERF_RA_RES_STALL_VS = 0x6d,
+ SPI_PERF_RA_RES_STALL_GS = 0x6e,
+ SPI_PERF_RA_RES_STALL_ES = 0x6f,
+ SPI_PERF_RA_RES_STALL_HS = 0x70,
+ SPI_PERF_RA_RES_STALL_LS = 0x71,
+ SPI_PERF_RA_RES_STALL_CSG = 0x72,
+ SPI_PERF_RA_RES_STALL_CSN = 0x73,
+ SPI_PERF_RA_TMP_STALL_PS = 0x74,
+ SPI_PERF_RA_TMP_STALL_VS = 0x75,
+ SPI_PERF_RA_TMP_STALL_GS = 0x76,
+ SPI_PERF_RA_TMP_STALL_ES = 0x77,
+ SPI_PERF_RA_TMP_STALL_HS = 0x78,
+ SPI_PERF_RA_TMP_STALL_LS = 0x79,
+ SPI_PERF_RA_TMP_STALL_CSG = 0x7a,
+ SPI_PERF_RA_TMP_STALL_CSN = 0x7b,
+ SPI_PERF_RA_WAVE_SIMD_FULL_PS = 0x7c,
+ SPI_PERF_RA_WAVE_SIMD_FULL_VS = 0x7d,
+ SPI_PERF_RA_WAVE_SIMD_FULL_GS = 0x7e,
+ SPI_PERF_RA_WAVE_SIMD_FULL_ES = 0x7f,
+ SPI_PERF_RA_WAVE_SIMD_FULL_HS = 0x80,
+ SPI_PERF_RA_WAVE_SIMD_FULL_LS = 0x81,
+ SPI_PERF_RA_WAVE_SIMD_FULL_CSG = 0x82,
+ SPI_PERF_RA_WAVE_SIMD_FULL_CSN = 0x83,
+ SPI_PERF_RA_VGPR_SIMD_FULL_PS = 0x84,
+ SPI_PERF_RA_VGPR_SIMD_FULL_VS = 0x85,
+ SPI_PERF_RA_VGPR_SIMD_FULL_GS = 0x86,
+ SPI_PERF_RA_VGPR_SIMD_FULL_ES = 0x87,
+ SPI_PERF_RA_VGPR_SIMD_FULL_HS = 0x88,
+ SPI_PERF_RA_VGPR_SIMD_FULL_LS = 0x89,
+ SPI_PERF_RA_VGPR_SIMD_FULL_CSG = 0x8a,
+ SPI_PERF_RA_VGPR_SIMD_FULL_CSN = 0x8b,
+ SPI_PERF_RA_SGPR_SIMD_FULL_PS = 0x8c,
+ SPI_PERF_RA_SGPR_SIMD_FULL_VS = 0x8d,
+ SPI_PERF_RA_SGPR_SIMD_FULL_GS = 0x8e,
+ SPI_PERF_RA_SGPR_SIMD_FULL_ES = 0x8f,
+ SPI_PERF_RA_SGPR_SIMD_FULL_HS = 0x90,
+ SPI_PERF_RA_SGPR_SIMD_FULL_LS = 0x91,
+ SPI_PERF_RA_SGPR_SIMD_FULL_CSG = 0x92,
+ SPI_PERF_RA_SGPR_SIMD_FULL_CSN = 0x93,
+ SPI_PERF_RA_LDS_CU_FULL_PS = 0x94,
+ SPI_PERF_RA_LDS_CU_FULL_LS = 0x95,
+ SPI_PERF_RA_LDS_CU_FULL_ES = 0x96,
+ SPI_PERF_RA_LDS_CU_FULL_CSG = 0x97,
+ SPI_PERF_RA_LDS_CU_FULL_CSN = 0x98,
+ SPI_PERF_RA_BAR_CU_FULL_HS = 0x99,
+ SPI_PERF_RA_BAR_CU_FULL_CSG = 0x9a,
+ SPI_PERF_RA_BAR_CU_FULL_CSN = 0x9b,
+ SPI_PERF_RA_BULKY_CU_FULL_CSG = 0x9c,
+ SPI_PERF_RA_BULKY_CU_FULL_CSN = 0x9d,
+ SPI_PERF_RA_TGLIM_CU_FULL_CSG = 0x9e,
+ SPI_PERF_RA_TGLIM_CU_FULL_CSN = 0x9f,
+ SPI_PERF_RA_WVLIM_STALL_PS = 0xa0,
+ SPI_PERF_RA_WVLIM_STALL_VS = 0xa1,
+ SPI_PERF_RA_WVLIM_STALL_GS = 0xa2,
+ SPI_PERF_RA_WVLIM_STALL_ES = 0xa3,
+ SPI_PERF_RA_WVLIM_STALL_HS = 0xa4,
+ SPI_PERF_RA_WVLIM_STALL_LS = 0xa5,
+ SPI_PERF_RA_WVLIM_STALL_CSG = 0xa6,
+ SPI_PERF_RA_WVLIM_STALL_CSN = 0xa7,
+ SPI_PERF_RA_PS_LOCK_NA = 0xa8,
+ SPI_PERF_RA_VS_LOCK = 0xa9,
+ SPI_PERF_RA_GS_LOCK = 0xaa,
+ SPI_PERF_RA_ES_LOCK = 0xab,
+ SPI_PERF_RA_HS_LOCK = 0xac,
+ SPI_PERF_RA_LS_LOCK = 0xad,
+ SPI_PERF_RA_CSG_LOCK = 0xae,
+ SPI_PERF_RA_CSN_LOCK = 0xaf,
+ SPI_PERF_RA_RSV_UPD = 0xb0,
+ SPI_PERF_EXP_ARB_COL_CNT = 0xb1,
+ SPI_PERF_EXP_ARB_PAR_CNT = 0xb2,
+ SPI_PERF_EXP_ARB_POS_CNT = 0xb3,
+ SPI_PERF_EXP_ARB_GDS_CNT = 0xb4,
+ SPI_PERF_CLKGATE_BUSY_STALL = 0xb5,
+ SPI_PERF_CLKGATE_ACTIVE_STALL = 0xb6,
+ SPI_PERF_CLKGATE_ALL_CLOCKS_ON = 0xb7,
+ SPI_PERF_CLKGATE_CGTT_DYN_ON = 0xb8,
+ SPI_PERF_CLKGATE_CGTT_REG_ON = 0xb9,
+ SPI_PERF_NUM_VS_POS_EXPORTS = 0xba,
+ SPI_PERF_NUM_VS_PARAM_EXPORTS = 0xbb,
+ SPI_PERF_NUM_PS_COL_EXPORTS = 0xbc,
+ SPI_PERF_ES_GRP_FIFO_FULL = 0xbd,
+ SPI_PERF_GS_GRP_FIFO_FULL = 0xbe,
+ SPI_PERF_HS_GRP_FIFO_FULL = 0xbf,
+ SPI_PERF_LS_GRP_FIFO_FULL = 0xc0,
+ SPI_PERF_VS_ALLOC_CNT = 0xc1,
+ SPI_PERF_VS_LATE_ALLOC_ACCUM = 0xc2,
+ SPI_PERF_PC_ALLOC_CNT = 0xc3,
+ SPI_PERF_PC_ALLOC_ACCUM = 0xc4,
+} SPI_PERFCNT_SEL;
+typedef enum SPI_SHADER_FORMAT {
+ SPI_SHADER_NONE = 0x0,
+ SPI_SHADER_1COMP = 0x1,
+ SPI_SHADER_2COMP = 0x2,
+ SPI_SHADER_4COMPRESS = 0x3,
+ SPI_SHADER_4COMP = 0x4,
+} SPI_SHADER_FORMAT;
+typedef enum SPI_SHADER_EX_FORMAT {
+ SPI_SHADER_ZERO = 0x0,
+ SPI_SHADER_32_R = 0x1,
+ SPI_SHADER_32_GR = 0x2,
+ SPI_SHADER_32_AR = 0x3,
+ SPI_SHADER_FP16_ABGR = 0x4,
+ SPI_SHADER_UNORM16_ABGR = 0x5,
+ SPI_SHADER_SNORM16_ABGR = 0x6,
+ SPI_SHADER_UINT16_ABGR = 0x7,
+ SPI_SHADER_SINT16_ABGR = 0x8,
+ SPI_SHADER_32_ABGR = 0x9,
+} SPI_SHADER_EX_FORMAT;
+typedef enum CLKGATE_SM_MODE {
+ ON_SEQ = 0x0,
+ OFF_SEQ = 0x1,
+ PROG_SEQ = 0x2,
+ READ_SEQ = 0x3,
+ SM_MODE_RESERVED = 0x4,
+} CLKGATE_SM_MODE;
+typedef enum CLKGATE_BASE_MODE {
+ MULT_8 = 0x0,
+ MULT_16 = 0x1,
+} CLKGATE_BASE_MODE;
+typedef enum SQ_TEX_CLAMP {
+ SQ_TEX_WRAP = 0x0,
+ SQ_TEX_MIRROR = 0x1,
+ SQ_TEX_CLAMP_LAST_TEXEL = 0x2,
+ SQ_TEX_MIRROR_ONCE_LAST_TEXEL = 0x3,
+ SQ_TEX_CLAMP_HALF_BORDER = 0x4,
+ SQ_TEX_MIRROR_ONCE_HALF_BORDER = 0x5,
+ SQ_TEX_CLAMP_BORDER = 0x6,
+ SQ_TEX_MIRROR_ONCE_BORDER = 0x7,
+} SQ_TEX_CLAMP;
+typedef enum SQ_TEX_XY_FILTER {
+ SQ_TEX_XY_FILTER_POINT = 0x0,
+ SQ_TEX_XY_FILTER_BILINEAR = 0x1,
+ SQ_TEX_XY_FILTER_ANISO_POINT = 0x2,
+ SQ_TEX_XY_FILTER_ANISO_BILINEAR = 0x3,
+} SQ_TEX_XY_FILTER;
+typedef enum SQ_TEX_Z_FILTER {
+ SQ_TEX_Z_FILTER_NONE = 0x0,
+ SQ_TEX_Z_FILTER_POINT = 0x1,
+ SQ_TEX_Z_FILTER_LINEAR = 0x2,
+} SQ_TEX_Z_FILTER;
+typedef enum SQ_TEX_MIP_FILTER {
+ SQ_TEX_MIP_FILTER_NONE = 0x0,
+ SQ_TEX_MIP_FILTER_POINT = 0x1,
+ SQ_TEX_MIP_FILTER_LINEAR = 0x2,
+ SQ_TEX_MIP_FILTER_POINT_ANISO_ADJ = 0x3,
+} SQ_TEX_MIP_FILTER;
+typedef enum SQ_TEX_ANISO_RATIO {
+ SQ_TEX_ANISO_RATIO_1 = 0x0,
+ SQ_TEX_ANISO_RATIO_2 = 0x1,
+ SQ_TEX_ANISO_RATIO_4 = 0x2,
+ SQ_TEX_ANISO_RATIO_8 = 0x3,
+ SQ_TEX_ANISO_RATIO_16 = 0x4,
+} SQ_TEX_ANISO_RATIO;
+typedef enum SQ_TEX_DEPTH_COMPARE {
+ SQ_TEX_DEPTH_COMPARE_NEVER = 0x0,
+ SQ_TEX_DEPTH_COMPARE_LESS = 0x1,
+ SQ_TEX_DEPTH_COMPARE_EQUAL = 0x2,
+ SQ_TEX_DEPTH_COMPARE_LESSEQUAL = 0x3,
+ SQ_TEX_DEPTH_COMPARE_GREATER = 0x4,
+ SQ_TEX_DEPTH_COMPARE_NOTEQUAL = 0x5,
+ SQ_TEX_DEPTH_COMPARE_GREATEREQUAL = 0x6,
+ SQ_TEX_DEPTH_COMPARE_ALWAYS = 0x7,
+} SQ_TEX_DEPTH_COMPARE;
+typedef enum SQ_TEX_BORDER_COLOR {
+ SQ_TEX_BORDER_COLOR_TRANS_BLACK = 0x0,
+ SQ_TEX_BORDER_COLOR_OPAQUE_BLACK = 0x1,
+ SQ_TEX_BORDER_COLOR_OPAQUE_WHITE = 0x2,
+ SQ_TEX_BORDER_COLOR_REGISTER = 0x3,
+} SQ_TEX_BORDER_COLOR;
+typedef enum SQ_RSRC_BUF_TYPE {
+ SQ_RSRC_BUF = 0x0,
+ SQ_RSRC_BUF_RSVD_1 = 0x1,
+ SQ_RSRC_BUF_RSVD_2 = 0x2,
+ SQ_RSRC_BUF_RSVD_3 = 0x3,
+} SQ_RSRC_BUF_TYPE;
+typedef enum SQ_RSRC_IMG_TYPE {
+ SQ_RSRC_IMG_RSVD_0 = 0x0,
+ SQ_RSRC_IMG_RSVD_1 = 0x1,
+ SQ_RSRC_IMG_RSVD_2 = 0x2,
+ SQ_RSRC_IMG_RSVD_3 = 0x3,
+ SQ_RSRC_IMG_RSVD_4 = 0x4,
+ SQ_RSRC_IMG_RSVD_5 = 0x5,
+ SQ_RSRC_IMG_RSVD_6 = 0x6,
+ SQ_RSRC_IMG_RSVD_7 = 0x7,
+ SQ_RSRC_IMG_1D = 0x8,
+ SQ_RSRC_IMG_2D = 0x9,
+ SQ_RSRC_IMG_3D = 0xa,
+ SQ_RSRC_IMG_CUBE = 0xb,
+ SQ_RSRC_IMG_1D_ARRAY = 0xc,
+ SQ_RSRC_IMG_2D_ARRAY = 0xd,
+ SQ_RSRC_IMG_2D_MSAA = 0xe,
+ SQ_RSRC_IMG_2D_MSAA_ARRAY = 0xf,
+} SQ_RSRC_IMG_TYPE;
+typedef enum SQ_RSRC_FLAT_TYPE {
+ SQ_RSRC_FLAT_RSVD_0 = 0x0,
+ SQ_RSRC_FLAT = 0x1,
+ SQ_RSRC_FLAT_RSVD_2 = 0x2,
+ SQ_RSRC_FLAT_RSVD_3 = 0x3,
+} SQ_RSRC_FLAT_TYPE;
+typedef enum SQ_IMG_FILTER_TYPE {
+ SQ_IMG_FILTER_MODE_BLEND = 0x0,
+ SQ_IMG_FILTER_MODE_MIN = 0x1,
+ SQ_IMG_FILTER_MODE_MAX = 0x2,
+} SQ_IMG_FILTER_TYPE;
+typedef enum SQ_SEL_XYZW01 {
+ SQ_SEL_0 = 0x0,
+ SQ_SEL_1 = 0x1,
+ SQ_SEL_RESERVED_0 = 0x2,
+ SQ_SEL_RESERVED_1 = 0x3,
+ SQ_SEL_X = 0x4,
+ SQ_SEL_Y = 0x5,
+ SQ_SEL_Z = 0x6,
+ SQ_SEL_W = 0x7,
+} SQ_SEL_XYZW01;
+typedef enum SQ_WAVE_TYPE {
+ SQ_WAVE_TYPE_PS = 0x0,
+ SQ_WAVE_TYPE_VS = 0x1,
+ SQ_WAVE_TYPE_GS = 0x2,
+ SQ_WAVE_TYPE_ES = 0x3,
+ SQ_WAVE_TYPE_HS = 0x4,
+ SQ_WAVE_TYPE_LS = 0x5,
+ SQ_WAVE_TYPE_CS = 0x6,
+ SQ_WAVE_TYPE_PS1 = 0x7,
+} SQ_WAVE_TYPE;
+typedef enum SQ_THREAD_TRACE_TOKEN_TYPE {
+ SQ_THREAD_TRACE_TOKEN_MISC = 0x0,
+ SQ_THREAD_TRACE_TOKEN_TIMESTAMP = 0x1,
+ SQ_THREAD_TRACE_TOKEN_REG = 0x2,
+ SQ_THREAD_TRACE_TOKEN_WAVE_START = 0x3,
+ SQ_THREAD_TRACE_TOKEN_WAVE_ALLOC = 0x4,
+ SQ_THREAD_TRACE_TOKEN_REG_CSPRIV = 0x5,
+ SQ_THREAD_TRACE_TOKEN_WAVE_END = 0x6,
+ SQ_THREAD_TRACE_TOKEN_EVENT = 0x7,
+ SQ_THREAD_TRACE_TOKEN_EVENT_CS = 0x8,
+ SQ_THREAD_TRACE_TOKEN_EVENT_GFX1 = 0x9,
+ SQ_THREAD_TRACE_TOKEN_INST = 0xa,
+ SQ_THREAD_TRACE_TOKEN_INST_PC = 0xb,
+ SQ_THREAD_TRACE_TOKEN_INST_USERDATA = 0xc,
+ SQ_THREAD_TRACE_TOKEN_ISSUE = 0xd,
+ SQ_THREAD_TRACE_TOKEN_PERF = 0xe,
+ SQ_THREAD_TRACE_TOKEN_REG_CS = 0xf,
+} SQ_THREAD_TRACE_TOKEN_TYPE;
+typedef enum SQ_THREAD_TRACE_MISC_TOKEN_TYPE {
+ SQ_THREAD_TRACE_MISC_TOKEN_TIME = 0x0,
+ SQ_THREAD_TRACE_MISC_TOKEN_TIME_RESET = 0x1,
+ SQ_THREAD_TRACE_MISC_TOKEN_PACKET_LOST = 0x2,
+ SQ_THREAD_TRACE_MISC_TOKEN_SURF_SYNC = 0x3,
+ SQ_THREAD_TRACE_MISC_TOKEN_TTRACE_STALL_BEGIN = 0x4,
+ SQ_THREAD_TRACE_MISC_TOKEN_TTRACE_STALL_END = 0x5,
+ SQ_THREAD_TRACE_MISC_TOKEN_SAVECTX = 0x6,
+ SQ_THREAD_TRACE_MISC_TOKEN_SHOOT_DOWN = 0x7,
+} SQ_THREAD_TRACE_MISC_TOKEN_TYPE;
+typedef enum SQ_THREAD_TRACE_INST_TYPE {
+ SQ_THREAD_TRACE_INST_TYPE_SMEM_RD = 0x0,
+ SQ_THREAD_TRACE_INST_TYPE_SALU_32 = 0x1,
+ SQ_THREAD_TRACE_INST_TYPE_VMEM_RD = 0x2,
+ SQ_THREAD_TRACE_INST_TYPE_VMEM_WR = 0x3,
+ SQ_THREAD_TRACE_INST_TYPE_FLAT_WR = 0x4,
+ SQ_THREAD_TRACE_INST_TYPE_VALU_32 = 0x5,
+ SQ_THREAD_TRACE_INST_TYPE_LDS = 0x6,
+ SQ_THREAD_TRACE_INST_TYPE_PC = 0x7,
+ SQ_THREAD_TRACE_INST_TYPE_EXPREQ_GDS = 0x8,
+ SQ_THREAD_TRACE_INST_TYPE_EXPREQ_GFX = 0x9,
+ SQ_THREAD_TRACE_INST_TYPE_EXPGNT_PAR_COL = 0xa,
+ SQ_THREAD_TRACE_INST_TYPE_EXPGNT_POS_GDS = 0xb,
+ SQ_THREAD_TRACE_INST_TYPE_JUMP = 0xc,
+ SQ_THREAD_TRACE_INST_TYPE_NEXT = 0xd,
+ SQ_THREAD_TRACE_INST_TYPE_FLAT_RD = 0xe,
+ SQ_THREAD_TRACE_INST_TYPE_OTHER_MSG = 0xf,
+ SQ_THREAD_TRACE_INST_TYPE_SMEM_WR = 0x10,
+ SQ_THREAD_TRACE_INST_TYPE_SALU_64 = 0x11,
+ SQ_THREAD_TRACE_INST_TYPE_VALU_64 = 0x12,
+ SQ_THREAD_TRACE_INST_TYPE_SMEM_RD_REPLAY = 0x13,
+ SQ_THREAD_TRACE_INST_TYPE_SMEM_WR_REPLAY = 0x14,
+ SQ_THREAD_TRACE_INST_TYPE_VMEM_RD_REPLAY = 0x15,
+ SQ_THREAD_TRACE_INST_TYPE_VMEM_WR_REPLAY = 0x16,
+ SQ_THREAD_TRACE_INST_TYPE_FLAT_RD_REPLAY = 0x17,
+ SQ_THREAD_TRACE_INST_TYPE_FLAT_WR_REPLAY = 0x18,
+} SQ_THREAD_TRACE_INST_TYPE;
+typedef enum SQ_THREAD_TRACE_REG_TYPE {
+ SQ_THREAD_TRACE_REG_TYPE_EVENT = 0x0,
+ SQ_THREAD_TRACE_REG_TYPE_DRAW = 0x1,
+ SQ_THREAD_TRACE_REG_TYPE_DISPATCH = 0x2,
+ SQ_THREAD_TRACE_REG_TYPE_USERDATA = 0x3,
+ SQ_THREAD_TRACE_REG_TYPE_MARKER = 0x4,
+ SQ_THREAD_TRACE_REG_TYPE_GFXDEC = 0x5,
+ SQ_THREAD_TRACE_REG_TYPE_SHDEC = 0x6,
+ SQ_THREAD_TRACE_REG_TYPE_OTHER = 0x7,
+} SQ_THREAD_TRACE_REG_TYPE;
+typedef enum SQ_THREAD_TRACE_REG_OP {
+ SQ_THREAD_TRACE_REG_OP_READ = 0x0,
+ SQ_THREAD_TRACE_REG_OP_WRITE = 0x1,
+} SQ_THREAD_TRACE_REG_OP;
+typedef enum SQ_THREAD_TRACE_MODE_SEL {
+ SQ_THREAD_TRACE_MODE_OFF = 0x0,
+ SQ_THREAD_TRACE_MODE_ON = 0x1,
+} SQ_THREAD_TRACE_MODE_SEL;
+typedef enum SQ_THREAD_TRACE_CAPTURE_MODE {
+ SQ_THREAD_TRACE_CAPTURE_MODE_ALL = 0x0,
+ SQ_THREAD_TRACE_CAPTURE_MODE_SELECT = 0x1,
+ SQ_THREAD_TRACE_CAPTURE_MODE_SELECT_DETAIL = 0x2,
+} SQ_THREAD_TRACE_CAPTURE_MODE;
+typedef enum SQ_THREAD_TRACE_VM_ID_MASK {
+ SQ_THREAD_TRACE_VM_ID_MASK_SINGLE = 0x0,
+ SQ_THREAD_TRACE_VM_ID_MASK_ALL = 0x1,
+ SQ_THREAD_TRACE_VM_ID_MASK_SINGLE_DETAIL = 0x2,
+} SQ_THREAD_TRACE_VM_ID_MASK;
+typedef enum SQ_THREAD_TRACE_WAVE_MASK {
+ SQ_THREAD_TRACE_WAVE_MASK_NONE = 0x0,
+ SQ_THREAD_TRACE_WAVE_MASK_ALL = 0x1,
+} SQ_THREAD_TRACE_WAVE_MASK;
+typedef enum SQ_THREAD_TRACE_ISSUE {
+ SQ_THREAD_TRACE_ISSUE_NULL = 0x0,
+ SQ_THREAD_TRACE_ISSUE_STALL = 0x1,
+ SQ_THREAD_TRACE_ISSUE_INST = 0x2,
+ SQ_THREAD_TRACE_ISSUE_IMMED = 0x3,
+} SQ_THREAD_TRACE_ISSUE;
+typedef enum SQ_THREAD_TRACE_ISSUE_MASK {
+ SQ_THREAD_TRACE_ISSUE_MASK_ALL = 0x0,
+ SQ_THREAD_TRACE_ISSUE_MASK_STALLED = 0x1,
+ SQ_THREAD_TRACE_ISSUE_MASK_STALLED_AND_IMMED = 0x2,
+ SQ_THREAD_TRACE_ISSUE_MASK_IMMED = 0x3,
+} SQ_THREAD_TRACE_ISSUE_MASK;
+typedef enum SQ_PERF_SEL {
+ SQ_PERF_SEL_NONE = 0x0,
+ SQ_PERF_SEL_ACCUM_PREV = 0x1,
+ SQ_PERF_SEL_CYCLES = 0x2,
+ SQ_PERF_SEL_BUSY_CYCLES = 0x3,
+ SQ_PERF_SEL_WAVES = 0x4,
+ SQ_PERF_SEL_LEVEL_WAVES = 0x5,
+ SQ_PERF_SEL_WAVES_EQ_64 = 0x6,
+ SQ_PERF_SEL_WAVES_LT_64 = 0x7,
+ SQ_PERF_SEL_WAVES_LT_48 = 0x8,
+ SQ_PERF_SEL_WAVES_LT_32 = 0x9,
+ SQ_PERF_SEL_WAVES_LT_16 = 0xa,
+ SQ_PERF_SEL_WAVES_CU = 0xb,
+ SQ_PERF_SEL_LEVEL_WAVES_CU = 0xc,
+ SQ_PERF_SEL_BUSY_CU_CYCLES = 0xd,
+ SQ_PERF_SEL_ITEMS = 0xe,
+ SQ_PERF_SEL_QUADS = 0xf,
+ SQ_PERF_SEL_EVENTS = 0x10,
+ SQ_PERF_SEL_SURF_SYNCS = 0x11,
+ SQ_PERF_SEL_TTRACE_REQS = 0x12,
+ SQ_PERF_SEL_TTRACE_INFLIGHT_REQS = 0x13,
+ SQ_PERF_SEL_TTRACE_STALL = 0x14,
+ SQ_PERF_SEL_MSG_CNTR = 0x15,
+ SQ_PERF_SEL_MSG_PERF = 0x16,
+ SQ_PERF_SEL_MSG_GSCNT = 0x17,
+ SQ_PERF_SEL_MSG_INTERRUPT = 0x18,
+ SQ_PERF_SEL_INSTS = 0x19,
+ SQ_PERF_SEL_INSTS_VALU = 0x1a,
+ SQ_PERF_SEL_INSTS_VMEM_WR = 0x1b,
+ SQ_PERF_SEL_INSTS_VMEM_RD = 0x1c,
+ SQ_PERF_SEL_INSTS_VMEM = 0x1d,
+ SQ_PERF_SEL_INSTS_SALU = 0x1e,
+ SQ_PERF_SEL_INSTS_SMEM = 0x1f,
+ SQ_PERF_SEL_INSTS_FLAT = 0x20,
+ SQ_PERF_SEL_INSTS_FLAT_LDS_ONLY = 0x21,
+ SQ_PERF_SEL_INSTS_LDS = 0x22,
+ SQ_PERF_SEL_INSTS_GDS = 0x23,
+ SQ_PERF_SEL_INSTS_EXP = 0x24,
+ SQ_PERF_SEL_INSTS_EXP_GDS = 0x25,
+ SQ_PERF_SEL_INSTS_BRANCH = 0x26,
+ SQ_PERF_SEL_INSTS_SENDMSG = 0x27,
+ SQ_PERF_SEL_INSTS_VSKIPPED = 0x28,
+ SQ_PERF_SEL_INST_LEVEL_VMEM = 0x29,
+ SQ_PERF_SEL_INST_LEVEL_SMEM = 0x2a,
+ SQ_PERF_SEL_INST_LEVEL_LDS = 0x2b,
+ SQ_PERF_SEL_INST_LEVEL_GDS = 0x2c,
+ SQ_PERF_SEL_INST_LEVEL_EXP = 0x2d,
+ SQ_PERF_SEL_WAVE_CYCLES = 0x2e,
+ SQ_PERF_SEL_WAVE_READY = 0x2f,
+ SQ_PERF_SEL_WAIT_CNT_VM = 0x30,
+ SQ_PERF_SEL_WAIT_CNT_LGKM = 0x31,
+ SQ_PERF_SEL_WAIT_CNT_EXP = 0x32,
+ SQ_PERF_SEL_WAIT_CNT_ANY = 0x33,
+ SQ_PERF_SEL_WAIT_BARRIER = 0x34,
+ SQ_PERF_SEL_WAIT_EXP_ALLOC = 0x35,
+ SQ_PERF_SEL_WAIT_SLEEP = 0x36,
+ SQ_PERF_SEL_WAIT_OTHER = 0x37,
+ SQ_PERF_SEL_WAIT_ANY = 0x38,
+ SQ_PERF_SEL_WAIT_TTRACE = 0x39,
+ SQ_PERF_SEL_WAIT_IFETCH = 0x3a,
+ SQ_PERF_SEL_WAIT_INST_VMEM = 0x3b,
+ SQ_PERF_SEL_WAIT_INST_SCA = 0x3c,
+ SQ_PERF_SEL_WAIT_INST_LDS = 0x3d,
+ SQ_PERF_SEL_WAIT_INST_VALU = 0x3e,
+ SQ_PERF_SEL_WAIT_INST_EXP_GDS = 0x3f,
+ SQ_PERF_SEL_WAIT_INST_MISC = 0x40,
+ SQ_PERF_SEL_WAIT_INST_FLAT = 0x41,
+ SQ_PERF_SEL_ACTIVE_INST_ANY = 0x42,
+ SQ_PERF_SEL_ACTIVE_INST_VMEM = 0x43,
+ SQ_PERF_SEL_ACTIVE_INST_LDS = 0x44,
+ SQ_PERF_SEL_ACTIVE_INST_VALU = 0x45,
+ SQ_PERF_SEL_ACTIVE_INST_SCA = 0x46,
+ SQ_PERF_SEL_ACTIVE_INST_EXP_GDS = 0x47,
+ SQ_PERF_SEL_ACTIVE_INST_MISC = 0x48,
+ SQ_PERF_SEL_ACTIVE_INST_FLAT = 0x49,
+ SQ_PERF_SEL_INST_CYCLES_VMEM_WR = 0x4a,
+ SQ_PERF_SEL_INST_CYCLES_VMEM_RD = 0x4b,
+ SQ_PERF_SEL_INST_CYCLES_VMEM_ADDR = 0x4c,
+ SQ_PERF_SEL_INST_CYCLES_VMEM_DATA = 0x4d,
+ SQ_PERF_SEL_INST_CYCLES_VMEM_CMD = 0x4e,
+ SQ_PERF_SEL_INST_CYCLES_VMEM = 0x4f,
+ SQ_PERF_SEL_INST_CYCLES_LDS = 0x50,
+ SQ_PERF_SEL_INST_CYCLES_VALU = 0x51,
+ SQ_PERF_SEL_INST_CYCLES_EXP = 0x52,
+ SQ_PERF_SEL_INST_CYCLES_GDS = 0x53,
+ SQ_PERF_SEL_INST_CYCLES_SCA = 0x54,
+ SQ_PERF_SEL_INST_CYCLES_SMEM = 0x55,
+ SQ_PERF_SEL_INST_CYCLES_SALU = 0x56,
+ SQ_PERF_SEL_INST_CYCLES_EXP_GDS = 0x57,
+ SQ_PERF_SEL_INST_CYCLES_MISC = 0x58,
+ SQ_PERF_SEL_THREAD_CYCLES_VALU = 0x59,
+ SQ_PERF_SEL_THREAD_CYCLES_VALU_MAX = 0x5a,
+ SQ_PERF_SEL_IFETCH = 0x5b,
+ SQ_PERF_SEL_IFETCH_LEVEL = 0x5c,
+ SQ_PERF_SEL_CBRANCH_FORK = 0x5d,
+ SQ_PERF_SEL_CBRANCH_FORK_SPLIT = 0x5e,
+ SQ_PERF_SEL_VALU_LDS_DIRECT_RD = 0x5f,
+ SQ_PERF_SEL_VALU_LDS_INTERP_OP = 0x60,
+ SQ_PERF_SEL_LDS_BANK_CONFLICT = 0x61,
+ SQ_PERF_SEL_LDS_ADDR_CONFLICT = 0x62,
+ SQ_PERF_SEL_LDS_UNALIGNED_STALL = 0x63,
+ SQ_PERF_SEL_LDS_MEM_VIOLATIONS = 0x64,
+ SQ_PERF_SEL_LDS_ATOMIC_RETURN = 0x65,
+ SQ_PERF_SEL_LDS_IDX_ACTIVE = 0x66,
+ SQ_PERF_SEL_VALU_DEP_STALL = 0x67,
+ SQ_PERF_SEL_VALU_STARVE = 0x68,
+ SQ_PERF_SEL_EXP_REQ_FIFO_FULL = 0x69,
+ SQ_PERF_SEL_LDS_BACK2BACK_STALL = 0x6a,
+ SQ_PERF_SEL_LDS_DATA_FIFO_FULL = 0x6b,
+ SQ_PERF_SEL_LDS_CMD_FIFO_FULL = 0x6c,
+ SQ_PERF_SEL_VMEM_BACK2BACK_STALL = 0x6d,
+ SQ_PERF_SEL_VMEM_TA_ADDR_FIFO_FULL = 0x6e,
+ SQ_PERF_SEL_VMEM_TA_CMD_FIFO_FULL = 0x6f,
+ SQ_PERF_SEL_VMEM_EX_DATA_REG_BUSY = 0x70,
+ SQ_PERF_SEL_VMEM_WR_BACK2BACK_STALL = 0x71,
+ SQ_PERF_SEL_VMEM_WR_TA_DATA_FIFO_FULL = 0x72,
+ SQ_PERF_SEL_VALU_SRC_C_CONFLICT = 0x73,
+ SQ_PERF_SEL_VMEM_RD_SRC_CD_CONFLICT = 0x74,
+ SQ_PERF_SEL_VMEM_WR_SRC_CD_CONFLICT = 0x75,
+ SQ_PERF_SEL_FLAT_SRC_CD_CONFLICT = 0x76,
+ SQ_PERF_SEL_LDS_SRC_CD_CONFLICT = 0x77,
+ SQ_PERF_SEL_SRC_CD_BUSY = 0x78,
+ SQ_PERF_SEL_PT_POWER_STALL = 0x79,
+ SQ_PERF_SEL_USER0 = 0x7a,
+ SQ_PERF_SEL_USER1 = 0x7b,
+ SQ_PERF_SEL_USER2 = 0x7c,
+ SQ_PERF_SEL_USER3 = 0x7d,
+ SQ_PERF_SEL_USER4 = 0x7e,
+ SQ_PERF_SEL_USER5 = 0x7f,
+ SQ_PERF_SEL_USER6 = 0x80,
+ SQ_PERF_SEL_USER7 = 0x81,
+ SQ_PERF_SEL_USER8 = 0x82,
+ SQ_PERF_SEL_USER9 = 0x83,
+ SQ_PERF_SEL_USER10 = 0x84,
+ SQ_PERF_SEL_USER11 = 0x85,
+ SQ_PERF_SEL_USER12 = 0x86,
+ SQ_PERF_SEL_USER13 = 0x87,
+ SQ_PERF_SEL_USER14 = 0x88,
+ SQ_PERF_SEL_USER15 = 0x89,
+ SQ_PERF_SEL_USER_LEVEL0 = 0x8a,
+ SQ_PERF_SEL_USER_LEVEL1 = 0x8b,
+ SQ_PERF_SEL_USER_LEVEL2 = 0x8c,
+ SQ_PERF_SEL_USER_LEVEL3 = 0x8d,
+ SQ_PERF_SEL_USER_LEVEL4 = 0x8e,
+ SQ_PERF_SEL_USER_LEVEL5 = 0x8f,
+ SQ_PERF_SEL_USER_LEVEL6 = 0x90,
+ SQ_PERF_SEL_USER_LEVEL7 = 0x91,
+ SQ_PERF_SEL_USER_LEVEL8 = 0x92,
+ SQ_PERF_SEL_USER_LEVEL9 = 0x93,
+ SQ_PERF_SEL_USER_LEVEL10 = 0x94,
+ SQ_PERF_SEL_USER_LEVEL11 = 0x95,
+ SQ_PERF_SEL_USER_LEVEL12 = 0x96,
+ SQ_PERF_SEL_USER_LEVEL13 = 0x97,
+ SQ_PERF_SEL_USER_LEVEL14 = 0x98,
+ SQ_PERF_SEL_USER_LEVEL15 = 0x99,
+ SQ_PERF_SEL_POWER_VALU = 0x9a,
+ SQ_PERF_SEL_POWER_VALU0 = 0x9b,
+ SQ_PERF_SEL_POWER_VALU1 = 0x9c,
+ SQ_PERF_SEL_POWER_VALU2 = 0x9d,
+ SQ_PERF_SEL_POWER_GPR_RD = 0x9e,
+ SQ_PERF_SEL_POWER_GPR_WR = 0x9f,
+ SQ_PERF_SEL_POWER_LDS_BUSY = 0xa0,
+ SQ_PERF_SEL_POWER_ALU_BUSY = 0xa1,
+ SQ_PERF_SEL_POWER_TEX_BUSY = 0xa2,
+ SQ_PERF_SEL_ACCUM_PREV_HIRES = 0xa3,
+ SQ_PERF_SEL_WAVES_RESTORED = 0xa4,
+ SQ_PERF_SEL_WAVES_SAVED = 0xa5,
+ SQ_PERF_SEL_DUMMY_LAST = 0xa7,
+ SQC_PERF_SEL_ICACHE_INPUT_VALID_READY = 0xa8,
+ SQC_PERF_SEL_ICACHE_INPUT_VALID_READYB = 0xa9,
+ SQC_PERF_SEL_ICACHE_INPUT_VALIDB = 0xaa,
+ SQC_PERF_SEL_DCACHE_INPUT_VALID_READY = 0xab,
+ SQC_PERF_SEL_DCACHE_INPUT_VALID_READYB = 0xac,
+ SQC_PERF_SEL_DCACHE_INPUT_VALIDB = 0xad,
+ SQC_PERF_SEL_TC_REQ = 0xae,
+ SQC_PERF_SEL_TC_INST_REQ = 0xaf,
+ SQC_PERF_SEL_TC_DATA_READ_REQ = 0xb0,
+ SQC_PERF_SEL_TC_DATA_WRITE_REQ = 0xb1,
+ SQC_PERF_SEL_TC_DATA_ATOMIC_REQ = 0xb2,
+ SQC_PERF_SEL_TC_STALL = 0xb3,
+ SQC_PERF_SEL_TC_STARVE = 0xb4,
+ SQC_PERF_SEL_ICACHE_BUSY_CYCLES = 0xb5,
+ SQC_PERF_SEL_ICACHE_REQ = 0xb6,
+ SQC_PERF_SEL_ICACHE_HITS = 0xb7,
+ SQC_PERF_SEL_ICACHE_MISSES = 0xb8,
+ SQC_PERF_SEL_ICACHE_MISSES_DUPLICATE = 0xb9,
+ SQC_PERF_SEL_ICACHE_INVAL_INST = 0xba,
+ SQC_PERF_SEL_ICACHE_INVAL_ASYNC = 0xbb,
+ SQC_PERF_SEL_ICACHE_INPUT_STALL_ARB_NO_GRANT = 0xbc,
+ SQC_PERF_SEL_ICACHE_INPUT_STALL_BANK_READYB = 0xbd,
+ SQC_PERF_SEL_ICACHE_CACHE_STALLED = 0xbe,
+ SQC_PERF_SEL_ICACHE_CACHE_STALL_INFLIGHT_NONZERO = 0xbf,
+ SQC_PERF_SEL_ICACHE_CACHE_STALL_INFLIGHT_MAX = 0xc0,
+ SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT = 0xc1,
+ SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT_MISS_FIFO = 0xc2,
+ SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT_HIT_FIFO = 0xc3,
+ SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT_TC_IF = 0xc4,
+ SQC_PERF_SEL_ICACHE_STALL_OUTXBAR_ARB_NO_GRANT = 0xc5,
+ SQC_PERF_SEL_DCACHE_BUSY_CYCLES = 0xc6,
+ SQC_PERF_SEL_DCACHE_REQ = 0xc7,
+ SQC_PERF_SEL_DCACHE_HITS = 0xc8,
+ SQC_PERF_SEL_DCACHE_MISSES = 0xc9,
+ SQC_PERF_SEL_DCACHE_MISSES_DUPLICATE = 0xca,
+ SQC_PERF_SEL_DCACHE_HIT_LRU_READ = 0xcb,
+ SQC_PERF_SEL_DCACHE_MISS_EVICT_READ = 0xcc,
+ SQC_PERF_SEL_DCACHE_WC_LRU_WRITE = 0xcd,
+ SQC_PERF_SEL_DCACHE_WT_EVICT_WRITE = 0xce,
+ SQC_PERF_SEL_DCACHE_ATOMIC = 0xcf,
+ SQC_PERF_SEL_DCACHE_VOLATILE = 0xd0,
+ SQC_PERF_SEL_DCACHE_INVAL_INST = 0xd1,
+ SQC_PERF_SEL_DCACHE_INVAL_ASYNC = 0xd2,
+ SQC_PERF_SEL_DCACHE_INVAL_VOLATILE_INST = 0xd3,
+ SQC_PERF_SEL_DCACHE_INVAL_VOLATILE_ASYNC = 0xd4,
+ SQC_PERF_SEL_DCACHE_WB_INST = 0xd5,
+ SQC_PERF_SEL_DCACHE_WB_ASYNC = 0xd6,
+ SQC_PERF_SEL_DCACHE_WB_VOLATILE_INST = 0xd7,
+ SQC_PERF_SEL_DCACHE_WB_VOLATILE_ASYNC = 0xd8,
+ SQC_PERF_SEL_DCACHE_INPUT_STALL_ARB_NO_GRANT = 0xd9,
+ SQC_PERF_SEL_DCACHE_INPUT_STALL_BANK_READYB = 0xda,
+ SQC_PERF_SEL_DCACHE_CACHE_STALLED = 0xdb,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_INFLIGHT_MAX = 0xdc,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT = 0xdd,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_EVICT = 0xde,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_UNORDERED = 0xdf,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_ALLOC_UNAVAILABLE= 0xe0,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_FORCE_EVICT = 0xe1,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_MULTI_FLUSH = 0xe2,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_FLUSH_DONE = 0xe3,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT_MISS_FIFO = 0xe4,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT_HIT_FIFO = 0xe5,
+ SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT_TC_IF = 0xe6,
+ SQC_PERF_SEL_DCACHE_STALL_OUTXBAR_ARB_NO_GRANT = 0xe7,
+ SQC_PERF_SEL_DCACHE_REQ_READ_1 = 0xe8,
+ SQC_PERF_SEL_DCACHE_REQ_READ_2 = 0xe9,
+ SQC_PERF_SEL_DCACHE_REQ_READ_4 = 0xea,
+ SQC_PERF_SEL_DCACHE_REQ_READ_8 = 0xeb,
+ SQC_PERF_SEL_DCACHE_REQ_READ_16 = 0xec,
+ SQC_PERF_SEL_DCACHE_REQ_TIME = 0xed,
+ SQC_PERF_SEL_DCACHE_REQ_WRITE_1 = 0xee,
+ SQC_PERF_SEL_DCACHE_REQ_WRITE_2 = 0xef,
+ SQC_PERF_SEL_DCACHE_REQ_WRITE_4 = 0xf0,
+ SQC_PERF_SEL_DCACHE_REQ_ATC_PROBE = 0xf1,
+ SQC_PERF_SEL_SQ_DCACHE_REQS = 0xf2,
+ SQC_PERF_SEL_DCACHE_FLAT_REQ = 0xf3,
+ SQC_PERF_SEL_DCACHE_NONFLAT_REQ = 0xf4,
+ SQC_PERF_SEL_ICACHE_INFLIGHT_LEVEL = 0xf5,
+ SQC_PERF_SEL_DCACHE_INFLIGHT_LEVEL = 0xf6,
+ SQC_PERF_SEL_TC_INFLIGHT_LEVEL = 0xf7,
+ SQC_PERF_SEL_ICACHE_TC_INFLIGHT_LEVEL = 0xf8,
+ SQC_PERF_SEL_DCACHE_TC_INFLIGHT_LEVEL = 0xf9,
+ SQC_PERF_SEL_ICACHE_GATCL1_TRANSLATION_MISS = 0xfa,
+ SQC_PERF_SEL_ICACHE_GATCL1_PERMISSION_MISS = 0xfb,
+ SQC_PERF_SEL_ICACHE_GATCL1_REQUEST = 0xfc,
+ SQC_PERF_SEL_ICACHE_GATCL1_STALL_INFLIGHT_MAX = 0xfd,
+ SQC_PERF_SEL_ICACHE_GATCL1_STALL_LRU_INFLIGHT = 0xfe,
+ SQC_PERF_SEL_ICACHE_GATCL1_LFIFO_FULL = 0xff,
+ SQC_PERF_SEL_ICACHE_GATCL1_STALL_LFIFO_NOT_RES = 0x100,
+ SQC_PERF_SEL_ICACHE_GATCL1_STALL_ATCL2_REQ_OUT_OF_CREDITS= 0x101,
+ SQC_PERF_SEL_ICACHE_GATCL1_ATCL2_INFLIGHT = 0x102,
+ SQC_PERF_SEL_ICACHE_GATCL1_STALL_MISSFIFO_FULL = 0x103,
+ SQC_PERF_SEL_DCACHE_GATCL1_TRANSLATION_MISS = 0x104,
+ SQC_PERF_SEL_DCACHE_GATCL1_PERMISSION_MISS = 0x105,
+ SQC_PERF_SEL_DCACHE_GATCL1_REQUEST = 0x106,
+ SQC_PERF_SEL_DCACHE_GATCL1_STALL_INFLIGHT_MAX = 0x107,
+ SQC_PERF_SEL_DCACHE_GATCL1_STALL_LRU_INFLIGHT = 0x108,
+ SQC_PERF_SEL_DCACHE_GATCL1_LFIFO_FULL = 0x109,
+ SQC_PERF_SEL_DCACHE_GATCL1_STALL_LFIFO_NOT_RES = 0x10a,
+ SQC_PERF_SEL_DCACHE_GATCL1_STALL_ATCL2_REQ_OUT_OF_CREDITS= 0x10b,
+ SQC_PERF_SEL_DCACHE_GATCL1_ATCL2_INFLIGHT = 0x10c,
+ SQC_PERF_SEL_DCACHE_GATCL1_STALL_MISSFIFO_FULL = 0x10d,
+ SQC_PERF_SEL_DCACHE_GATCL1_STALL_MULTI_MISS = 0x10e,
+ SQC_PERF_SEL_DCACHE_GATCL1_HIT_FIFO_FULL = 0x10f,
+ SQC_PERF_SEL_DUMMY_LAST = 0x110,
+ SQ_PERF_SEL_INSTS_SMEM_NORM = 0x111,
+ SQ_PERF_SEL_ATC_INSTS_VMEM = 0x112,
+ SQ_PERF_SEL_ATC_INST_LEVEL_VMEM = 0x113,
+ SQ_PERF_SEL_ATC_XNACK_FIRST = 0x114,
+ SQ_PERF_SEL_ATC_XNACK_ALL = 0x115,
+ SQ_PERF_SEL_ATC_XNACK_FIFO_FULL = 0x116,
+ SQ_PERF_SEL_ATC_INSTS_SMEM = 0x117,
+ SQ_PERF_SEL_ATC_INST_LEVEL_SMEM = 0x118,
+ SQ_PERF_SEL_IFETCH_XNACK = 0x119,
+ SQ_PERF_SEL_TLB_SHOOTDOWN = 0x11a,
+ SQ_PERF_SEL_TLB_SHOOTDOWN_CYCLES = 0x11b,
+ SQ_PERF_SEL_INSTS_VMEM_WR_REPLAY = 0x11c,
+ SQ_PERF_SEL_INSTS_VMEM_RD_REPLAY = 0x11d,
+ SQ_PERF_SEL_INSTS_VMEM_REPLAY = 0x11e,
+ SQ_PERF_SEL_INSTS_SMEM_REPLAY = 0x11f,
+ SQ_PERF_SEL_INSTS_SMEM_NORM_REPLAY = 0x120,
+ SQ_PERF_SEL_INSTS_FLAT_REPLAY = 0x121,
+ SQ_PERF_SEL_ATC_INSTS_VMEM_REPLAY = 0x122,
+ SQ_PERF_SEL_ATC_INSTS_SMEM_REPLAY = 0x123,
+ SQ_PERF_SEL_DUMMY_LAST1 = 0x12a,
+} SQ_PERF_SEL;
+typedef enum SQ_CAC_POWER_SEL {
+ SQ_CAC_POWER_VALU = 0x0,
+ SQ_CAC_POWER_VALU0 = 0x1,
+ SQ_CAC_POWER_VALU1 = 0x2,
+ SQ_CAC_POWER_VALU2 = 0x3,
+ SQ_CAC_POWER_GPR_RD = 0x4,
+ SQ_CAC_POWER_GPR_WR = 0x5,
+ SQ_CAC_POWER_LDS_BUSY = 0x6,
+ SQ_CAC_POWER_ALU_BUSY = 0x7,
+ SQ_CAC_POWER_TEX_BUSY = 0x8,
+} SQ_CAC_POWER_SEL;
+typedef enum SQ_IND_CMD_CMD {
+ SQ_IND_CMD_CMD_NULL = 0x0,
+ SQ_IND_CMD_CMD_SETHALT = 0x1,
+ SQ_IND_CMD_CMD_SAVECTX = 0x2,
+ SQ_IND_CMD_CMD_KILL = 0x3,
+ SQ_IND_CMD_CMD_DEBUG = 0x4,
+ SQ_IND_CMD_CMD_TRAP = 0x5,
+ SQ_IND_CMD_CMD_SET_SPI_PRIO = 0x6,
+} SQ_IND_CMD_CMD;
+typedef enum SQ_IND_CMD_MODE {
+ SQ_IND_CMD_MODE_SINGLE = 0x0,
+ SQ_IND_CMD_MODE_BROADCAST = 0x1,
+ SQ_IND_CMD_MODE_BROADCAST_QUEUE = 0x2,
+ SQ_IND_CMD_MODE_BROADCAST_PIPE = 0x3,
+ SQ_IND_CMD_MODE_BROADCAST_ME = 0x4,
+} SQ_IND_CMD_MODE;
+typedef enum SQ_EDC_INFO_SOURCE {
+ SQ_EDC_INFO_SOURCE_INVALID = 0x0,
+ SQ_EDC_INFO_SOURCE_INST = 0x1,
+ SQ_EDC_INFO_SOURCE_SGPR = 0x2,
+ SQ_EDC_INFO_SOURCE_VGPR = 0x3,
+ SQ_EDC_INFO_SOURCE_LDS = 0x4,
+ SQ_EDC_INFO_SOURCE_GDS = 0x5,
+ SQ_EDC_INFO_SOURCE_TA = 0x6,
+} SQ_EDC_INFO_SOURCE;
+typedef enum SQ_ROUND_MODE {
+ SQ_ROUND_NEAREST_EVEN = 0x0,
+ SQ_ROUND_PLUS_INFINITY = 0x1,
+ SQ_ROUND_MINUS_INFINITY = 0x2,
+ SQ_ROUND_TO_ZERO = 0x3,
+} SQ_ROUND_MODE;
+typedef enum SQ_INTERRUPT_WORD_ENCODING {
+ SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,
+ SQ_INTERRUPT_WORD_ENCODING_INST = 0x1,
+ SQ_INTERRUPT_WORD_ENCODING_ERROR = 0x2,
+} SQ_INTERRUPT_WORD_ENCODING;
+typedef enum ENUM_SQ_EXPORT_RAT_INST {
+ SQ_EXPORT_RAT_INST_NOP = 0x0,
+ SQ_EXPORT_RAT_INST_STORE_TYPED = 0x1,
+ SQ_EXPORT_RAT_INST_STORE_RAW = 0x2,
+ SQ_EXPORT_RAT_INST_STORE_RAW_FDENORM = 0x3,
+ SQ_EXPORT_RAT_INST_CMPXCHG_INT = 0x4,
+ SQ_EXPORT_RAT_INST_CMPXCHG_FLT = 0x5,
+ SQ_EXPORT_RAT_INST_CMPXCHG_FDENORM = 0x6,
+ SQ_EXPORT_RAT_INST_ADD = 0x7,
+ SQ_EXPORT_RAT_INST_SUB = 0x8,
+ SQ_EXPORT_RAT_INST_RSUB = 0x9,
+ SQ_EXPORT_RAT_INST_MIN_INT = 0xa,
+ SQ_EXPORT_RAT_INST_MIN_UINT = 0xb,
+ SQ_EXPORT_RAT_INST_MAX_INT = 0xc,
+ SQ_EXPORT_RAT_INST_MAX_UINT = 0xd,
+ SQ_EXPORT_RAT_INST_AND = 0xe,
+ SQ_EXPORT_RAT_INST_OR = 0xf,
+ SQ_EXPORT_RAT_INST_XOR = 0x10,
+ SQ_EXPORT_RAT_INST_MSKOR = 0x11,
+ SQ_EXPORT_RAT_INST_INC_UINT = 0x12,
+ SQ_EXPORT_RAT_INST_DEC_UINT = 0x13,
+ SQ_EXPORT_RAT_INST_STORE_DWORD = 0x14,
+ SQ_EXPORT_RAT_INST_STORE_SHORT = 0x15,
+ SQ_EXPORT_RAT_INST_STORE_BYTE = 0x16,
+ SQ_EXPORT_RAT_INST_NOP_RTN = 0x20,
+ SQ_EXPORT_RAT_INST_XCHG_RTN = 0x22,
+ SQ_EXPORT_RAT_INST_XCHG_FDENORM_RTN = 0x23,
+ SQ_EXPORT_RAT_INST_CMPXCHG_INT_RTN = 0x24,
+ SQ_EXPORT_RAT_INST_CMPXCHG_FLT_RTN = 0x25,
+ SQ_EXPORT_RAT_INST_CMPXCHG_FDENORM_RTN = 0x26,
+ SQ_EXPORT_RAT_INST_ADD_RTN = 0x27,
+ SQ_EXPORT_RAT_INST_SUB_RTN = 0x28,
+ SQ_EXPORT_RAT_INST_RSUB_RTN = 0x29,
+ SQ_EXPORT_RAT_INST_MIN_INT_RTN = 0x2a,
+ SQ_EXPORT_RAT_INST_MIN_UINT_RTN = 0x2b,
+ SQ_EXPORT_RAT_INST_MAX_INT_RTN = 0x2c,
+ SQ_EXPORT_RAT_INST_MAX_UINT_RTN = 0x2d,
+ SQ_EXPORT_RAT_INST_AND_RTN = 0x2e,
+ SQ_EXPORT_RAT_INST_OR_RTN = 0x2f,
+ SQ_EXPORT_RAT_INST_XOR_RTN = 0x30,
+ SQ_EXPORT_RAT_INST_MSKOR_RTN = 0x31,
+ SQ_EXPORT_RAT_INST_INC_UINT_RTN = 0x32,
+ SQ_EXPORT_RAT_INST_DEC_UINT_RTN = 0x33,
+} ENUM_SQ_EXPORT_RAT_INST;
+typedef enum SQ_IBUF_ST {
+ SQ_IBUF_IB_IDLE = 0x0,
+ SQ_IBUF_IB_INI_WAIT_GNT = 0x1,
+ SQ_IBUF_IB_INI_WAIT_DRET = 0x2,
+ SQ_IBUF_IB_LE_4DW = 0x3,
+ SQ_IBUF_IB_WAIT_DRET = 0x4,
+ SQ_IBUF_IB_EMPTY_WAIT_DRET = 0x5,
+ SQ_IBUF_IB_DRET = 0x6,
+ SQ_IBUF_IB_EMPTY_WAIT_GNT = 0x7,
+} SQ_IBUF_ST;
+typedef enum SQ_INST_STR_ST {
+ SQ_INST_STR_IB_WAVE_NORML = 0x0,
+ SQ_INST_STR_IB_WAVE2ID_NORMAL_INST_AV = 0x1,
+ SQ_INST_STR_IB_WAVE_INTERNAL_INST_AV = 0x2,
+ SQ_INST_STR_IB_WAVE_INST_SKIP_AV = 0x3,
+ SQ_INST_STR_IB_WAVE_SETVSKIP_ST0 = 0x4,
+ SQ_INST_STR_IB_WAVE_SETVSKIP_ST1 = 0x5,
+ SQ_INST_STR_IB_WAVE_NOP_SLEEP_WAIT = 0x6,
+ SQ_INST_STR_IB_WAVE_PC_FROM_SGPR_MSG_WAIT = 0x7,
+} SQ_INST_STR_ST;
+typedef enum SQ_WAVE_IB_ECC_ST {
+ SQ_WAVE_IB_ECC_CLEAN = 0x0,
+ SQ_WAVE_IB_ECC_ERR_CONTINUE = 0x1,
+ SQ_WAVE_IB_ECC_ERR_HALT = 0x2,
+ SQ_WAVE_IB_ECC_WITH_ERR_MSG = 0x3,
+} SQ_WAVE_IB_ECC_ST;
+typedef enum SH_MEM_ADDRESS_MODE {
+ SH_MEM_ADDRESS_MODE_GPUVM64 = 0x0,
+ SH_MEM_ADDRESS_MODE_GPUVM32 = 0x1,
+ SH_MEM_ADDRESS_MODE_HSA64 = 0x2,
+ SH_MEM_ADDRESS_MODE_HSA32 = 0x3,
+} SH_MEM_ADDRESS_MODE;
+typedef enum SH_MEM_ALIGNMENT_MODE {
+ SH_MEM_ALIGNMENT_MODE_DWORD = 0x0,
+ SH_MEM_ALIGNMENT_MODE_DWORD_STRICT = 0x1,
+ SH_MEM_ALIGNMENT_MODE_STRICT = 0x2,
+ SH_MEM_ALIGNMENT_MODE_UNALIGNED = 0x3,
+} SH_MEM_ALIGNMENT_MODE;
+typedef enum SQ_THREAD_TRACE_WAVE_START_COUNT_PREFIX {
+ SQ_THREAD_TRACE_WAVE_START_COUNT_PREFIX_WREXEC = 0x18,
+ SQ_THREAD_TRACE_WAVE_START_COUNT_PREFIX_RESTORE = 0x19,
+} SQ_THREAD_TRACE_WAVE_START_COUNT_PREFIX;
+#define SQ_WAVE_TYPE_PS0 0x0
+#define SQIND_GLOBAL_REGS_OFFSET 0x0
+#define SQIND_GLOBAL_REGS_SIZE 0x8
+#define SQIND_LOCAL_REGS_OFFSET 0x8
+#define SQIND_LOCAL_REGS_SIZE 0x8
+#define SQIND_WAVE_HWREGS_OFFSET 0x10
+#define SQIND_WAVE_HWREGS_SIZE 0x1f0
+#define SQIND_WAVE_SGPRS_OFFSET 0x200
+#define SQIND_WAVE_SGPRS_SIZE 0x200
+#define SQ_GFXDEC_BEGIN 0xa000
+#define SQ_GFXDEC_END 0xc000
+#define SQ_GFXDEC_STATE_ID_SHIFT 0xa
+#define SQDEC_BEGIN 0x2300
+#define SQDEC_END 0x23ff
+#define SQPERFSDEC_BEGIN 0xd9c0
+#define SQPERFSDEC_END 0xda40
+#define SQPERFDDEC_BEGIN 0xd1c0
+#define SQPERFDDEC_END 0xd240
+#define SQGFXUDEC_BEGIN 0xc330
+#define SQGFXUDEC_END 0xc380
+#define SQPWRDEC_BEGIN 0xf08c
+#define SQPWRDEC_END 0xf094
+#define SQ_DISPATCHER_GFX_MIN 0x10
+#define SQ_DISPATCHER_GFX_CNT_PER_RING 0x8
+#define SQ_MAX_PGM_SGPRS 0x68
+#define SQ_MAX_PGM_VGPRS 0x100
+#define SQ_THREAD_TRACE_TIME_UNIT 0x4
+#define SQ_EX_MODE_EXCP_VALU_BASE 0x0
+#define SQ_EX_MODE_EXCP_VALU_SIZE 0x7
+#define SQ_EX_MODE_EXCP_INVALID 0x0
+#define SQ_EX_MODE_EXCP_INPUT_DENORM 0x1
+#define SQ_EX_MODE_EXCP_DIV0 0x2
+#define SQ_EX_MODE_EXCP_OVERFLOW 0x3
+#define SQ_EX_MODE_EXCP_UNDERFLOW 0x4
+#define SQ_EX_MODE_EXCP_INEXACT 0x5
+#define SQ_EX_MODE_EXCP_INT_DIV0 0x6
+#define SQ_EX_MODE_EXCP_ADDR_WATCH 0x7
+#define SQ_EX_MODE_EXCP_MEM_VIOL 0x8
+#define INST_ID_PRIV_START 0x80000000
+#define INST_ID_ECC_INTERRUPT_MSG 0xfffffff0
+#define INST_ID_TTRACE_NEW_PC_MSG 0xfffffff1
+#define INST_ID_HW_TRAP 0xfffffff2
+#define INST_ID_KILL_SEQ 0xfffffff3
+#define INST_ID_SPI_WREXEC 0xfffffff4
+#define INST_ID_HOST_REG_TRAP_MSG 0xfffffffe
+#define SQ_ENC_SOP1_BITS 0xbe800000
+#define SQ_ENC_SOP1_MASK 0xff800000
+#define SQ_ENC_SOP1_FIELD 0x17d
+#define SQ_ENC_SOPC_BITS 0xbf000000
+#define SQ_ENC_SOPC_MASK 0xff800000
+#define SQ_ENC_SOPC_FIELD 0x17e
+#define SQ_ENC_SOPP_BITS 0xbf800000
+#define SQ_ENC_SOPP_MASK 0xff800000
+#define SQ_ENC_SOPP_FIELD 0x17f
+#define SQ_ENC_SOPK_BITS 0xb0000000
+#define SQ_ENC_SOPK_MASK 0xf0000000
+#define SQ_ENC_SOPK_FIELD 0xb
+#define SQ_ENC_SOP2_BITS 0x80000000
+#define SQ_ENC_SOP2_MASK 0xc0000000
+#define SQ_ENC_SOP2_FIELD 0x2
+#define SQ_ENC_SMEM_BITS 0xc0000000
+#define SQ_ENC_SMEM_MASK 0xfc000000
+#define SQ_ENC_SMEM_FIELD 0x30
+#define SQ_ENC_VOP1_BITS 0x7e000000
+#define SQ_ENC_VOP1_MASK 0xfe000000
+#define SQ_ENC_VOP1_FIELD 0x3f
+#define SQ_ENC_VOPC_BITS 0x7c000000
+#define SQ_ENC_VOPC_MASK 0xfe000000
+#define SQ_ENC_VOPC_FIELD 0x3e
+#define SQ_ENC_VOP2_BITS 0x0
+#define SQ_ENC_VOP2_MASK 0x80000000
+#define SQ_ENC_VOP2_FIELD 0x0
+#define SQ_ENC_VINTRP_BITS 0xd4000000
+#define SQ_ENC_VINTRP_MASK 0xfc000000
+#define SQ_ENC_VINTRP_FIELD 0x35
+#define SQ_ENC_VOP3_BITS 0xd0000000
+#define SQ_ENC_VOP3_MASK 0xfc000000
+#define SQ_ENC_VOP3_FIELD 0x34
+#define SQ_ENC_DS_BITS 0xd8000000
+#define SQ_ENC_DS_MASK 0xfc000000
+#define SQ_ENC_DS_FIELD 0x36
+#define SQ_ENC_MUBUF_BITS 0xe0000000
+#define SQ_ENC_MUBUF_MASK 0xfc000000
+#define SQ_ENC_MUBUF_FIELD 0x38
+#define SQ_ENC_MTBUF_BITS 0xe8000000
+#define SQ_ENC_MTBUF_MASK 0xfc000000
+#define SQ_ENC_MTBUF_FIELD 0x3a
+#define SQ_ENC_MIMG_BITS 0xf0000000
+#define SQ_ENC_MIMG_MASK 0xfc000000
+#define SQ_ENC_MIMG_FIELD 0x3c
+#define SQ_ENC_EXP_BITS 0xc4000000
+#define SQ_ENC_EXP_MASK 0xfc000000
+#define SQ_ENC_EXP_FIELD 0x31
+#define SQ_ENC_FLAT_BITS 0xdc000000
+#define SQ_ENC_FLAT_MASK 0xfc000000
+#define SQ_ENC_FLAT_FIELD 0x37
+#define SQ_V_OP3_INTRP_OFFSET 0x274
+#define SQ_WAITCNT_VM_SHIFT 0x0
+#define SQ_SENDMSG_STREAMID_SIZE 0x2
+#define SQ_V_OPC_COUNT 0x100
+#define SQ_V_OP3_INTRP_COUNT 0xc
+#define SQ_XLATE_VOP3_TO_VOP2_OFFSET 0x100
+#define SQ_HWREG_OFFSET_SIZE 0x5
+#define SQ_HWREG_OFFSET_SHIFT 0x6
+#define SQ_V_OP3_3IN_OFFSET 0x1c0
+#define SQ_NUM_ATTR 0x21
+#define SQ_NUM_VGPR 0x100
+#define SQ_XLATE_VOP3_TO_VINTRP_COUNT 0x4
+#define SQ_SENDMSG_MSG_SIZE 0x4
+#define SQ_NUM_TTMP 0xc
+#define SQ_HWREG_ID_SIZE 0x6
+#define SQ_SENDMSG_GSOP_SIZE 0x2
+#define SQ_NUM_SGPR 0x66
+#define SQ_EXP_NUM_MRT 0x8
+#define SQ_SENDMSG_SYSTEM_SIZE 0x3
+#define SQ_WAITCNT_LGKM_SHIFT 0x8
+#define SQ_XLATE_VOP3_TO_VOP2_COUNT 0x40
+#define SQ_V_OP3_3IN_COUNT 0xb0
+#define SQ_V_INTRP_COUNT 0x4
+#define SQ_WAITCNT_EXP_SIZE 0x3
+#define SQ_SENDMSG_SYSTEM_SHIFT 0x4
+#define SQ_EXP_NUM_GDS 0x5
+#define SQ_HWREG_SIZE_SHIFT 0xb
+#define SQ_XLATE_VOP3_TO_VOPC_OFFSET 0x0
+#define SQ_V_OP3_2IN_COUNT 0x80
+#define SQ_XLATE_VOP3_TO_VINTRP_OFFSET 0x270
+#define SQ_SENDMSG_MSG_SHIFT 0x0
+#define SQ_WAITCNT_EXP_SHIFT 0x4
+#define SQ_WAITCNT_VM_SIZE 0x4
+#define SQ_XLATE_VOP3_TO_VOP1_OFFSET 0x140
+#define SQ_SENDMSG_GSOP_SHIFT 0x4
+#define SQ_XLATE_VOP3_TO_VOP1_COUNT 0x80
+#define SQ_SRC_VGPR_BIT 0x100
+#define SQ_V_OP2_COUNT 0x40
+#define SQ_EXP_NUM_PARAM 0x20
+#define SQ_V_OP1_COUNT 0x80
+#define SQ_SENDMSG_STREAMID_SHIFT 0x8
+#define SQ_V_OP3_2IN_OFFSET 0x280
+#define SQ_WAITCNT_LGKM_SIZE 0x4
+#define SQ_XLATE_VOP3_TO_VOPC_COUNT 0x100
+#define SQ_EXP_NUM_POS 0x4
+#define SQ_HWREG_SIZE_SIZE 0x5
+#define SQ_HWREG_ID_SHIFT 0x0
+#define SQ_S_MOV_B32 0x0
+#define SQ_S_MOV_B64 0x1
+#define SQ_S_CMOV_B32 0x2
+#define SQ_S_CMOV_B64 0x3
+#define SQ_S_NOT_B32 0x4
+#define SQ_S_NOT_B64 0x5
+#define SQ_S_WQM_B32 0x6
+#define SQ_S_WQM_B64 0x7
+#define SQ_S_BREV_B32 0x8
+#define SQ_S_BREV_B64 0x9
+#define SQ_S_BCNT0_I32_B32 0xa
+#define SQ_S_BCNT0_I32_B64 0xb
+#define SQ_S_BCNT1_I32_B32 0xc
+#define SQ_S_BCNT1_I32_B64 0xd
+#define SQ_S_FF0_I32_B32 0xe
+#define SQ_S_FF0_I32_B64 0xf
+#define SQ_S_FF1_I32_B32 0x10
+#define SQ_S_FF1_I32_B64 0x11
+#define SQ_S_FLBIT_I32_B32 0x12
+#define SQ_S_FLBIT_I32_B64 0x13
+#define SQ_S_FLBIT_I32 0x14
+#define SQ_S_FLBIT_I32_I64 0x15
+#define SQ_S_SEXT_I32_I8 0x16
+#define SQ_S_SEXT_I32_I16 0x17
+#define SQ_S_BITSET0_B32 0x18
+#define SQ_S_BITSET0_B64 0x19
+#define SQ_S_BITSET1_B32 0x1a
+#define SQ_S_BITSET1_B64 0x1b
+#define SQ_S_GETPC_B64 0x1c
+#define SQ_S_SETPC_B64 0x1d
+#define SQ_S_SWAPPC_B64 0x1e
+#define SQ_S_RFE_B64 0x1f
+#define SQ_S_AND_SAVEEXEC_B64 0x20
+#define SQ_S_OR_SAVEEXEC_B64 0x21
+#define SQ_S_XOR_SAVEEXEC_B64 0x22
+#define SQ_S_ANDN2_SAVEEXEC_B64 0x23
+#define SQ_S_ORN2_SAVEEXEC_B64 0x24
+#define SQ_S_NAND_SAVEEXEC_B64 0x25
+#define SQ_S_NOR_SAVEEXEC_B64 0x26
+#define SQ_S_XNOR_SAVEEXEC_B64 0x27
+#define SQ_S_QUADMASK_B32 0x28
+#define SQ_S_QUADMASK_B64 0x29
+#define SQ_S_MOVRELS_B32 0x2a
+#define SQ_S_MOVRELS_B64 0x2b
+#define SQ_S_MOVRELD_B32 0x2c
+#define SQ_S_MOVRELD_B64 0x2d
+#define SQ_S_CBRANCH_JOIN 0x2e
+#define SQ_S_MOV_REGRD_B32 0x2f
+#define SQ_S_ABS_I32 0x30
+#define SQ_S_MOV_FED_B32 0x31
+#define SQ_S_SET_GPR_IDX_IDX 0x32
+#define SQ_ATTR0 0x0
+#define SQ_S_MOVK_I32 0x0
+#define SQ_S_CMOVK_I32 0x1
+#define SQ_S_CMPK_EQ_I32 0x2
+#define SQ_S_CMPK_LG_I32 0x3
+#define SQ_S_CMPK_GT_I32 0x4
+#define SQ_S_CMPK_GE_I32 0x5
+#define SQ_S_CMPK_LT_I32 0x6
+#define SQ_S_CMPK_LE_I32 0x7
+#define SQ_S_CMPK_EQ_U32 0x8
+#define SQ_S_CMPK_LG_U32 0x9
+#define SQ_S_CMPK_GT_U32 0xa
+#define SQ_S_CMPK_GE_U32 0xb
+#define SQ_S_CMPK_LT_U32 0xc
+#define SQ_S_CMPK_LE_U32 0xd
+#define SQ_S_ADDK_I32 0xe
+#define SQ_S_MULK_I32 0xf
+#define SQ_S_CBRANCH_I_FORK 0x10
+#define SQ_S_GETREG_B32 0x11
+#define SQ_S_SETREG_B32 0x12
+#define SQ_S_GETREG_REGRD_B32 0x13
+#define SQ_S_SETREG_IMM32_B32 0x14
+#define SQ_TBA_LO 0x6c
+#define SQ_TBA_HI 0x6d
+#define SQ_TMA_LO 0x6e
+#define SQ_TMA_HI 0x6f
+#define SQ_TTMP0 0x70
+#define SQ_TTMP1 0x71
+#define SQ_TTMP2 0x72
+#define SQ_TTMP3 0x73
+#define SQ_TTMP4 0x74
+#define SQ_TTMP5 0x75
+#define SQ_TTMP6 0x76
+#define SQ_TTMP7 0x77
+#define SQ_TTMP8 0x78
+#define SQ_TTMP9 0x79
+#define SQ_TTMP10 0x7a
+#define SQ_TTMP11 0x7b
+#define SQ_VGPR0 0x0
+#define SQ_EXP 0x0
+#define SQ_EXP_MRT0 0x0
+#define SQ_EXP_MRTZ 0x8
+#define SQ_EXP_NULL 0x9
+#define SQ_EXP_POS0 0xc
+#define SQ_EXP_PARAM0 0x20
+#define SQ_CNT1 0x0
+#define SQ_CNT2 0x1
+#define SQ_CNT3 0x2
+#define SQ_CNT4 0x3
+#define SQ_S_LOAD_DWORD 0x0
+#define SQ_S_LOAD_DWORDX2 0x1
+#define SQ_S_LOAD_DWORDX4 0x2
+#define SQ_S_LOAD_DWORDX8 0x3
+#define SQ_S_LOAD_DWORDX16 0x4
+#define SQ_S_BUFFER_LOAD_DWORD 0x8
+#define SQ_S_BUFFER_LOAD_DWORDX2 0x9
+#define SQ_S_BUFFER_LOAD_DWORDX4 0xa
+#define SQ_S_BUFFER_LOAD_DWORDX8 0xb
+#define SQ_S_BUFFER_LOAD_DWORDX16 0xc
+#define SQ_S_STORE_DWORD 0x10
+#define SQ_S_STORE_DWORDX2 0x11
+#define SQ_S_STORE_DWORDX4 0x12
+#define SQ_S_BUFFER_STORE_DWORD 0x18
+#define SQ_S_BUFFER_STORE_DWORDX2 0x19
+#define SQ_S_BUFFER_STORE_DWORDX4 0x1a
+#define SQ_S_DCACHE_INV 0x20
+#define SQ_S_DCACHE_WB 0x21
+#define SQ_S_DCACHE_INV_VOL 0x22
+#define SQ_S_DCACHE_WB_VOL 0x23
+#define SQ_S_MEMTIME 0x24
+#define SQ_S_MEMREALTIME 0x25
+#define SQ_S_ATC_PROBE 0x26
+#define SQ_S_ATC_PROBE_BUFFER 0x27
+#define SQ_S_BUFFER_ATOMIC_SWAP 0x40
+#define SQ_S_BUFFER_ATOMIC_CMPSWAP 0x41
+#define SQ_S_BUFFER_ATOMIC_ADD 0x42
+#define SQ_S_BUFFER_ATOMIC_SUB 0x43
+#define SQ_S_BUFFER_ATOMIC_SMIN 0x44
+#define SQ_S_BUFFER_ATOMIC_UMIN 0x45
+#define SQ_S_BUFFER_ATOMIC_SMAX 0x46
+#define SQ_S_BUFFER_ATOMIC_UMAX 0x47
+#define SQ_S_BUFFER_ATOMIC_AND 0x48
+#define SQ_S_BUFFER_ATOMIC_OR 0x49
+#define SQ_S_BUFFER_ATOMIC_XOR 0x4a
+#define SQ_S_BUFFER_ATOMIC_INC 0x4b
+#define SQ_S_BUFFER_ATOMIC_DEC 0x4c
+#define SQ_S_BUFFER_ATOMIC_SWAP_X2 0x60
+#define SQ_S_BUFFER_ATOMIC_CMPSWAP_X2 0x61
+#define SQ_S_BUFFER_ATOMIC_ADD_X2 0x62
+#define SQ_S_BUFFER_ATOMIC_SUB_X2 0x63
+#define SQ_S_BUFFER_ATOMIC_SMIN_X2 0x64
+#define SQ_S_BUFFER_ATOMIC_UMIN_X2 0x65
+#define SQ_S_BUFFER_ATOMIC_SMAX_X2 0x66
+#define SQ_S_BUFFER_ATOMIC_UMAX_X2 0x67
+#define SQ_S_BUFFER_ATOMIC_AND_X2 0x68
+#define SQ_S_BUFFER_ATOMIC_OR_X2 0x69
+#define SQ_S_BUFFER_ATOMIC_XOR_X2 0x6a
+#define SQ_S_BUFFER_ATOMIC_INC_X2 0x6b
+#define SQ_S_BUFFER_ATOMIC_DEC_X2 0x6c
+#define SQ_F 0x0
+#define SQ_LT 0x1
+#define SQ_EQ 0x2
+#define SQ_LE 0x3
+#define SQ_GT 0x4
+#define SQ_LG 0x5
+#define SQ_GE 0x6
+#define SQ_O 0x7
+#define SQ_U 0x8
+#define SQ_NGE 0x9
+#define SQ_NLG 0xa
+#define SQ_NGT 0xb
+#define SQ_NLE 0xc
+#define SQ_NEQ 0xd
+#define SQ_NLT 0xe
+#define SQ_TRU 0xf
+#define SQ_V_CMP_CLASS_F32 0x10
+#define SQ_V_CMPX_CLASS_F32 0x11
+#define SQ_V_CMP_CLASS_F64 0x12
+#define SQ_V_CMPX_CLASS_F64 0x13
+#define SQ_V_CMP_CLASS_F16 0x14
+#define SQ_V_CMPX_CLASS_F16 0x15
+#define SQ_V_CMP_F_F16 0x20
+#define SQ_V_CMP_LT_F16 0x21
+#define SQ_V_CMP_EQ_F16 0x22
+#define SQ_V_CMP_LE_F16 0x23
+#define SQ_V_CMP_GT_F16 0x24
+#define SQ_V_CMP_LG_F16 0x25
+#define SQ_V_CMP_GE_F16 0x26
+#define SQ_V_CMP_O_F16 0x27
+#define SQ_V_CMP_U_F16 0x28
+#define SQ_V_CMP_NGE_F16 0x29
+#define SQ_V_CMP_NLG_F16 0x2a
+#define SQ_V_CMP_NGT_F16 0x2b
+#define SQ_V_CMP_NLE_F16 0x2c
+#define SQ_V_CMP_NEQ_F16 0x2d
+#define SQ_V_CMP_NLT_F16 0x2e
+#define SQ_V_CMP_TRU_F16 0x2f
+#define SQ_V_CMPX_F_F16 0x30
+#define SQ_V_CMPX_LT_F16 0x31
+#define SQ_V_CMPX_EQ_F16 0x32
+#define SQ_V_CMPX_LE_F16 0x33
+#define SQ_V_CMPX_GT_F16 0x34
+#define SQ_V_CMPX_LG_F16 0x35
+#define SQ_V_CMPX_GE_F16 0x36
+#define SQ_V_CMPX_O_F16 0x37
+#define SQ_V_CMPX_U_F16 0x38
+#define SQ_V_CMPX_NGE_F16 0x39
+#define SQ_V_CMPX_NLG_F16 0x3a
+#define SQ_V_CMPX_NGT_F16 0x3b
+#define SQ_V_CMPX_NLE_F16 0x3c
+#define SQ_V_CMPX_NEQ_F16 0x3d
+#define SQ_V_CMPX_NLT_F16 0x3e
+#define SQ_V_CMPX_TRU_F16 0x3f
+#define SQ_V_CMP_F_F32 0x40
+#define SQ_V_CMP_LT_F32 0x41
+#define SQ_V_CMP_EQ_F32 0x42
+#define SQ_V_CMP_LE_F32 0x43
+#define SQ_V_CMP_GT_F32 0x44
+#define SQ_V_CMP_LG_F32 0x45
+#define SQ_V_CMP_GE_F32 0x46
+#define SQ_V_CMP_O_F32 0x47
+#define SQ_V_CMP_U_F32 0x48
+#define SQ_V_CMP_NGE_F32 0x49
+#define SQ_V_CMP_NLG_F32 0x4a
+#define SQ_V_CMP_NGT_F32 0x4b
+#define SQ_V_CMP_NLE_F32 0x4c
+#define SQ_V_CMP_NEQ_F32 0x4d
+#define SQ_V_CMP_NLT_F32 0x4e
+#define SQ_V_CMP_TRU_F32 0x4f
+#define SQ_V_CMPX_F_F32 0x50
+#define SQ_V_CMPX_LT_F32 0x51
+#define SQ_V_CMPX_EQ_F32 0x52
+#define SQ_V_CMPX_LE_F32 0x53
+#define SQ_V_CMPX_GT_F32 0x54
+#define SQ_V_CMPX_LG_F32 0x55
+#define SQ_V_CMPX_GE_F32 0x56
+#define SQ_V_CMPX_O_F32 0x57
+#define SQ_V_CMPX_U_F32 0x58
+#define SQ_V_CMPX_NGE_F32 0x59
+#define SQ_V_CMPX_NLG_F32 0x5a
+#define SQ_V_CMPX_NGT_F32 0x5b
+#define SQ_V_CMPX_NLE_F32 0x5c
+#define SQ_V_CMPX_NEQ_F32 0x5d
+#define SQ_V_CMPX_NLT_F32 0x5e
+#define SQ_V_CMPX_TRU_F32 0x5f
+#define SQ_V_CMP_F_F64 0x60
+#define SQ_V_CMP_LT_F64 0x61
+#define SQ_V_CMP_EQ_F64 0x62
+#define SQ_V_CMP_LE_F64 0x63
+#define SQ_V_CMP_GT_F64 0x64
+#define SQ_V_CMP_LG_F64 0x65
+#define SQ_V_CMP_GE_F64 0x66
+#define SQ_V_CMP_O_F64 0x67
+#define SQ_V_CMP_U_F64 0x68
+#define SQ_V_CMP_NGE_F64 0x69
+#define SQ_V_CMP_NLG_F64 0x6a
+#define SQ_V_CMP_NGT_F64 0x6b
+#define SQ_V_CMP_NLE_F64 0x6c
+#define SQ_V_CMP_NEQ_F64 0x6d
+#define SQ_V_CMP_NLT_F64 0x6e
+#define SQ_V_CMP_TRU_F64 0x6f
+#define SQ_V_CMPX_F_F64 0x70
+#define SQ_V_CMPX_LT_F64 0x71
+#define SQ_V_CMPX_EQ_F64 0x72
+#define SQ_V_CMPX_LE_F64 0x73
+#define SQ_V_CMPX_GT_F64 0x74
+#define SQ_V_CMPX_LG_F64 0x75
+#define SQ_V_CMPX_GE_F64 0x76
+#define SQ_V_CMPX_O_F64 0x77
+#define SQ_V_CMPX_U_F64 0x78
+#define SQ_V_CMPX_NGE_F64 0x79
+#define SQ_V_CMPX_NLG_F64 0x7a
+#define SQ_V_CMPX_NGT_F64 0x7b
+#define SQ_V_CMPX_NLE_F64 0x7c
+#define SQ_V_CMPX_NEQ_F64 0x7d
+#define SQ_V_CMPX_NLT_F64 0x7e
+#define SQ_V_CMPX_TRU_F64 0x7f
+#define SQ_V_CMP_F_I16 0xa0
+#define SQ_V_CMP_LT_I16 0xa1
+#define SQ_V_CMP_EQ_I16 0xa2
+#define SQ_V_CMP_LE_I16 0xa3
+#define SQ_V_CMP_GT_I16 0xa4
+#define SQ_V_CMP_NE_I16 0xa5
+#define SQ_V_CMP_GE_I16 0xa6
+#define SQ_V_CMP_T_I16 0xa7
+#define SQ_V_CMP_F_U16 0xa8
+#define SQ_V_CMP_LT_U16 0xa9
+#define SQ_V_CMP_EQ_U16 0xaa
+#define SQ_V_CMP_LE_U16 0xab
+#define SQ_V_CMP_GT_U16 0xac
+#define SQ_V_CMP_NE_U16 0xad
+#define SQ_V_CMP_GE_U16 0xae
+#define SQ_V_CMP_T_U16 0xaf
+#define SQ_V_CMPX_F_I16 0xb0
+#define SQ_V_CMPX_LT_I16 0xb1
+#define SQ_V_CMPX_EQ_I16 0xb2
+#define SQ_V_CMPX_LE_I16 0xb3
+#define SQ_V_CMPX_GT_I16 0xb4
+#define SQ_V_CMPX_NE_I16 0xb5
+#define SQ_V_CMPX_GE_I16 0xb6
+#define SQ_V_CMPX_T_I16 0xb7
+#define SQ_V_CMPX_F_U16 0xb8
+#define SQ_V_CMPX_LT_U16 0xb9
+#define SQ_V_CMPX_EQ_U16 0xba
+#define SQ_V_CMPX_LE_U16 0xbb
+#define SQ_V_CMPX_GT_U16 0xbc
+#define SQ_V_CMPX_NE_U16 0xbd
+#define SQ_V_CMPX_GE_U16 0xbe
+#define SQ_V_CMPX_T_U16 0xbf
+#define SQ_V_CMP_F_I32 0xc0
+#define SQ_V_CMP_LT_I32 0xc1
+#define SQ_V_CMP_EQ_I32 0xc2
+#define SQ_V_CMP_LE_I32 0xc3
+#define SQ_V_CMP_GT_I32 0xc4
+#define SQ_V_CMP_NE_I32 0xc5
+#define SQ_V_CMP_GE_I32 0xc6
+#define SQ_V_CMP_T_I32 0xc7
+#define SQ_V_CMP_F_U32 0xc8
+#define SQ_V_CMP_LT_U32 0xc9
+#define SQ_V_CMP_EQ_U32 0xca
+#define SQ_V_CMP_LE_U32 0xcb
+#define SQ_V_CMP_GT_U32 0xcc
+#define SQ_V_CMP_NE_U32 0xcd
+#define SQ_V_CMP_GE_U32 0xce
+#define SQ_V_CMP_T_U32 0xcf
+#define SQ_V_CMPX_F_I32 0xd0
+#define SQ_V_CMPX_LT_I32 0xd1
+#define SQ_V_CMPX_EQ_I32 0xd2
+#define SQ_V_CMPX_LE_I32 0xd3
+#define SQ_V_CMPX_GT_I32 0xd4
+#define SQ_V_CMPX_NE_I32 0xd5
+#define SQ_V_CMPX_GE_I32 0xd6
+#define SQ_V_CMPX_T_I32 0xd7
+#define SQ_V_CMPX_F_U32 0xd8
+#define SQ_V_CMPX_LT_U32 0xd9
+#define SQ_V_CMPX_EQ_U32 0xda
+#define SQ_V_CMPX_LE_U32 0xdb
+#define SQ_V_CMPX_GT_U32 0xdc
+#define SQ_V_CMPX_NE_U32 0xdd
+#define SQ_V_CMPX_GE_U32 0xde
+#define SQ_V_CMPX_T_U32 0xdf
+#define SQ_V_CMP_F_I64 0xe0
+#define SQ_V_CMP_LT_I64 0xe1
+#define SQ_V_CMP_EQ_I64 0xe2
+#define SQ_V_CMP_LE_I64 0xe3
+#define SQ_V_CMP_GT_I64 0xe4
+#define SQ_V_CMP_NE_I64 0xe5
+#define SQ_V_CMP_GE_I64 0xe6
+#define SQ_V_CMP_T_I64 0xe7
+#define SQ_V_CMP_F_U64 0xe8
+#define SQ_V_CMP_LT_U64 0xe9
+#define SQ_V_CMP_EQ_U64 0xea
+#define SQ_V_CMP_LE_U64 0xeb
+#define SQ_V_CMP_GT_U64 0xec
+#define SQ_V_CMP_NE_U64 0xed
+#define SQ_V_CMP_GE_U64 0xee
+#define SQ_V_CMP_T_U64 0xef
+#define SQ_V_CMPX_F_I64 0xf0
+#define SQ_V_CMPX_LT_I64 0xf1
+#define SQ_V_CMPX_EQ_I64 0xf2
+#define SQ_V_CMPX_LE_I64 0xf3
+#define SQ_V_CMPX_GT_I64 0xf4
+#define SQ_V_CMPX_NE_I64 0xf5
+#define SQ_V_CMPX_GE_I64 0xf6
+#define SQ_V_CMPX_T_I64 0xf7
+#define SQ_V_CMPX_F_U64 0xf8
+#define SQ_V_CMPX_LT_U64 0xf9
+#define SQ_V_CMPX_EQ_U64 0xfa
+#define SQ_V_CMPX_LE_U64 0xfb
+#define SQ_V_CMPX_GT_U64 0xfc
+#define SQ_V_CMPX_NE_U64 0xfd
+#define SQ_V_CMPX_GE_U64 0xfe
+#define SQ_V_CMPX_T_U64 0xff
+#define SQ_L1 0x1
+#define SQ_L2 0x2
+#define SQ_L3 0x3
+#define SQ_L4 0x4
+#define SQ_L5 0x5
+#define SQ_L6 0x6
+#define SQ_L7 0x7
+#define SQ_L8 0x8
+#define SQ_L9 0x9
+#define SQ_L10 0xa
+#define SQ_L11 0xb
+#define SQ_L12 0xc
+#define SQ_L13 0xd
+#define SQ_L14 0xe
+#define SQ_L15 0xf
+#define SQ_SGPR0 0x0
+#define SQ_SDWA_UNUSED_PAD 0x0
+#define SQ_SDWA_UNUSED_SEXT 0x1
+#define SQ_SDWA_UNUSED_PRESERVE 0x2
+#define SQ_F 0x0
+#define SQ_LT 0x1
+#define SQ_EQ 0x2
+#define SQ_LE 0x3
+#define SQ_GT 0x4
+#define SQ_NE 0x5
+#define SQ_GE 0x6
+#define SQ_T 0x7
+#define SQ_SRC_64_INT 0xc0
+#define SQ_SRC_M_1_INT 0xc1
+#define SQ_SRC_M_2_INT 0xc2
+#define SQ_SRC_M_3_INT 0xc3
+#define SQ_SRC_M_4_INT 0xc4
+#define SQ_SRC_M_5_INT 0xc5
+#define SQ_SRC_M_6_INT 0xc6
+#define SQ_SRC_M_7_INT 0xc7
+#define SQ_SRC_M_8_INT 0xc8
+#define SQ_SRC_M_9_INT 0xc9
+#define SQ_SRC_M_10_INT 0xca
+#define SQ_SRC_M_11_INT 0xcb
+#define SQ_SRC_M_12_INT 0xcc
+#define SQ_SRC_M_13_INT 0xcd
+#define SQ_SRC_M_14_INT 0xce
+#define SQ_SRC_M_15_INT 0xcf
+#define SQ_SRC_M_16_INT 0xd0
+#define SQ_SRC_0_5 0xf0
+#define SQ_SRC_M_0_5 0xf1
+#define SQ_SRC_1 0xf2
+#define SQ_SRC_M_1 0xf3
+#define SQ_SRC_2 0xf4
+#define SQ_SRC_M_2 0xf5
+#define SQ_SRC_4 0xf6
+#define SQ_SRC_M_4 0xf7
+#define SQ_SRC_INV_2PI 0xf8
+#define SQ_SRC_0 0x80
+#define SQ_SRC_1_INT 0x81
+#define SQ_SRC_2_INT 0x82
+#define SQ_SRC_3_INT 0x83
+#define SQ_SRC_4_INT 0x84
+#define SQ_SRC_5_INT 0x85
+#define SQ_SRC_6_INT 0x86
+#define SQ_SRC_7_INT 0x87
+#define SQ_SRC_8_INT 0x88
+#define SQ_SRC_9_INT 0x89
+#define SQ_SRC_10_INT 0x8a
+#define SQ_SRC_11_INT 0x8b
+#define SQ_SRC_12_INT 0x8c
+#define SQ_SRC_13_INT 0x8d
+#define SQ_SRC_14_INT 0x8e
+#define SQ_SRC_15_INT 0x8f
+#define SQ_SRC_16_INT 0x90
+#define SQ_SRC_17_INT 0x91
+#define SQ_SRC_18_INT 0x92
+#define SQ_SRC_19_INT 0x93
+#define SQ_SRC_20_INT 0x94
+#define SQ_SRC_21_INT 0x95
+#define SQ_SRC_22_INT 0x96
+#define SQ_SRC_23_INT 0x97
+#define SQ_SRC_24_INT 0x98
+#define SQ_SRC_25_INT 0x99
+#define SQ_SRC_26_INT 0x9a
+#define SQ_SRC_27_INT 0x9b
+#define SQ_SRC_28_INT 0x9c
+#define SQ_SRC_29_INT 0x9d
+#define SQ_SRC_30_INT 0x9e
+#define SQ_SRC_31_INT 0x9f
+#define SQ_SRC_32_INT 0xa0
+#define SQ_SRC_33_INT 0xa1
+#define SQ_SRC_34_INT 0xa2
+#define SQ_SRC_35_INT 0xa3
+#define SQ_SRC_36_INT 0xa4
+#define SQ_SRC_37_INT 0xa5
+#define SQ_SRC_38_INT 0xa6
+#define SQ_SRC_39_INT 0xa7
+#define SQ_SRC_40_INT 0xa8
+#define SQ_SRC_41_INT 0xa9
+#define SQ_SRC_42_INT 0xaa
+#define SQ_SRC_43_INT 0xab
+#define SQ_SRC_44_INT 0xac
+#define SQ_SRC_45_INT 0xad
+#define SQ_SRC_46_INT 0xae
+#define SQ_SRC_47_INT 0xaf
+#define SQ_SRC_48_INT 0xb0
+#define SQ_SRC_49_INT 0xb1
+#define SQ_SRC_50_INT 0xb2
+#define SQ_SRC_51_INT 0xb3
+#define SQ_SRC_52_INT 0xb4
+#define SQ_SRC_53_INT 0xb5
+#define SQ_SRC_54_INT 0xb6
+#define SQ_SRC_55_INT 0xb7
+#define SQ_SRC_56_INT 0xb8
+#define SQ_SRC_57_INT 0xb9
+#define SQ_SRC_58_INT 0xba
+#define SQ_SRC_59_INT 0xbb
+#define SQ_SRC_60_INT 0xbc
+#define SQ_SRC_61_INT 0xbd
+#define SQ_SRC_62_INT 0xbe
+#define SQ_SRC_63_INT 0xbf
+#define SQ_DS_ADD_U32 0x0
+#define SQ_DS_SUB_U32 0x1
+#define SQ_DS_RSUB_U32 0x2
+#define SQ_DS_INC_U32 0x3
+#define SQ_DS_DEC_U32 0x4
+#define SQ_DS_MIN_I32 0x5
+#define SQ_DS_MAX_I32 0x6
+#define SQ_DS_MIN_U32 0x7
+#define SQ_DS_MAX_U32 0x8
+#define SQ_DS_AND_B32 0x9
+#define SQ_DS_OR_B32 0xa
+#define SQ_DS_XOR_B32 0xb
+#define SQ_DS_MSKOR_B32 0xc
+#define SQ_DS_WRITE_B32 0xd
+#define SQ_DS_WRITE2_B32 0xe
+#define SQ_DS_WRITE2ST64_B32 0xf
+#define SQ_DS_CMPST_B32 0x10
+#define SQ_DS_CMPST_F32 0x11
+#define SQ_DS_MIN_F32 0x12
+#define SQ_DS_MAX_F32 0x13
+#define SQ_DS_NOP 0x14
+#define SQ_DS_ADD_F32 0x15
+#define SQ_DS_WRITE_B8 0x1e
+#define SQ_DS_WRITE_B16 0x1f
+#define SQ_DS_ADD_RTN_U32 0x20
+#define SQ_DS_SUB_RTN_U32 0x21
+#define SQ_DS_RSUB_RTN_U32 0x22
+#define SQ_DS_INC_RTN_U32 0x23
+#define SQ_DS_DEC_RTN_U32 0x24
+#define SQ_DS_MIN_RTN_I32 0x25
+#define SQ_DS_MAX_RTN_I32 0x26
+#define SQ_DS_MIN_RTN_U32 0x27
+#define SQ_DS_MAX_RTN_U32 0x28
+#define SQ_DS_AND_RTN_B32 0x29
+#define SQ_DS_OR_RTN_B32 0x2a
+#define SQ_DS_XOR_RTN_B32 0x2b
+#define SQ_DS_MSKOR_RTN_B32 0x2c
+#define SQ_DS_WRXCHG_RTN_B32 0x2d
+#define SQ_DS_WRXCHG2_RTN_B32 0x2e
+#define SQ_DS_WRXCHG2ST64_RTN_B32 0x2f
+#define SQ_DS_CMPST_RTN_B32 0x30
+#define SQ_DS_CMPST_RTN_F32 0x31
+#define SQ_DS_MIN_RTN_F32 0x32
+#define SQ_DS_MAX_RTN_F32 0x33
+#define SQ_DS_WRAP_RTN_B32 0x34
+#define SQ_DS_ADD_RTN_F32 0x35
+#define SQ_DS_READ_B32 0x36
+#define SQ_DS_READ2_B32 0x37
+#define SQ_DS_READ2ST64_B32 0x38
+#define SQ_DS_READ_I8 0x39
+#define SQ_DS_READ_U8 0x3a
+#define SQ_DS_READ_I16 0x3b
+#define SQ_DS_READ_U16 0x3c
+#define SQ_DS_SWIZZLE_B32 0x3d
+#define SQ_DS_PERMUTE_B32 0x3e
+#define SQ_DS_BPERMUTE_B32 0x3f
+#define SQ_DS_ADD_U64 0x40
+#define SQ_DS_SUB_U64 0x41
+#define SQ_DS_RSUB_U64 0x42
+#define SQ_DS_INC_U64 0x43
+#define SQ_DS_DEC_U64 0x44
+#define SQ_DS_MIN_I64 0x45
+#define SQ_DS_MAX_I64 0x46
+#define SQ_DS_MIN_U64 0x47
+#define SQ_DS_MAX_U64 0x48
+#define SQ_DS_AND_B64 0x49
+#define SQ_DS_OR_B64 0x4a
+#define SQ_DS_XOR_B64 0x4b
+#define SQ_DS_MSKOR_B64 0x4c
+#define SQ_DS_WRITE_B64 0x4d
+#define SQ_DS_WRITE2_B64 0x4e
+#define SQ_DS_WRITE2ST64_B64 0x4f
+#define SQ_DS_CMPST_B64 0x50
+#define SQ_DS_CMPST_F64 0x51
+#define SQ_DS_MIN_F64 0x52
+#define SQ_DS_MAX_F64 0x53
+#define SQ_DS_ADD_RTN_U64 0x60
+#define SQ_DS_SUB_RTN_U64 0x61
+#define SQ_DS_RSUB_RTN_U64 0x62
+#define SQ_DS_INC_RTN_U64 0x63
+#define SQ_DS_DEC_RTN_U64 0x64
+#define SQ_DS_MIN_RTN_I64 0x65
+#define SQ_DS_MAX_RTN_I64 0x66
+#define SQ_DS_MIN_RTN_U64 0x67
+#define SQ_DS_MAX_RTN_U64 0x68
+#define SQ_DS_AND_RTN_B64 0x69
+#define SQ_DS_OR_RTN_B64 0x6a
+#define SQ_DS_XOR_RTN_B64 0x6b
+#define SQ_DS_MSKOR_RTN_B64 0x6c
+#define SQ_DS_WRXCHG_RTN_B64 0x6d
+#define SQ_DS_WRXCHG2_RTN_B64 0x6e
+#define SQ_DS_WRXCHG2ST64_RTN_B64 0x6f
+#define SQ_DS_CMPST_RTN_B64 0x70
+#define SQ_DS_CMPST_RTN_F64 0x71
+#define SQ_DS_MIN_RTN_F64 0x72
+#define SQ_DS_MAX_RTN_F64 0x73
+#define SQ_DS_READ_B64 0x76
+#define SQ_DS_READ2_B64 0x77
+#define SQ_DS_READ2ST64_B64 0x78
+#define SQ_DS_CONDXCHG32_RTN_B64 0x7e
+#define SQ_DS_ADD_SRC2_U32 0x80
+#define SQ_DS_SUB_SRC2_U32 0x81
+#define SQ_DS_RSUB_SRC2_U32 0x82
+#define SQ_DS_INC_SRC2_U32 0x83
+#define SQ_DS_DEC_SRC2_U32 0x84
+#define SQ_DS_MIN_SRC2_I32 0x85
+#define SQ_DS_MAX_SRC2_I32 0x86
+#define SQ_DS_MIN_SRC2_U32 0x87
+#define SQ_DS_MAX_SRC2_U32 0x88
+#define SQ_DS_AND_SRC2_B32 0x89
+#define SQ_DS_OR_SRC2_B32 0x8a
+#define SQ_DS_XOR_SRC2_B32 0x8b
+#define SQ_DS_WRITE_SRC2_B32 0x8d
+#define SQ_DS_MIN_SRC2_F32 0x92
+#define SQ_DS_MAX_SRC2_F32 0x93
+#define SQ_DS_ADD_SRC2_F32 0x95
+#define SQ_DS_GWS_SEMA_RELEASE_ALL 0x98
+#define SQ_DS_GWS_INIT 0x99
+#define SQ_DS_GWS_SEMA_V 0x9a
+#define SQ_DS_GWS_SEMA_BR 0x9b
+#define SQ_DS_GWS_SEMA_P 0x9c
+#define SQ_DS_GWS_BARRIER 0x9d
+#define SQ_DS_CONSUME 0xbd
+#define SQ_DS_APPEND 0xbe
+#define SQ_DS_ORDERED_COUNT 0xbf
+#define SQ_DS_ADD_SRC2_U64 0xc0
+#define SQ_DS_SUB_SRC2_U64 0xc1
+#define SQ_DS_RSUB_SRC2_U64 0xc2
+#define SQ_DS_INC_SRC2_U64 0xc3
+#define SQ_DS_DEC_SRC2_U64 0xc4
+#define SQ_DS_MIN_SRC2_I64 0xc5
+#define SQ_DS_MAX_SRC2_I64 0xc6
+#define SQ_DS_MIN_SRC2_U64 0xc7
+#define SQ_DS_MAX_SRC2_U64 0xc8
+#define SQ_DS_AND_SRC2_B64 0xc9
+#define SQ_DS_OR_SRC2_B64 0xca
+#define SQ_DS_XOR_SRC2_B64 0xcb
+#define SQ_DS_WRITE_SRC2_B64 0xcd
+#define SQ_DS_MIN_SRC2_F64 0xd2
+#define SQ_DS_MAX_SRC2_F64 0xd3
+#define SQ_DS_WRITE_B96 0xde
+#define SQ_DS_WRITE_B128 0xdf
+#define SQ_DS_CONDXCHG32_RTN_B128 0xfd
+#define SQ_DS_READ_B96 0xfe
+#define SQ_DS_READ_B128 0xff
+#define SQ_BUFFER_LOAD_FORMAT_X 0x0
+#define SQ_BUFFER_LOAD_FORMAT_XY 0x1
+#define SQ_BUFFER_LOAD_FORMAT_XYZ 0x2
+#define SQ_BUFFER_LOAD_FORMAT_XYZW 0x3
+#define SQ_BUFFER_STORE_FORMAT_X 0x4
+#define SQ_BUFFER_STORE_FORMAT_XY 0x5
+#define SQ_BUFFER_STORE_FORMAT_XYZ 0x6
+#define SQ_BUFFER_STORE_FORMAT_XYZW 0x7
+#define SQ_BUFFER_LOAD_FORMAT_D16_X 0x8
+#define SQ_BUFFER_LOAD_FORMAT_D16_XY 0x9
+#define SQ_BUFFER_LOAD_FORMAT_D16_XYZ 0xa
+#define SQ_BUFFER_LOAD_FORMAT_D16_XYZW 0xb
+#define SQ_BUFFER_STORE_FORMAT_D16_X 0xc
+#define SQ_BUFFER_STORE_FORMAT_D16_XY 0xd
+#define SQ_BUFFER_STORE_FORMAT_D16_XYZ 0xe
+#define SQ_BUFFER_STORE_FORMAT_D16_XYZW 0xf
+#define SQ_BUFFER_LOAD_UBYTE 0x10
+#define SQ_BUFFER_LOAD_SBYTE 0x11
+#define SQ_BUFFER_LOAD_USHORT 0x12
+#define SQ_BUFFER_LOAD_SSHORT 0x13
+#define SQ_BUFFER_LOAD_DWORD 0x14
+#define SQ_BUFFER_LOAD_DWORDX2 0x15
+#define SQ_BUFFER_LOAD_DWORDX3 0x16
+#define SQ_BUFFER_LOAD_DWORDX4 0x17
+#define SQ_BUFFER_STORE_BYTE 0x18
+#define SQ_BUFFER_STORE_SHORT 0x1a
+#define SQ_BUFFER_STORE_DWORD 0x1c
+#define SQ_BUFFER_STORE_DWORDX2 0x1d
+#define SQ_BUFFER_STORE_DWORDX3 0x1e
+#define SQ_BUFFER_STORE_DWORDX4 0x1f
+#define SQ_BUFFER_STORE_LDS_DWORD 0x3d
+#define SQ_BUFFER_WBINVL1 0x3e
+#define SQ_BUFFER_WBINVL1_VOL 0x3f
+#define SQ_BUFFER_ATOMIC_SWAP 0x40
+#define SQ_BUFFER_ATOMIC_CMPSWAP 0x41
+#define SQ_BUFFER_ATOMIC_ADD 0x42
+#define SQ_BUFFER_ATOMIC_SUB 0x43
+#define SQ_BUFFER_ATOMIC_SMIN 0x44
+#define SQ_BUFFER_ATOMIC_UMIN 0x45
+#define SQ_BUFFER_ATOMIC_SMAX 0x46
+#define SQ_BUFFER_ATOMIC_UMAX 0x47
+#define SQ_BUFFER_ATOMIC_AND 0x48
+#define SQ_BUFFER_ATOMIC_OR 0x49
+#define SQ_BUFFER_ATOMIC_XOR 0x4a
+#define SQ_BUFFER_ATOMIC_INC 0x4b
+#define SQ_BUFFER_ATOMIC_DEC 0x4c
+#define SQ_BUFFER_ATOMIC_SWAP_X2 0x60
+#define SQ_BUFFER_ATOMIC_CMPSWAP_X2 0x61
+#define SQ_BUFFER_ATOMIC_ADD_X2 0x62
+#define SQ_BUFFER_ATOMIC_SUB_X2 0x63
+#define SQ_BUFFER_ATOMIC_SMIN_X2 0x64
+#define SQ_BUFFER_ATOMIC_UMIN_X2 0x65
+#define SQ_BUFFER_ATOMIC_SMAX_X2 0x66
+#define SQ_BUFFER_ATOMIC_UMAX_X2 0x67
+#define SQ_BUFFER_ATOMIC_AND_X2 0x68
+#define SQ_BUFFER_ATOMIC_OR_X2 0x69
+#define SQ_BUFFER_ATOMIC_XOR_X2 0x6a
+#define SQ_BUFFER_ATOMIC_INC_X2 0x6b
+#define SQ_BUFFER_ATOMIC_DEC_X2 0x6c
+#define SQ_EXEC_LO 0x7e
+#define SQ_EXEC_HI 0x7f
+#define SQ_SRC_SCC 0xfd
+#define SQ_OMOD_OFF 0x0
+#define SQ_OMOD_M2 0x1
+#define SQ_OMOD_M4 0x2
+#define SQ_OMOD_D2 0x3
+#define SQ_DPP_QUAD_PERM 0x0
+#define SQ_DPP_ROW_SL1 0x101
+#define SQ_DPP_ROW_SL2 0x102
+#define SQ_DPP_ROW_SL3 0x103
+#define SQ_DPP_ROW_SL4 0x104
+#define SQ_DPP_ROW_SL5 0x105
+#define SQ_DPP_ROW_SL6 0x106
+#define SQ_DPP_ROW_SL7 0x107
+#define SQ_DPP_ROW_SL8 0x108
+#define SQ_DPP_ROW_SL9 0x109
+#define SQ_DPP_ROW_SL10 0x10a
+#define SQ_DPP_ROW_SL11 0x10b
+#define SQ_DPP_ROW_SL12 0x10c
+#define SQ_DPP_ROW_SL13 0x10d
+#define SQ_DPP_ROW_SL14 0x10e
+#define SQ_DPP_ROW_SL15 0x10f
+#define SQ_DPP_ROW_SR1 0x111
+#define SQ_DPP_ROW_SR2 0x112
+#define SQ_DPP_ROW_SR3 0x113
+#define SQ_DPP_ROW_SR4 0x114
+#define SQ_DPP_ROW_SR5 0x115
+#define SQ_DPP_ROW_SR6 0x116
+#define SQ_DPP_ROW_SR7 0x117
+#define SQ_DPP_ROW_SR8 0x118
+#define SQ_DPP_ROW_SR9 0x119
+#define SQ_DPP_ROW_SR10 0x11a
+#define SQ_DPP_ROW_SR11 0x11b
+#define SQ_DPP_ROW_SR12 0x11c
+#define SQ_DPP_ROW_SR13 0x11d
+#define SQ_DPP_ROW_SR14 0x11e
+#define SQ_DPP_ROW_SR15 0x11f
+#define SQ_DPP_ROW_RR1 0x121
+#define SQ_DPP_ROW_RR2 0x122
+#define SQ_DPP_ROW_RR3 0x123
+#define SQ_DPP_ROW_RR4 0x124
+#define SQ_DPP_ROW_RR5 0x125
+#define SQ_DPP_ROW_RR6 0x126
+#define SQ_DPP_ROW_RR7 0x127
+#define SQ_DPP_ROW_RR8 0x128
+#define SQ_DPP_ROW_RR9 0x129
+#define SQ_DPP_ROW_RR10 0x12a
+#define SQ_DPP_ROW_RR11 0x12b
+#define SQ_DPP_ROW_RR12 0x12c
+#define SQ_DPP_ROW_RR13 0x12d
+#define SQ_DPP_ROW_RR14 0x12e
+#define SQ_DPP_ROW_RR15 0x12f
+#define SQ_DPP_WF_SL1 0x130
+#define SQ_DPP_WF_RL1 0x134
+#define SQ_DPP_WF_SR1 0x138
+#define SQ_DPP_WF_RR1 0x13c
+#define SQ_DPP_ROW_MIRROR 0x140
+#define SQ_DPP_ROW_HALF_MIRROR 0x141
+#define SQ_DPP_ROW_BCAST15 0x142
+#define SQ_DPP_ROW_BCAST31 0x143
+#define SQ_EXP_GDS0 0x18
+#define SQ_GS_OP_NOP 0x0
+#define SQ_GS_OP_CUT 0x1
+#define SQ_GS_OP_EMIT 0x2
+#define SQ_GS_OP_EMIT_CUT 0x3
+#define SQ_IMAGE_LOAD 0x0
+#define SQ_IMAGE_LOAD_MIP 0x1
+#define SQ_IMAGE_LOAD_PCK 0x2
+#define SQ_IMAGE_LOAD_PCK_SGN 0x3
+#define SQ_IMAGE_LOAD_MIP_PCK 0x4
+#define SQ_IMAGE_LOAD_MIP_PCK_SGN 0x5
+#define SQ_IMAGE_STORE 0x8
+#define SQ_IMAGE_STORE_MIP 0x9
+#define SQ_IMAGE_STORE_PCK 0xa
+#define SQ_IMAGE_STORE_MIP_PCK 0xb
+#define SQ_IMAGE_GET_RESINFO 0xe
+#define SQ_IMAGE_ATOMIC_SWAP 0x10
+#define SQ_IMAGE_ATOMIC_CMPSWAP 0x11
+#define SQ_IMAGE_ATOMIC_ADD 0x12
+#define SQ_IMAGE_ATOMIC_SUB 0x13
+#define SQ_IMAGE_ATOMIC_SMIN 0x14
+#define SQ_IMAGE_ATOMIC_UMIN 0x15
+#define SQ_IMAGE_ATOMIC_SMAX 0x16
+#define SQ_IMAGE_ATOMIC_UMAX 0x17
+#define SQ_IMAGE_ATOMIC_AND 0x18
+#define SQ_IMAGE_ATOMIC_OR 0x19
+#define SQ_IMAGE_ATOMIC_XOR 0x1a
+#define SQ_IMAGE_ATOMIC_INC 0x1b
+#define SQ_IMAGE_ATOMIC_DEC 0x1c
+#define SQ_IMAGE_SAMPLE 0x20
+#define SQ_IMAGE_SAMPLE_CL 0x21
+#define SQ_IMAGE_SAMPLE_D 0x22
+#define SQ_IMAGE_SAMPLE_D_CL 0x23
+#define SQ_IMAGE_SAMPLE_L 0x24
+#define SQ_IMAGE_SAMPLE_B 0x25
+#define SQ_IMAGE_SAMPLE_B_CL 0x26
+#define SQ_IMAGE_SAMPLE_LZ 0x27
+#define SQ_IMAGE_SAMPLE_C 0x28
+#define SQ_IMAGE_SAMPLE_C_CL 0x29
+#define SQ_IMAGE_SAMPLE_C_D 0x2a
+#define SQ_IMAGE_SAMPLE_C_D_CL 0x2b
+#define SQ_IMAGE_SAMPLE_C_L 0x2c
+#define SQ_IMAGE_SAMPLE_C_B 0x2d
+#define SQ_IMAGE_SAMPLE_C_B_CL 0x2e
+#define SQ_IMAGE_SAMPLE_C_LZ 0x2f
+#define SQ_IMAGE_SAMPLE_O 0x30
+#define SQ_IMAGE_SAMPLE_CL_O 0x31
+#define SQ_IMAGE_SAMPLE_D_O 0x32
+#define SQ_IMAGE_SAMPLE_D_CL_O 0x33
+#define SQ_IMAGE_SAMPLE_L_O 0x34
+#define SQ_IMAGE_SAMPLE_B_O 0x35
+#define SQ_IMAGE_SAMPLE_B_CL_O 0x36
+#define SQ_IMAGE_SAMPLE_LZ_O 0x37
+#define SQ_IMAGE_SAMPLE_C_O 0x38
+#define SQ_IMAGE_SAMPLE_C_CL_O 0x39
+#define SQ_IMAGE_SAMPLE_C_D_O 0x3a
+#define SQ_IMAGE_SAMPLE_C_D_CL_O 0x3b
+#define SQ_IMAGE_SAMPLE_C_L_O 0x3c
+#define SQ_IMAGE_SAMPLE_C_B_O 0x3d
+#define SQ_IMAGE_SAMPLE_C_B_CL_O 0x3e
+#define SQ_IMAGE_SAMPLE_C_LZ_O 0x3f
+#define SQ_IMAGE_GATHER4 0x40
+#define SQ_IMAGE_GATHER4_CL 0x41
+#define SQ_IMAGE_GATHER4_L 0x44
+#define SQ_IMAGE_GATHER4_B 0x45
+#define SQ_IMAGE_GATHER4_B_CL 0x46
+#define SQ_IMAGE_GATHER4_LZ 0x47
+#define SQ_IMAGE_GATHER4_C 0x48
+#define SQ_IMAGE_GATHER4_C_CL 0x49
+#define SQ_IMAGE_GATHER4_C_L 0x4c
+#define SQ_IMAGE_GATHER4_C_B 0x4d
+#define SQ_IMAGE_GATHER4_C_B_CL 0x4e
+#define SQ_IMAGE_GATHER4_C_LZ 0x4f
+#define SQ_IMAGE_GATHER4_O 0x50
+#define SQ_IMAGE_GATHER4_CL_O 0x51
+#define SQ_IMAGE_GATHER4_L_O 0x54
+#define SQ_IMAGE_GATHER4_B_O 0x55
+#define SQ_IMAGE_GATHER4_B_CL_O 0x56
+#define SQ_IMAGE_GATHER4_LZ_O 0x57
+#define SQ_IMAGE_GATHER4_C_O 0x58
+#define SQ_IMAGE_GATHER4_C_CL_O 0x59
+#define SQ_IMAGE_GATHER4_C_L_O 0x5c
+#define SQ_IMAGE_GATHER4_C_B_O 0x5d
+#define SQ_IMAGE_GATHER4_C_B_CL_O 0x5e
+#define SQ_IMAGE_GATHER4_C_LZ_O 0x5f
+#define SQ_IMAGE_GET_LOD 0x60
+#define SQ_IMAGE_SAMPLE_CD 0x68
+#define SQ_IMAGE_SAMPLE_CD_CL 0x69
+#define SQ_IMAGE_SAMPLE_C_CD 0x6a
+#define SQ_IMAGE_SAMPLE_C_CD_CL 0x6b
+#define SQ_IMAGE_SAMPLE_CD_O 0x6c
+#define SQ_IMAGE_SAMPLE_CD_CL_O 0x6d
+#define SQ_IMAGE_SAMPLE_C_CD_O 0x6e
+#define SQ_IMAGE_SAMPLE_C_CD_CL_O 0x6f
+#define SQ_IMAGE_RSRC256 0x7e
+#define SQ_IMAGE_SAMPLER 0x7f
+#define SQ_SRC_VCCZ 0xfb
+#define SQ_SRC_VGPR0 0x100
+#define SQ_SDWA_BYTE_0 0x0
+#define SQ_SDWA_BYTE_1 0x1
+#define SQ_SDWA_BYTE_2 0x2
+#define SQ_SDWA_BYTE_3 0x3
+#define SQ_SDWA_WORD_0 0x4
+#define SQ_SDWA_WORD_1 0x5
+#define SQ_SDWA_DWORD 0x6
+#define SQ_XNACK_MASK_LO 0x68
+#define SQ_XNACK_MASK_HI 0x69
+#define SQ_TBUFFER_LOAD_FORMAT_X 0x0
+#define SQ_TBUFFER_LOAD_FORMAT_XY 0x1
+#define SQ_TBUFFER_LOAD_FORMAT_XYZ 0x2
+#define SQ_TBUFFER_LOAD_FORMAT_XYZW 0x3
+#define SQ_TBUFFER_STORE_FORMAT_X 0x4
+#define SQ_TBUFFER_STORE_FORMAT_XY 0x5
+#define SQ_TBUFFER_STORE_FORMAT_XYZ 0x6
+#define SQ_TBUFFER_STORE_FORMAT_XYZW 0x7
+#define SQ_TBUFFER_LOAD_FORMAT_D16_X 0x8
+#define SQ_TBUFFER_LOAD_FORMAT_D16_XY 0x9
+#define SQ_TBUFFER_LOAD_FORMAT_D16_XYZ 0xa
+#define SQ_TBUFFER_LOAD_FORMAT_D16_XYZW 0xb
+#define SQ_TBUFFER_STORE_FORMAT_D16_X 0xc
+#define SQ_TBUFFER_STORE_FORMAT_D16_XY 0xd
+#define SQ_TBUFFER_STORE_FORMAT_D16_XYZ 0xe
+#define SQ_TBUFFER_STORE_FORMAT_D16_XYZW 0xf
+#define SQ_CHAN_X 0x0
+#define SQ_CHAN_Y 0x1
+#define SQ_CHAN_Z 0x2
+#define SQ_CHAN_W 0x3
+#define SQ_V_NOP 0x0
+#define SQ_V_MOV_B32 0x1
+#define SQ_V_READFIRSTLANE_B32 0x2
+#define SQ_V_CVT_I32_F64 0x3
+#define SQ_V_CVT_F64_I32 0x4
+#define SQ_V_CVT_F32_I32 0x5
+#define SQ_V_CVT_F32_U32 0x6
+#define SQ_V_CVT_U32_F32 0x7
+#define SQ_V_CVT_I32_F32 0x8
+#define SQ_V_MOV_FED_B32 0x9
+#define SQ_V_CVT_F16_F32 0xa
+#define SQ_V_CVT_F32_F16 0xb
+#define SQ_V_CVT_RPI_I32_F32 0xc
+#define SQ_V_CVT_FLR_I32_F32 0xd
+#define SQ_V_CVT_OFF_F32_I4 0xe
+#define SQ_V_CVT_F32_F64 0xf
+#define SQ_V_CVT_F64_F32 0x10
+#define SQ_V_CVT_F32_UBYTE0 0x11
+#define SQ_V_CVT_F32_UBYTE1 0x12
+#define SQ_V_CVT_F32_UBYTE2 0x13
+#define SQ_V_CVT_F32_UBYTE3 0x14
+#define SQ_V_CVT_U32_F64 0x15
+#define SQ_V_CVT_F64_U32 0x16
+#define SQ_V_TRUNC_F64 0x17
+#define SQ_V_CEIL_F64 0x18
+#define SQ_V_RNDNE_F64 0x19
+#define SQ_V_FLOOR_F64 0x1a
+#define SQ_V_FRACT_F32 0x1b
+#define SQ_V_TRUNC_F32 0x1c
+#define SQ_V_CEIL_F32 0x1d
+#define SQ_V_RNDNE_F32 0x1e
+#define SQ_V_FLOOR_F32 0x1f
+#define SQ_V_EXP_F32 0x20
+#define SQ_V_LOG_F32 0x21
+#define SQ_V_RCP_F32 0x22
+#define SQ_V_RCP_IFLAG_F32 0x23
+#define SQ_V_RSQ_F32 0x24
+#define SQ_V_RCP_F64 0x25
+#define SQ_V_RSQ_F64 0x26
+#define SQ_V_SQRT_F32 0x27
+#define SQ_V_SQRT_F64 0x28
+#define SQ_V_SIN_F32 0x29
+#define SQ_V_COS_F32 0x2a
+#define SQ_V_NOT_B32 0x2b
+#define SQ_V_BFREV_B32 0x2c
+#define SQ_V_FFBH_U32 0x2d
+#define SQ_V_FFBL_B32 0x2e
+#define SQ_V_FFBH_I32 0x2f
+#define SQ_V_FREXP_EXP_I32_F64 0x30
+#define SQ_V_FREXP_MANT_F64 0x31
+#define SQ_V_FRACT_F64 0x32
+#define SQ_V_FREXP_EXP_I32_F32 0x33
+#define SQ_V_FREXP_MANT_F32 0x34
+#define SQ_V_CLREXCP 0x35
+#define SQ_V_MOVRELD_B32 0x36
+#define SQ_V_MOVRELS_B32 0x37
+#define SQ_V_MOVRELSD_B32 0x38
+#define SQ_V_CVT_F16_U16 0x39
+#define SQ_V_CVT_F16_I16 0x3a
+#define SQ_V_CVT_U16_F16 0x3b
+#define SQ_V_CVT_I16_F16 0x3c
+#define SQ_V_RCP_F16 0x3d
+#define SQ_V_SQRT_F16 0x3e
+#define SQ_V_RSQ_F16 0x3f
+#define SQ_V_LOG_F16 0x40
+#define SQ_V_EXP_F16 0x41
+#define SQ_V_FREXP_MANT_F16 0x42
+#define SQ_V_FREXP_EXP_I16_F16 0x43
+#define SQ_V_FLOOR_F16 0x44
+#define SQ_V_CEIL_F16 0x45
+#define SQ_V_TRUNC_F16 0x46
+#define SQ_V_RNDNE_F16 0x47
+#define SQ_V_FRACT_F16 0x48
+#define SQ_V_SIN_F16 0x49
+#define SQ_V_COS_F16 0x4a
+#define SQ_V_EXP_LEGACY_F32 0x4b
+#define SQ_V_LOG_LEGACY_F32 0x4c
+#define SQ_V_CVT_NORM_I16_F16 0x4d
+#define SQ_V_CVT_NORM_U16_F16 0x4e
+#define SQ_SRC_SDWA 0xf9
+#define SQ_V_OPC_OFFSET 0x0
+#define SQ_V_OP2_OFFSET 0x100
+#define SQ_V_OP1_OFFSET 0x140
+#define SQ_V_INTRP_OFFSET 0x270
+#define SQ_V_INTERP_P1_F32 0x0
+#define SQ_V_INTERP_P2_F32 0x1
+#define SQ_V_INTERP_MOV_F32 0x2
+#define SQ_S_NOP 0x0
+#define SQ_S_ENDPGM 0x1
+#define SQ_S_BRANCH 0x2
+#define SQ_S_WAKEUP 0x3
+#define SQ_S_CBRANCH_SCC0 0x4
+#define SQ_S_CBRANCH_SCC1 0x5
+#define SQ_S_CBRANCH_VCCZ 0x6
+#define SQ_S_CBRANCH_VCCNZ 0x7
+#define SQ_S_CBRANCH_EXECZ 0x8
+#define SQ_S_CBRANCH_EXECNZ 0x9
+#define SQ_S_BARRIER 0xa
+#define SQ_S_SETKILL 0xb
+#define SQ_S_WAITCNT 0xc
+#define SQ_S_SETHALT 0xd
+#define SQ_S_SLEEP 0xe
+#define SQ_S_SETPRIO 0xf
+#define SQ_S_SENDMSG 0x10
+#define SQ_S_SENDMSGHALT 0x11
+#define SQ_S_TRAP 0x12
+#define SQ_S_ICACHE_INV 0x13
+#define SQ_S_INCPERFLEVEL 0x14
+#define SQ_S_DECPERFLEVEL 0x15
+#define SQ_S_TTRACEDATA 0x16
+#define SQ_S_CBRANCH_CDBGSYS 0x17
+#define SQ_S_CBRANCH_CDBGUSER 0x18
+#define SQ_S_CBRANCH_CDBGSYS_OR_USER 0x19
+#define SQ_S_CBRANCH_CDBGSYS_AND_USER 0x1a
+#define SQ_S_ENDPGM_SAVED 0x1b
+#define SQ_S_SET_GPR_IDX_OFF 0x1c
+#define SQ_S_SET_GPR_IDX_MODE 0x1d
+#define SQ_SRC_DPP 0xfa
+#define SQ_SRC_LITERAL 0xff
+#define SQ_VCC_LO 0x6a
+#define SQ_VCC_HI 0x6b
+#define SQ_PARAM_P10 0x0
+#define SQ_PARAM_P20 0x1
+#define SQ_PARAM_P0 0x2
+#define SQ_SRC_LDS_DIRECT 0xfe
+#define SQ_V_CNDMASK_B32 0x0
+#define SQ_V_ADD_F32 0x1
+#define SQ_V_SUB_F32 0x2
+#define SQ_V_SUBREV_F32 0x3
+#define SQ_V_MUL_LEGACY_F32 0x4
+#define SQ_V_MUL_F32 0x5
+#define SQ_V_MUL_I32_I24 0x6
+#define SQ_V_MUL_HI_I32_I24 0x7
+#define SQ_V_MUL_U32_U24 0x8
+#define SQ_V_MUL_HI_U32_U24 0x9
+#define SQ_V_MIN_F32 0xa
+#define SQ_V_MAX_F32 0xb
+#define SQ_V_MIN_I32 0xc
+#define SQ_V_MAX_I32 0xd
+#define SQ_V_MIN_U32 0xe
+#define SQ_V_MAX_U32 0xf
+#define SQ_V_LSHRREV_B32 0x10
+#define SQ_V_ASHRREV_I32 0x11
+#define SQ_V_LSHLREV_B32 0x12
+#define SQ_V_AND_B32 0x13
+#define SQ_V_OR_B32 0x14
+#define SQ_V_XOR_B32 0x15
+#define SQ_V_MAC_F32 0x16
+#define SQ_V_MADMK_F32 0x17
+#define SQ_V_MADAK_F32 0x18
+#define SQ_V_ADD_U32 0x19
+#define SQ_V_SUB_U32 0x1a
+#define SQ_V_SUBREV_U32 0x1b
+#define SQ_V_ADDC_U32 0x1c
+#define SQ_V_SUBB_U32 0x1d
+#define SQ_V_SUBBREV_U32 0x1e
+#define SQ_V_ADD_F16 0x1f
+#define SQ_V_SUB_F16 0x20
+#define SQ_V_SUBREV_F16 0x21
+#define SQ_V_MUL_F16 0x22
+#define SQ_V_MAC_F16 0x23
+#define SQ_V_MADMK_F16 0x24
+#define SQ_V_MADAK_F16 0x25
+#define SQ_V_ADD_U16 0x26
+#define SQ_V_SUB_U16 0x27
+#define SQ_V_SUBREV_U16 0x28
+#define SQ_V_MUL_LO_U16 0x29
+#define SQ_V_LSHLREV_B16 0x2a
+#define SQ_V_LSHRREV_B16 0x2b
+#define SQ_V_ASHRREV_I16 0x2c
+#define SQ_V_MAX_F16 0x2d
+#define SQ_V_MIN_F16 0x2e
+#define SQ_V_MAX_U16 0x2f
+#define SQ_V_MAX_I16 0x30
+#define SQ_V_MIN_U16 0x31
+#define SQ_V_MIN_I16 0x32
+#define SQ_V_LDEXP_F16 0x33
+#define SQ_FLAT_LOAD_UBYTE 0x10
+#define SQ_FLAT_LOAD_SBYTE 0x11
+#define SQ_FLAT_LOAD_USHORT 0x12
+#define SQ_FLAT_LOAD_SSHORT 0x13
+#define SQ_FLAT_LOAD_DWORD 0x14
+#define SQ_FLAT_LOAD_DWORDX2 0x15
+#define SQ_FLAT_LOAD_DWORDX3 0x16
+#define SQ_FLAT_LOAD_DWORDX4 0x17
+#define SQ_FLAT_STORE_BYTE 0x18
+#define SQ_FLAT_STORE_SHORT 0x1a
+#define SQ_FLAT_STORE_DWORD 0x1c
+#define SQ_FLAT_STORE_DWORDX2 0x1d
+#define SQ_FLAT_STORE_DWORDX3 0x1e
+#define SQ_FLAT_STORE_DWORDX4 0x1f
+#define SQ_FLAT_ATOMIC_SWAP 0x40
+#define SQ_FLAT_ATOMIC_CMPSWAP 0x41
+#define SQ_FLAT_ATOMIC_ADD 0x42
+#define SQ_FLAT_ATOMIC_SUB 0x43
+#define SQ_FLAT_ATOMIC_SMIN 0x44
+#define SQ_FLAT_ATOMIC_UMIN 0x45
+#define SQ_FLAT_ATOMIC_SMAX 0x46
+#define SQ_FLAT_ATOMIC_UMAX 0x47
+#define SQ_FLAT_ATOMIC_AND 0x48
+#define SQ_FLAT_ATOMIC_OR 0x49
+#define SQ_FLAT_ATOMIC_XOR 0x4a
+#define SQ_FLAT_ATOMIC_INC 0x4b
+#define SQ_FLAT_ATOMIC_DEC 0x4c
+#define SQ_FLAT_ATOMIC_SWAP_X2 0x60
+#define SQ_FLAT_ATOMIC_CMPSWAP_X2 0x61
+#define SQ_FLAT_ATOMIC_ADD_X2 0x62
+#define SQ_FLAT_ATOMIC_SUB_X2 0x63
+#define SQ_FLAT_ATOMIC_SMIN_X2 0x64
+#define SQ_FLAT_ATOMIC_UMIN_X2 0x65
+#define SQ_FLAT_ATOMIC_SMAX_X2 0x66
+#define SQ_FLAT_ATOMIC_UMAX_X2 0x67
+#define SQ_FLAT_ATOMIC_AND_X2 0x68
+#define SQ_FLAT_ATOMIC_OR_X2 0x69
+#define SQ_FLAT_ATOMIC_XOR_X2 0x6a
+#define SQ_FLAT_ATOMIC_INC_X2 0x6b
+#define SQ_FLAT_ATOMIC_DEC_X2 0x6c
+#define SQ_S_CMP_EQ_I32 0x0
+#define SQ_S_CMP_LG_I32 0x1
+#define SQ_S_CMP_GT_I32 0x2
+#define SQ_S_CMP_GE_I32 0x3
+#define SQ_S_CMP_LT_I32 0x4
+#define SQ_S_CMP_LE_I32 0x5
+#define SQ_S_CMP_EQ_U32 0x6
+#define SQ_S_CMP_LG_U32 0x7
+#define SQ_S_CMP_GT_U32 0x8
+#define SQ_S_CMP_GE_U32 0x9
+#define SQ_S_CMP_LT_U32 0xa
+#define SQ_S_CMP_LE_U32 0xb
+#define SQ_S_BITCMP0_B32 0xc
+#define SQ_S_BITCMP1_B32 0xd
+#define SQ_S_BITCMP0_B64 0xe
+#define SQ_S_BITCMP1_B64 0xf
+#define SQ_S_SETVSKIP 0x10
+#define SQ_S_SET_GPR_IDX_ON 0x11
+#define SQ_S_CMP_EQ_U64 0x12
+#define SQ_S_CMP_LG_U64 0x13
+#define SQ_M0 0x7c
+#define SQ_V_MAD_LEGACY_F32 0x1c0
+#define SQ_V_MAD_F32 0x1c1
+#define SQ_V_MAD_I32_I24 0x1c2
+#define SQ_V_MAD_U32_U24 0x1c3
+#define SQ_V_CUBEID_F32 0x1c4
+#define SQ_V_CUBESC_F32 0x1c5
+#define SQ_V_CUBETC_F32 0x1c6
+#define SQ_V_CUBEMA_F32 0x1c7
+#define SQ_V_BFE_U32 0x1c8
+#define SQ_V_BFE_I32 0x1c9
+#define SQ_V_BFI_B32 0x1ca
+#define SQ_V_FMA_F32 0x1cb
+#define SQ_V_FMA_F64 0x1cc
+#define SQ_V_LERP_U8 0x1cd
+#define SQ_V_ALIGNBIT_B32 0x1ce
+#define SQ_V_ALIGNBYTE_B32 0x1cf
+#define SQ_V_MIN3_F32 0x1d0
+#define SQ_V_MIN3_I32 0x1d1
+#define SQ_V_MIN3_U32 0x1d2
+#define SQ_V_MAX3_F32 0x1d3
+#define SQ_V_MAX3_I32 0x1d4
+#define SQ_V_MAX3_U32 0x1d5
+#define SQ_V_MED3_F32 0x1d6
+#define SQ_V_MED3_I32 0x1d7
+#define SQ_V_MED3_U32 0x1d8
+#define SQ_V_SAD_U8 0x1d9
+#define SQ_V_SAD_HI_U8 0x1da
+#define SQ_V_SAD_U16 0x1db
+#define SQ_V_SAD_U32 0x1dc
+#define SQ_V_CVT_PK_U8_F32 0x1dd
+#define SQ_V_DIV_FIXUP_F32 0x1de
+#define SQ_V_DIV_FIXUP_F64 0x1df
+#define SQ_V_DIV_SCALE_F32 0x1e0
+#define SQ_V_DIV_SCALE_F64 0x1e1
+#define SQ_V_DIV_FMAS_F32 0x1e2
+#define SQ_V_DIV_FMAS_F64 0x1e3
+#define SQ_V_MSAD_U8 0x1e4
+#define SQ_V_QSAD_PK_U16_U8 0x1e5
+#define SQ_V_MQSAD_PK_U16_U8 0x1e6
+#define SQ_V_MQSAD_U32_U8 0x1e7
+#define SQ_V_MAD_U64_U32 0x1e8
+#define SQ_V_MAD_I64_I32 0x1e9
+#define SQ_V_MAD_F16 0x1ea
+#define SQ_V_MAD_U16 0x1eb
+#define SQ_V_MAD_I16 0x1ec
+#define SQ_V_PERM_B32 0x1ed
+#define SQ_V_FMA_F16 0x1ee
+#define SQ_V_DIV_FIXUP_F16 0x1ef
+#define SQ_V_CVT_PKACCUM_U8_F32 0x1f0
+#define SQ_V_INTERP_P1LL_F16 0x274
+#define SQ_V_INTERP_P1LV_F16 0x275
+#define SQ_V_INTERP_P2_F16 0x276
+#define SQ_V_ADD_F64 0x280
+#define SQ_V_MUL_F64 0x281
+#define SQ_V_MIN_F64 0x282
+#define SQ_V_MAX_F64 0x283
+#define SQ_V_LDEXP_F64 0x284
+#define SQ_V_MUL_LO_U32 0x285
+#define SQ_V_MUL_HI_U32 0x286
+#define SQ_V_MUL_HI_I32 0x287
+#define SQ_V_LDEXP_F32 0x288
+#define SQ_V_READLANE_B32 0x289
+#define SQ_V_WRITELANE_B32 0x28a
+#define SQ_V_BCNT_U32_B32 0x28b
+#define SQ_V_MBCNT_LO_U32_B32 0x28c
+#define SQ_V_MBCNT_HI_U32_B32 0x28d
+#define SQ_V_MAC_LEGACY_F32 0x28e
+#define SQ_V_LSHLREV_B64 0x28f
+#define SQ_V_LSHRREV_B64 0x290
+#define SQ_V_ASHRREV_I64 0x291
+#define SQ_V_TRIG_PREOP_F64 0x292
+#define SQ_V_BFM_B32 0x293
+#define SQ_V_CVT_PKNORM_I16_F32 0x294
+#define SQ_V_CVT_PKNORM_U16_F32 0x295
+#define SQ_V_CVT_PKRTZ_F16_F32 0x296
+#define SQ_V_CVT_PK_U16_U32 0x297
+#define SQ_V_CVT_PK_I16_I32 0x298
+#define SQ_V_CVT_PKNORM_I16_F16 0x299
+#define SQ_V_CVT_PKNORM_U16_F16 0x29a
+#define SQ_VCC_ALL 0x0
+#define SQ_SRC_EXECZ 0xfc
+#define SQ_FLAT_SCRATCH_LO 0x66
+#define SQ_FLAT_SCRATCH_HI 0x67
+#define SQ_SYSMSG_OP_ECC_ERR_INTERRUPT 0x1
+#define SQ_SYSMSG_OP_REG_RD 0x2
+#define SQ_SYSMSG_OP_HOST_TRAP_ACK 0x3
+#define SQ_SYSMSG_OP_TTRACE_PC 0x4
+#define SQ_HW_REG_MODE 0x1
+#define SQ_HW_REG_STATUS 0x2
+#define SQ_HW_REG_TRAPSTS 0x3
+#define SQ_HW_REG_HW_ID 0x4
+#define SQ_HW_REG_GPR_ALLOC 0x5
+#define SQ_HW_REG_LDS_ALLOC 0x6
+#define SQ_HW_REG_IB_STS 0x7
+#define SQ_HW_REG_PC_LO 0x8
+#define SQ_HW_REG_PC_HI 0x9
+#define SQ_HW_REG_INST_DW0 0xa
+#define SQ_HW_REG_INST_DW1 0xb
+#define SQ_HW_REG_IB_DBG0 0xc
+#define SQ_HW_REG_IB_DBG1 0xd
+#define SQ_DPP_BOUND_OFF 0x0
+#define SQ_DPP_BOUND_ZERO 0x1
+#define SQ_R1 0x1
+#define SQ_R2 0x2
+#define SQ_R3 0x3
+#define SQ_R4 0x4
+#define SQ_R5 0x5
+#define SQ_R6 0x6
+#define SQ_R7 0x7
+#define SQ_R8 0x8
+#define SQ_R9 0x9
+#define SQ_R10 0xa
+#define SQ_R11 0xb
+#define SQ_R12 0xc
+#define SQ_R13 0xd
+#define SQ_R14 0xe
+#define SQ_R15 0xf
+#define SQ_S_ADD_U32 0x0
+#define SQ_S_SUB_U32 0x1
+#define SQ_S_ADD_I32 0x2
+#define SQ_S_SUB_I32 0x3
+#define SQ_S_ADDC_U32 0x4
+#define SQ_S_SUBB_U32 0x5
+#define SQ_S_MIN_I32 0x6
+#define SQ_S_MIN_U32 0x7
+#define SQ_S_MAX_I32 0x8
+#define SQ_S_MAX_U32 0x9
+#define SQ_S_CSELECT_B32 0xa
+#define SQ_S_CSELECT_B64 0xb
+#define SQ_S_AND_B32 0xc
+#define SQ_S_AND_B64 0xd
+#define SQ_S_OR_B32 0xe
+#define SQ_S_OR_B64 0xf
+#define SQ_S_XOR_B32 0x10
+#define SQ_S_XOR_B64 0x11
+#define SQ_S_ANDN2_B32 0x12
+#define SQ_S_ANDN2_B64 0x13
+#define SQ_S_ORN2_B32 0x14
+#define SQ_S_ORN2_B64 0x15
+#define SQ_S_NAND_B32 0x16
+#define SQ_S_NAND_B64 0x17
+#define SQ_S_NOR_B32 0x18
+#define SQ_S_NOR_B64 0x19
+#define SQ_S_XNOR_B32 0x1a
+#define SQ_S_XNOR_B64 0x1b
+#define SQ_S_LSHL_B32 0x1c
+#define SQ_S_LSHL_B64 0x1d
+#define SQ_S_LSHR_B32 0x1e
+#define SQ_S_LSHR_B64 0x1f
+#define SQ_S_ASHR_I32 0x20
+#define SQ_S_ASHR_I64 0x21
+#define SQ_S_BFM_B32 0x22
+#define SQ_S_BFM_B64 0x23
+#define SQ_S_MUL_I32 0x24
+#define SQ_S_BFE_U32 0x25
+#define SQ_S_BFE_I32 0x26
+#define SQ_S_BFE_U64 0x27
+#define SQ_S_BFE_I64 0x28
+#define SQ_S_CBRANCH_G_FORK 0x29
+#define SQ_S_ABSDIFF_I32 0x2a
+#define SQ_S_RFE_RESTORE_B64 0x2b
+#define SQ_MSG_INTERRUPT 0x1
+#define SQ_MSG_GS 0x2
+#define SQ_MSG_GS_DONE 0x3
+#define SQ_MSG_SAVEWAVE 0x4
+#define SQ_MSG_SYSMSG 0xf
+typedef enum SX_BLEND_OPT {
+ BLEND_OPT_PRESERVE_NONE_IGNORE_ALL = 0x0,
+ BLEND_OPT_PRESERVE_ALL_IGNORE_NONE = 0x1,
+ BLEND_OPT_PRESERVE_C1_IGNORE_C0 = 0x2,
+ BLEND_OPT_PRESERVE_C0_IGNORE_C1 = 0x3,
+ BLEND_OPT_PRESERVE_A1_IGNORE_A0 = 0x4,
+ BLEND_OPT_PRESERVE_A0_IGNORE_A1 = 0x5,
+ BLEND_OPT_PRESERVE_NONE_IGNORE_A0 = 0x6,
+ BLEND_OPT_PRESERVE_NONE_IGNORE_NONE = 0x7,
+} SX_BLEND_OPT;
+typedef enum SX_OPT_COMB_FCN {
+ OPT_COMB_NONE = 0x0,
+ OPT_COMB_ADD = 0x1,
+ OPT_COMB_SUBTRACT = 0x2,
+ OPT_COMB_MIN = 0x3,
+ OPT_COMB_MAX = 0x4,
+ OPT_COMB_REVSUBTRACT = 0x5,
+ OPT_COMB_BLEND_DISABLED = 0x6,
+ OPT_COMB_SAFE_ADD = 0x7,
+} SX_OPT_COMB_FCN;
+typedef enum SX_DOWNCONVERT_FORMAT {
+ SX_RT_EXPORT_NO_CONVERSION = 0x0,
+ SX_RT_EXPORT_32_R = 0x1,
+ SX_RT_EXPORT_32_A = 0x2,
+ SX_RT_EXPORT_10_11_11 = 0x3,
+ SX_RT_EXPORT_2_10_10_10 = 0x4,
+ SX_RT_EXPORT_8_8_8_8 = 0x5,
+ SX_RT_EXPORT_5_6_5 = 0x6,
+ SX_RT_EXPORT_1_5_5_5 = 0x7,
+ SX_RT_EXPORT_4_4_4_4 = 0x8,
+ SX_RT_EXPORT_16_16_GR = 0x9,
+ SX_RT_EXPORT_16_16_AR = 0xa,
+} SX_DOWNCONVERT_FORMAT;
+typedef enum TEX_BORDER_COLOR_TYPE {
+ TEX_BorderColor_TransparentBlack = 0x0,
+ TEX_BorderColor_OpaqueBlack = 0x1,
+ TEX_BorderColor_OpaqueWhite = 0x2,
+ TEX_BorderColor_Register = 0x3,
+} TEX_BORDER_COLOR_TYPE;
+typedef enum TEX_CHROMA_KEY {
+ TEX_ChromaKey_Disabled = 0x0,
+ TEX_ChromaKey_Kill = 0x1,
+ TEX_ChromaKey_Blend = 0x2,
+ TEX_ChromaKey_RESERVED_3 = 0x3,
+} TEX_CHROMA_KEY;
+typedef enum TEX_CLAMP {
+ TEX_Clamp_Repeat = 0x0,
+ TEX_Clamp_Mirror = 0x1,
+ TEX_Clamp_ClampToLast = 0x2,
+ TEX_Clamp_MirrorOnceToLast = 0x3,
+ TEX_Clamp_ClampHalfToBorder = 0x4,
+ TEX_Clamp_MirrorOnceHalfToBorder = 0x5,
+ TEX_Clamp_ClampToBorder = 0x6,
+ TEX_Clamp_MirrorOnceToBorder = 0x7,
+} TEX_CLAMP;
+typedef enum TEX_COORD_TYPE {
+ TEX_CoordType_Unnormalized = 0x0,
+ TEX_CoordType_Normalized = 0x1,
+} TEX_COORD_TYPE;
+typedef enum TEX_DEPTH_COMPARE_FUNCTION {
+ TEX_DepthCompareFunction_Never = 0x0,
+ TEX_DepthCompareFunction_Less = 0x1,
+ TEX_DepthCompareFunction_Equal = 0x2,
+ TEX_DepthCompareFunction_LessEqual = 0x3,
+ TEX_DepthCompareFunction_Greater = 0x4,
+ TEX_DepthCompareFunction_NotEqual = 0x5,
+ TEX_DepthCompareFunction_GreaterEqual = 0x6,
+ TEX_DepthCompareFunction_Always = 0x7,
+} TEX_DEPTH_COMPARE_FUNCTION;
+typedef enum TEX_DIM {
+ TEX_Dim_1D = 0x0,
+ TEX_Dim_2D = 0x1,
+ TEX_Dim_3D = 0x2,
+ TEX_Dim_CubeMap = 0x3,
+ TEX_Dim_1DArray = 0x4,
+ TEX_Dim_2DArray = 0x5,
+ TEX_Dim_2D_MSAA = 0x6,
+ TEX_Dim_2DArray_MSAA = 0x7,
+} TEX_DIM;
+typedef enum TEX_FORMAT_COMP {
+ TEX_FormatComp_Unsigned = 0x0,
+ TEX_FormatComp_Signed = 0x1,
+ TEX_FormatComp_UnsignedBiased = 0x2,
+ TEX_FormatComp_RESERVED_3 = 0x3,
+} TEX_FORMAT_COMP;
+typedef enum TEX_MAX_ANISO_RATIO {
+ TEX_MaxAnisoRatio_1to1 = 0x0,
+ TEX_MaxAnisoRatio_2to1 = 0x1,
+ TEX_MaxAnisoRatio_4to1 = 0x2,
+ TEX_MaxAnisoRatio_8to1 = 0x3,
+ TEX_MaxAnisoRatio_16to1 = 0x4,
+ TEX_MaxAnisoRatio_RESERVED_5 = 0x5,
+ TEX_MaxAnisoRatio_RESERVED_6 = 0x6,
+ TEX_MaxAnisoRatio_RESERVED_7 = 0x7,
+} TEX_MAX_ANISO_RATIO;
+typedef enum TEX_MIP_FILTER {
+ TEX_MipFilter_None = 0x0,
+ TEX_MipFilter_Point = 0x1,
+ TEX_MipFilter_Linear = 0x2,
+ TEX_MipFilter_Point_Aniso_Adj = 0x3,
+} TEX_MIP_FILTER;
+typedef enum TEX_REQUEST_SIZE {
+ TEX_RequestSize_32B = 0x0,
+ TEX_RequestSize_64B = 0x1,
+ TEX_RequestSize_128B = 0x2,
+ TEX_RequestSize_2X64B = 0x3,
+} TEX_REQUEST_SIZE;
+typedef enum TEX_SAMPLER_TYPE {
+ TEX_SamplerType_Invalid = 0x0,
+ TEX_SamplerType_Valid = 0x1,
+} TEX_SAMPLER_TYPE;
+typedef enum TEX_XY_FILTER {
+ TEX_XYFilter_Point = 0x0,
+ TEX_XYFilter_Linear = 0x1,
+ TEX_XYFilter_AnisoPoint = 0x2,
+ TEX_XYFilter_AnisoLinear = 0x3,
+} TEX_XY_FILTER;
+typedef enum TEX_Z_FILTER {
+ TEX_ZFilter_None = 0x0,
+ TEX_ZFilter_Point = 0x1,
+ TEX_ZFilter_Linear = 0x2,
+ TEX_ZFilter_RESERVED_3 = 0x3,
+} TEX_Z_FILTER;
+typedef enum VTX_CLAMP {
+ VTX_Clamp_ClampToZero = 0x0,
+ VTX_Clamp_ClampToNAN = 0x1,
+} VTX_CLAMP;
+typedef enum VTX_FETCH_TYPE {
+ VTX_FetchType_VertexData = 0x0,
+ VTX_FetchType_InstanceData = 0x1,
+ VTX_FetchType_NoIndexOffset = 0x2,
+ VTX_FetchType_RESERVED_3 = 0x3,
+} VTX_FETCH_TYPE;
+typedef enum VTX_FORMAT_COMP_ALL {
+ VTX_FormatCompAll_Unsigned = 0x0,
+ VTX_FormatCompAll_Signed = 0x1,
+} VTX_FORMAT_COMP_ALL;
+typedef enum VTX_MEM_REQUEST_SIZE {
+ VTX_MemRequestSize_32B = 0x0,
+ VTX_MemRequestSize_64B = 0x1,
+} VTX_MEM_REQUEST_SIZE;
+typedef enum TVX_DATA_FORMAT {
+ TVX_FMT_INVALID = 0x0,
+ TVX_FMT_8 = 0x1,
+ TVX_FMT_4_4 = 0x2,
+ TVX_FMT_3_3_2 = 0x3,
+ TVX_FMT_RESERVED_4 = 0x4,
+ TVX_FMT_16 = 0x5,
+ TVX_FMT_16_FLOAT = 0x6,
+ TVX_FMT_8_8 = 0x7,
+ TVX_FMT_5_6_5 = 0x8,
+ TVX_FMT_6_5_5 = 0x9,
+ TVX_FMT_1_5_5_5 = 0xa,
+ TVX_FMT_4_4_4_4 = 0xb,
+ TVX_FMT_5_5_5_1 = 0xc,
+ TVX_FMT_32 = 0xd,
+ TVX_FMT_32_FLOAT = 0xe,
+ TVX_FMT_16_16 = 0xf,
+ TVX_FMT_16_16_FLOAT = 0x10,
+ TVX_FMT_8_24 = 0x11,
+ TVX_FMT_8_24_FLOAT = 0x12,
+ TVX_FMT_24_8 = 0x13,
+ TVX_FMT_24_8_FLOAT = 0x14,
+ TVX_FMT_10_11_11 = 0x15,
+ TVX_FMT_10_11_11_FLOAT = 0x16,
+ TVX_FMT_11_11_10 = 0x17,
+ TVX_FMT_11_11_10_FLOAT = 0x18,
+ TVX_FMT_2_10_10_10 = 0x19,
+ TVX_FMT_8_8_8_8 = 0x1a,
+ TVX_FMT_10_10_10_2 = 0x1b,
+ TVX_FMT_X24_8_32_FLOAT = 0x1c,
+ TVX_FMT_32_32 = 0x1d,
+ TVX_FMT_32_32_FLOAT = 0x1e,
+ TVX_FMT_16_16_16_16 = 0x1f,
+ TVX_FMT_16_16_16_16_FLOAT = 0x20,
+ TVX_FMT_RESERVED_33 = 0x21,
+ TVX_FMT_32_32_32_32 = 0x22,
+ TVX_FMT_32_32_32_32_FLOAT = 0x23,
+ TVX_FMT_RESERVED_36 = 0x24,
+ TVX_FMT_1 = 0x25,
+ TVX_FMT_1_REVERSED = 0x26,
+ TVX_FMT_GB_GR = 0x27,
+ TVX_FMT_BG_RG = 0x28,
+ TVX_FMT_32_AS_8 = 0x29,
+ TVX_FMT_32_AS_8_8 = 0x2a,
+ TVX_FMT_5_9_9_9_SHAREDEXP = 0x2b,
+ TVX_FMT_8_8_8 = 0x2c,
+ TVX_FMT_16_16_16 = 0x2d,
+ TVX_FMT_16_16_16_FLOAT = 0x2e,
+ TVX_FMT_32_32_32 = 0x2f,
+ TVX_FMT_32_32_32_FLOAT = 0x30,
+ TVX_FMT_BC1 = 0x31,
+ TVX_FMT_BC2 = 0x32,
+ TVX_FMT_BC3 = 0x33,
+ TVX_FMT_BC4 = 0x34,
+ TVX_FMT_BC5 = 0x35,
+ TVX_FMT_APC0 = 0x36,
+ TVX_FMT_APC1 = 0x37,
+ TVX_FMT_APC2 = 0x38,
+ TVX_FMT_APC3 = 0x39,
+ TVX_FMT_APC4 = 0x3a,
+ TVX_FMT_APC5 = 0x3b,
+ TVX_FMT_APC6 = 0x3c,
+ TVX_FMT_APC7 = 0x3d,
+ TVX_FMT_CTX1 = 0x3e,
+ TVX_FMT_RESERVED_63 = 0x3f,
+} TVX_DATA_FORMAT;
+typedef enum TVX_DST_SEL {
+ TVX_DstSel_X = 0x0,
+ TVX_DstSel_Y = 0x1,
+ TVX_DstSel_Z = 0x2,
+ TVX_DstSel_W = 0x3,
+ TVX_DstSel_0f = 0x4,
+ TVX_DstSel_1f = 0x5,
+ TVX_DstSel_RESERVED_6 = 0x6,
+ TVX_DstSel_Mask = 0x7,
+} TVX_DST_SEL;
+typedef enum TVX_ENDIAN_SWAP {
+ TVX_EndianSwap_None = 0x0,
+ TVX_EndianSwap_8in16 = 0x1,
+ TVX_EndianSwap_8in32 = 0x2,
+ TVX_EndianSwap_8in64 = 0x3,
+} TVX_ENDIAN_SWAP;
+typedef enum TVX_INST {
+ TVX_Inst_NormalVertexFetch = 0x0,
+ TVX_Inst_SemanticVertexFetch = 0x1,
+ TVX_Inst_RESERVED_2 = 0x2,
+ TVX_Inst_LD = 0x3,
+ TVX_Inst_GetTextureResInfo = 0x4,
+ TVX_Inst_GetNumberOfSamples = 0x5,
+ TVX_Inst_GetLOD = 0x6,
+ TVX_Inst_GetGradientsH = 0x7,
+ TVX_Inst_GetGradientsV = 0x8,
+ TVX_Inst_SetTextureOffsets = 0x9,
+ TVX_Inst_KeepGradients = 0xa,
+ TVX_Inst_SetGradientsH = 0xb,
+ TVX_Inst_SetGradientsV = 0xc,
+ TVX_Inst_Pass = 0xd,
+ TVX_Inst_GetBufferResInfo = 0xe,
+ TVX_Inst_RESERVED_15 = 0xf,
+ TVX_Inst_Sample = 0x10,
+ TVX_Inst_Sample_L = 0x11,
+ TVX_Inst_Sample_LB = 0x12,
+ TVX_Inst_Sample_LZ = 0x13,
+ TVX_Inst_Sample_G = 0x14,
+ TVX_Inst_Gather4 = 0x15,
+ TVX_Inst_Sample_G_LB = 0x16,
+ TVX_Inst_Gather4_O = 0x17,
+ TVX_Inst_Sample_C = 0x18,
+ TVX_Inst_Sample_C_L = 0x19,
+ TVX_Inst_Sample_C_LB = 0x1a,
+ TVX_Inst_Sample_C_LZ = 0x1b,
+ TVX_Inst_Sample_C_G = 0x1c,
+ TVX_Inst_Gather4_C = 0x1d,
+ TVX_Inst_Sample_C_G_LB = 0x1e,
+ TVX_Inst_Gather4_C_O = 0x1f,
+} TVX_INST;
+typedef enum TVX_NUM_FORMAT_ALL {
+ TVX_NumFormatAll_Norm = 0x0,
+ TVX_NumFormatAll_Int = 0x1,
+ TVX_NumFormatAll_Scaled = 0x2,
+ TVX_NumFormatAll_RESERVED_3 = 0x3,
+} TVX_NUM_FORMAT_ALL;
+typedef enum TVX_SRC_SEL {
+ TVX_SrcSel_X = 0x0,
+ TVX_SrcSel_Y = 0x1,
+ TVX_SrcSel_Z = 0x2,
+ TVX_SrcSel_W = 0x3,
+ TVX_SrcSel_0f = 0x4,
+ TVX_SrcSel_1f = 0x5,
+} TVX_SRC_SEL;
+typedef enum TVX_SRF_MODE_ALL {
+ TVX_SRFModeAll_ZCMO = 0x0,
+ TVX_SRFModeAll_NZ = 0x1,
+} TVX_SRF_MODE_ALL;
+typedef enum TVX_TYPE {
+ TVX_Type_InvalidTextureResource = 0x0,
+ TVX_Type_InvalidVertexBuffer = 0x1,
+ TVX_Type_ValidTextureResource = 0x2,
+ TVX_Type_ValidVertexBuffer = 0x3,
+} TVX_TYPE;
+typedef enum TC_OP_MASKS {
+ TC_OP_MASK_FLUSH_DENROM = 0x8,
+ TC_OP_MASK_64 = 0x20,
+ TC_OP_MASK_NO_RTN = 0x40,
+} TC_OP_MASKS;
+typedef enum TC_OP {
+ TC_OP_READ = 0x0,
+ TC_OP_ATOMIC_FCMPSWAP_RTN_32 = 0x1,
+ TC_OP_ATOMIC_FMIN_RTN_32 = 0x2,
+ TC_OP_ATOMIC_FMAX_RTN_32 = 0x3,
+ TC_OP_RESERVED_FOP_RTN_32_0 = 0x4,
+ TC_OP_RESERVED_FOP_RTN_32_1 = 0x5,
+ TC_OP_RESERVED_FOP_RTN_32_2 = 0x6,
+ TC_OP_ATOMIC_SWAP_RTN_32 = 0x7,
+ TC_OP_ATOMIC_CMPSWAP_RTN_32 = 0x8,
+ TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_RTN_32 = 0x9,
+ TC_OP_ATOMIC_FMIN_FLUSH_DENORM_RTN_32 = 0xa,
+ TC_OP_ATOMIC_FMAX_FLUSH_DENORM_RTN_32 = 0xb,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_0 = 0xc,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_1 = 0xd,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_2 = 0xe,
+ TC_OP_ATOMIC_ADD_RTN_32 = 0xf,
+ TC_OP_ATOMIC_SUB_RTN_32 = 0x10,
+ TC_OP_ATOMIC_SMIN_RTN_32 = 0x11,
+ TC_OP_ATOMIC_UMIN_RTN_32 = 0x12,
+ TC_OP_ATOMIC_SMAX_RTN_32 = 0x13,
+ TC_OP_ATOMIC_UMAX_RTN_32 = 0x14,
+ TC_OP_ATOMIC_AND_RTN_32 = 0x15,
+ TC_OP_ATOMIC_OR_RTN_32 = 0x16,
+ TC_OP_ATOMIC_XOR_RTN_32 = 0x17,
+ TC_OP_ATOMIC_INC_RTN_32 = 0x18,
+ TC_OP_ATOMIC_DEC_RTN_32 = 0x19,
+ TC_OP_WBINVL1_VOL = 0x1a,
+ TC_OP_WBINVL1_SD = 0x1b,
+ TC_OP_RESERVED_NON_FLOAT_RTN_32_0 = 0x1c,
+ TC_OP_RESERVED_NON_FLOAT_RTN_32_1 = 0x1d,
+ TC_OP_RESERVED_NON_FLOAT_RTN_32_2 = 0x1e,
+ TC_OP_RESERVED_NON_FLOAT_RTN_32_3 = 0x1f,
+ TC_OP_WRITE = 0x20,
+ TC_OP_ATOMIC_FCMPSWAP_RTN_64 = 0x21,
+ TC_OP_ATOMIC_FMIN_RTN_64 = 0x22,
+ TC_OP_ATOMIC_FMAX_RTN_64 = 0x23,
+ TC_OP_RESERVED_FOP_RTN_64_0 = 0x24,
+ TC_OP_RESERVED_FOP_RTN_64_1 = 0x25,
+ TC_OP_RESERVED_FOP_RTN_64_2 = 0x26,
+ TC_OP_ATOMIC_SWAP_RTN_64 = 0x27,
+ TC_OP_ATOMIC_CMPSWAP_RTN_64 = 0x28,
+ TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_RTN_64 = 0x29,
+ TC_OP_ATOMIC_FMIN_FLUSH_DENORM_RTN_64 = 0x2a,
+ TC_OP_ATOMIC_FMAX_FLUSH_DENORM_RTN_64 = 0x2b,
+ TC_OP_WBINVL2_SD = 0x2c,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_64_0 = 0x2d,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_64_1 = 0x2e,
+ TC_OP_ATOMIC_ADD_RTN_64 = 0x2f,
+ TC_OP_ATOMIC_SUB_RTN_64 = 0x30,
+ TC_OP_ATOMIC_SMIN_RTN_64 = 0x31,
+ TC_OP_ATOMIC_UMIN_RTN_64 = 0x32,
+ TC_OP_ATOMIC_SMAX_RTN_64 = 0x33,
+ TC_OP_ATOMIC_UMAX_RTN_64 = 0x34,
+ TC_OP_ATOMIC_AND_RTN_64 = 0x35,
+ TC_OP_ATOMIC_OR_RTN_64 = 0x36,
+ TC_OP_ATOMIC_XOR_RTN_64 = 0x37,
+ TC_OP_ATOMIC_INC_RTN_64 = 0x38,
+ TC_OP_ATOMIC_DEC_RTN_64 = 0x39,
+ TC_OP_WBL2_NC = 0x3a,
+ TC_OP_RESERVED_NON_FLOAT_RTN_64_0 = 0x3b,
+ TC_OP_RESERVED_NON_FLOAT_RTN_64_1 = 0x3c,
+ TC_OP_RESERVED_NON_FLOAT_RTN_64_2 = 0x3d,
+ TC_OP_RESERVED_NON_FLOAT_RTN_64_3 = 0x3e,
+ TC_OP_RESERVED_NON_FLOAT_RTN_64_4 = 0x3f,
+ TC_OP_WBINVL1 = 0x40,
+ TC_OP_ATOMIC_FCMPSWAP_32 = 0x41,
+ TC_OP_ATOMIC_FMIN_32 = 0x42,
+ TC_OP_ATOMIC_FMAX_32 = 0x43,
+ TC_OP_RESERVED_FOP_32_0 = 0x44,
+ TC_OP_RESERVED_FOP_32_1 = 0x45,
+ TC_OP_RESERVED_FOP_32_2 = 0x46,
+ TC_OP_ATOMIC_SWAP_32 = 0x47,
+ TC_OP_ATOMIC_CMPSWAP_32 = 0x48,
+ TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_32 = 0x49,
+ TC_OP_ATOMIC_FMIN_FLUSH_DENORM_32 = 0x4a,
+ TC_OP_ATOMIC_FMAX_FLUSH_DENORM_32 = 0x4b,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_32_0 = 0x4c,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_32_1 = 0x4d,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_32_2 = 0x4e,
+ TC_OP_ATOMIC_ADD_32 = 0x4f,
+ TC_OP_ATOMIC_SUB_32 = 0x50,
+ TC_OP_ATOMIC_SMIN_32 = 0x51,
+ TC_OP_ATOMIC_UMIN_32 = 0x52,
+ TC_OP_ATOMIC_SMAX_32 = 0x53,
+ TC_OP_ATOMIC_UMAX_32 = 0x54,
+ TC_OP_ATOMIC_AND_32 = 0x55,
+ TC_OP_ATOMIC_OR_32 = 0x56,
+ TC_OP_ATOMIC_XOR_32 = 0x57,
+ TC_OP_ATOMIC_INC_32 = 0x58,
+ TC_OP_ATOMIC_DEC_32 = 0x59,
+ TC_OP_INVL2_NC = 0x5a,
+ TC_OP_RESERVED_NON_FLOAT_32_0 = 0x5b,
+ TC_OP_RESERVED_NON_FLOAT_32_1 = 0x5c,
+ TC_OP_RESERVED_NON_FLOAT_32_2 = 0x5d,
+ TC_OP_RESERVED_NON_FLOAT_32_3 = 0x5e,
+ TC_OP_RESERVED_NON_FLOAT_32_4 = 0x5f,
+ TC_OP_WBINVL2 = 0x60,
+ TC_OP_ATOMIC_FCMPSWAP_64 = 0x61,
+ TC_OP_ATOMIC_FMIN_64 = 0x62,
+ TC_OP_ATOMIC_FMAX_64 = 0x63,
+ TC_OP_RESERVED_FOP_64_0 = 0x64,
+ TC_OP_RESERVED_FOP_64_1 = 0x65,
+ TC_OP_RESERVED_FOP_64_2 = 0x66,
+ TC_OP_ATOMIC_SWAP_64 = 0x67,
+ TC_OP_ATOMIC_CMPSWAP_64 = 0x68,
+ TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_64 = 0x69,
+ TC_OP_ATOMIC_FMIN_FLUSH_DENORM_64 = 0x6a,
+ TC_OP_ATOMIC_FMAX_FLUSH_DENORM_64 = 0x6b,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_64_0 = 0x6c,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_64_1 = 0x6d,
+ TC_OP_RESERVED_FOP_FLUSH_DENORM_64_2 = 0x6e,
+ TC_OP_ATOMIC_ADD_64 = 0x6f,
+ TC_OP_ATOMIC_SUB_64 = 0x70,
+ TC_OP_ATOMIC_SMIN_64 = 0x71,
+ TC_OP_ATOMIC_UMIN_64 = 0x72,
+ TC_OP_ATOMIC_SMAX_64 = 0x73,
+ TC_OP_ATOMIC_UMAX_64 = 0x74,
+ TC_OP_ATOMIC_AND_64 = 0x75,
+ TC_OP_ATOMIC_OR_64 = 0x76,
+ TC_OP_ATOMIC_XOR_64 = 0x77,
+ TC_OP_ATOMIC_INC_64 = 0x78,
+ TC_OP_ATOMIC_DEC_64 = 0x79,
+ TC_OP_WBINVL2_NC = 0x7a,
+ TC_OP_RESERVED_NON_FLOAT_64_0 = 0x7b,
+ TC_OP_RESERVED_NON_FLOAT_64_1 = 0x7c,
+ TC_OP_RESERVED_NON_FLOAT_64_2 = 0x7d,
+ TC_OP_RESERVED_NON_FLOAT_64_3 = 0x7e,
+ TC_OP_RESERVED_NON_FLOAT_64_4 = 0x7f,
+} TC_OP;
+typedef enum TC_CHUB_REQ_CREDITS_ENUM {
+ TC_CHUB_REQ_CREDITS = 0x10,
+} TC_CHUB_REQ_CREDITS_ENUM;
+typedef enum CHUB_TC_RET_CREDITS_ENUM {
+ CHUB_TC_RET_CREDITS = 0x20,
+} CHUB_TC_RET_CREDITS_ENUM;
+typedef enum TC_NACKS {
+ TC_NACK_NO_FAULT = 0x0,
+ TC_NACK_PAGE_FAULT = 0x1,
+ TC_NACK_PROTECTION_FAULT = 0x2,
+ TC_NACK_DATA_ERROR = 0x3,
+} TC_NACKS;
+typedef enum TCC_PERF_SEL {
+ TCC_PERF_SEL_NONE = 0x0,
+ TCC_PERF_SEL_CYCLE = 0x1,
+ TCC_PERF_SEL_BUSY = 0x2,
+ TCC_PERF_SEL_REQ = 0x3,
+ TCC_PERF_SEL_STREAMING_REQ = 0x4,
+ TCC_PERF_SEL_EXE_REQ = 0x5,
+ TCC_PERF_SEL_COMPRESSED_REQ = 0x6,
+ TCC_PERF_SEL_COMPRESSED_0_REQ = 0x7,
+ TCC_PERF_SEL_METADATA_REQ = 0x8,
+ TCC_PERF_SEL_NC_VIRTUAL_REQ = 0x9,
+ TCC_PERF_SEL_NC_PHYSICAL_REQ = 0xa,
+ TCC_PERF_SEL_UC_VIRTUAL_REQ = 0xb,
+ TCC_PERF_SEL_UC_PHYSICAL_REQ = 0xc,
+ TCC_PERF_SEL_CC_PHYSICAL_REQ = 0xd,
+ TCC_PERF_SEL_PROBE = 0xe,
+ TCC_PERF_SEL_READ = 0xf,
+ TCC_PERF_SEL_WRITE = 0x10,
+ TCC_PERF_SEL_ATOMIC = 0x11,
+ TCC_PERF_SEL_HIT = 0x12,
+ TCC_PERF_SEL_MISS = 0x13,
+ TCC_PERF_SEL_DEWRITE_ALLOCATE_HIT = 0x14,
+ TCC_PERF_SEL_FULLY_WRITTEN_HIT = 0x15,
+ TCC_PERF_SEL_WRITEBACK = 0x16,
+ TCC_PERF_SEL_LATENCY_FIFO_FULL = 0x17,
+ TCC_PERF_SEL_SRC_FIFO_FULL = 0x18,
+ TCC_PERF_SEL_HOLE_FIFO_FULL = 0x19,
+ TCC_PERF_SEL_MC_WRREQ = 0x1a,
+ TCC_PERF_SEL_MC_WRREQ_UNCACHED = 0x1b,
+ TCC_PERF_SEL_MC_WRREQ_STALL = 0x1c,
+ TCC_PERF_SEL_MC_WRREQ_CREDIT_STALL = 0x1d,
+ TCC_PERF_SEL_MC_WRREQ_MC_HALT_STALL = 0x1e,
+ TCC_PERF_SEL_TOO_MANY_MC_WRREQS_STALL = 0x1f,
+ TCC_PERF_SEL_MC_WRREQ_LEVEL = 0x20,
+ TCC_PERF_SEL_MC_ATOMIC = 0x21,
+ TCC_PERF_SEL_MC_ATOMIC_LEVEL = 0x22,
+ TCC_PERF_SEL_MC_RDREQ = 0x23,
+ TCC_PERF_SEL_MC_RDREQ_UNCACHED = 0x24,
+ TCC_PERF_SEL_MC_RDREQ_MDC = 0x25,
+ TCC_PERF_SEL_MC_RDREQ_COMPRESSED = 0x26,
+ TCC_PERF_SEL_MC_RDREQ_CREDIT_STALL = 0x27,
+ TCC_PERF_SEL_MC_RDREQ_MC_HALT_STALL = 0x28,
+ TCC_PERF_SEL_MC_RDREQ_LEVEL = 0x29,
+ TCC_PERF_SEL_TAG_STALL = 0x2a,
+ TCC_PERF_SEL_TAG_WRITEBACK_FIFO_FULL_STALL = 0x2b,
+ TCC_PERF_SEL_TAG_MISS_NOTHING_REPLACEABLE_STALL = 0x2c,
+ TCC_PERF_SEL_TAG_UNCACHED_WRITE_ATOMIC_FIFO_FULL_STALL= 0x2d,
+ TCC_PERF_SEL_TAG_NO_UNCACHED_WRITE_ATOMIC_ENTRIES_STALL= 0x2e,
+ TCC_PERF_SEL_TAG_PROBE_STALL = 0x2f,
+ TCC_PERF_SEL_TAG_PROBE_FILTER_STALL = 0x30,
+ TCC_PERF_SEL_READ_RETURN_TIMEOUT = 0x31,
+ TCC_PERF_SEL_WRITEBACK_READ_TIMEOUT = 0x32,
+ TCC_PERF_SEL_READ_RETURN_FULL_BUBBLE = 0x33,
+ TCC_PERF_SEL_BUBBLE = 0x34,
+ TCC_PERF_SEL_RETURN_ACK = 0x35,
+ TCC_PERF_SEL_RETURN_DATA = 0x36,
+ TCC_PERF_SEL_RETURN_HOLE = 0x37,
+ TCC_PERF_SEL_RETURN_ACK_HOLE = 0x38,
+ TCC_PERF_SEL_IB_REQ = 0x39,
+ TCC_PERF_SEL_IB_STALL = 0x3a,
+ TCC_PERF_SEL_IB_TAG_STALL = 0x3b,
+ TCC_PERF_SEL_IB_MDC_STALL = 0x3c,
+ TCC_PERF_SEL_TCA_LEVEL = 0x3d,
+ TCC_PERF_SEL_HOLE_LEVEL = 0x3e,
+ TCC_PERF_SEL_MC_RDRET_NACK = 0x3f,
+ TCC_PERF_SEL_MC_WRRET_NACK = 0x40,
+ TCC_PERF_SEL_NORMAL_WRITEBACK = 0x41,
+ TCC_PERF_SEL_TC_OP_WBL2_NC_WRITEBACK = 0x42,
+ TCC_PERF_SEL_TC_OP_WBINVL2_WRITEBACK = 0x43,
+ TCC_PERF_SEL_TC_OP_WBINVL2_NC_WRITEBACK = 0x44,
+ TCC_PERF_SEL_TC_OP_WBINVL2_SD_WRITEBACK = 0x45,
+ TCC_PERF_SEL_ALL_TC_OP_WB_WRITEBACK = 0x46,
+ TCC_PERF_SEL_NORMAL_EVICT = 0x47,
+ TCC_PERF_SEL_TC_OP_WBL2_NC_EVICT = 0x48,
+ TCC_PERF_SEL_TC_OP_INVL2_NC_EVICT = 0x49,
+ TCC_PERF_SEL_TC_OP_WBINVL2_EVICT = 0x4a,
+ TCC_PERF_SEL_TC_OP_WBINVL2_NC_EVICT = 0x4b,
+ TCC_PERF_SEL_TC_OP_WBINVL2_SD_EVICT = 0x4c,
+ TCC_PERF_SEL_ALL_TC_OP_INV_EVICT = 0x4d,
+ TCC_PERF_SEL_PROBE_EVICT = 0x4e,
+ TCC_PERF_SEL_TC_OP_WBL2_NC_CYCLE = 0x4f,
+ TCC_PERF_SEL_TC_OP_INVL2_NC_CYCLE = 0x50,
+ TCC_PERF_SEL_TC_OP_WBINVL2_CYCLE = 0x51,
+ TCC_PERF_SEL_TC_OP_WBINVL2_NC_CYCLE = 0x52,
+ TCC_PERF_SEL_TC_OP_WBINVL2_SD_CYCLE = 0x53,
+ TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_CYCLE = 0x54,
+ TCC_PERF_SEL_TC_OP_WBL2_NC_START = 0x55,
+ TCC_PERF_SEL_TC_OP_INVL2_NC_START = 0x56,
+ TCC_PERF_SEL_TC_OP_WBINVL2_START = 0x57,
+ TCC_PERF_SEL_TC_OP_WBINVL2_NC_START = 0x58,
+ TCC_PERF_SEL_TC_OP_WBINVL2_SD_START = 0x59,
+ TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_START = 0x5a,
+ TCC_PERF_SEL_TC_OP_WBL2_NC_FINISH = 0x5b,
+ TCC_PERF_SEL_TC_OP_INVL2_NC_FINISH = 0x5c,
+ TCC_PERF_SEL_TC_OP_WBINVL2_FINISH = 0x5d,
+ TCC_PERF_SEL_TC_OP_WBINVL2_NC_FINISH = 0x5e,
+ TCC_PERF_SEL_TC_OP_WBINVL2_SD_FINISH = 0x5f,
+ TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_FINISH = 0x60,
+ TCC_PERF_SEL_MDC_REQ = 0x61,
+ TCC_PERF_SEL_MDC_LEVEL = 0x62,
+ TCC_PERF_SEL_MDC_TAG_HIT = 0x63,
+ TCC_PERF_SEL_MDC_SECTOR_HIT = 0x64,
+ TCC_PERF_SEL_MDC_SECTOR_MISS = 0x65,
+ TCC_PERF_SEL_MDC_TAG_STALL = 0x66,
+ TCC_PERF_SEL_MDC_TAG_REPLACEMENT_LINE_IN_USE_STALL= 0x67,
+ TCC_PERF_SEL_MDC_TAG_DESECTORIZATION_FIFO_FULL_STALL= 0x68,
+ TCC_PERF_SEL_MDC_TAG_WAITING_FOR_INVALIDATE_COMPLETION_STALL= 0x69,
+ TCC_PERF_SEL_PROBE_FILTER_DISABLE_TRANSITION = 0x6a,
+ TCC_PERF_SEL_PROBE_FILTER_DISABLED = 0x6b,
+ TCC_PERF_SEL_CLIENT0_REQ = 0x80,
+ TCC_PERF_SEL_CLIENT1_REQ = 0x81,
+ TCC_PERF_SEL_CLIENT2_REQ = 0x82,
+ TCC_PERF_SEL_CLIENT3_REQ = 0x83,
+ TCC_PERF_SEL_CLIENT4_REQ = 0x84,
+ TCC_PERF_SEL_CLIENT5_REQ = 0x85,
+ TCC_PERF_SEL_CLIENT6_REQ = 0x86,
+ TCC_PERF_SEL_CLIENT7_REQ = 0x87,
+ TCC_PERF_SEL_CLIENT8_REQ = 0x88,
+ TCC_PERF_SEL_CLIENT9_REQ = 0x89,
+ TCC_PERF_SEL_CLIENT10_REQ = 0x8a,
+ TCC_PERF_SEL_CLIENT11_REQ = 0x8b,
+ TCC_PERF_SEL_CLIENT12_REQ = 0x8c,
+ TCC_PERF_SEL_CLIENT13_REQ = 0x8d,
+ TCC_PERF_SEL_CLIENT14_REQ = 0x8e,
+ TCC_PERF_SEL_CLIENT15_REQ = 0x8f,
+ TCC_PERF_SEL_CLIENT16_REQ = 0x90,
+ TCC_PERF_SEL_CLIENT17_REQ = 0x91,
+ TCC_PERF_SEL_CLIENT18_REQ = 0x92,
+ TCC_PERF_SEL_CLIENT19_REQ = 0x93,
+ TCC_PERF_SEL_CLIENT20_REQ = 0x94,
+ TCC_PERF_SEL_CLIENT21_REQ = 0x95,
+ TCC_PERF_SEL_CLIENT22_REQ = 0x96,
+ TCC_PERF_SEL_CLIENT23_REQ = 0x97,
+ TCC_PERF_SEL_CLIENT24_REQ = 0x98,
+ TCC_PERF_SEL_CLIENT25_REQ = 0x99,
+ TCC_PERF_SEL_CLIENT26_REQ = 0x9a,
+ TCC_PERF_SEL_CLIENT27_REQ = 0x9b,
+ TCC_PERF_SEL_CLIENT28_REQ = 0x9c,
+ TCC_PERF_SEL_CLIENT29_REQ = 0x9d,
+ TCC_PERF_SEL_CLIENT30_REQ = 0x9e,
+ TCC_PERF_SEL_CLIENT31_REQ = 0x9f,
+ TCC_PERF_SEL_CLIENT32_REQ = 0xa0,
+ TCC_PERF_SEL_CLIENT33_REQ = 0xa1,
+ TCC_PERF_SEL_CLIENT34_REQ = 0xa2,
+ TCC_PERF_SEL_CLIENT35_REQ = 0xa3,
+ TCC_PERF_SEL_CLIENT36_REQ = 0xa4,
+ TCC_PERF_SEL_CLIENT37_REQ = 0xa5,
+ TCC_PERF_SEL_CLIENT38_REQ = 0xa6,
+ TCC_PERF_SEL_CLIENT39_REQ = 0xa7,
+ TCC_PERF_SEL_CLIENT40_REQ = 0xa8,
+ TCC_PERF_SEL_CLIENT41_REQ = 0xa9,
+ TCC_PERF_SEL_CLIENT42_REQ = 0xaa,
+ TCC_PERF_SEL_CLIENT43_REQ = 0xab,
+ TCC_PERF_SEL_CLIENT44_REQ = 0xac,
+ TCC_PERF_SEL_CLIENT45_REQ = 0xad,
+ TCC_PERF_SEL_CLIENT46_REQ = 0xae,
+ TCC_PERF_SEL_CLIENT47_REQ = 0xaf,
+ TCC_PERF_SEL_CLIENT48_REQ = 0xb0,
+ TCC_PERF_SEL_CLIENT49_REQ = 0xb1,
+ TCC_PERF_SEL_CLIENT50_REQ = 0xb2,
+ TCC_PERF_SEL_CLIENT51_REQ = 0xb3,
+ TCC_PERF_SEL_CLIENT52_REQ = 0xb4,
+ TCC_PERF_SEL_CLIENT53_REQ = 0xb5,
+ TCC_PERF_SEL_CLIENT54_REQ = 0xb6,
+ TCC_PERF_SEL_CLIENT55_REQ = 0xb7,
+ TCC_PERF_SEL_CLIENT56_REQ = 0xb8,
+ TCC_PERF_SEL_CLIENT57_REQ = 0xb9,
+ TCC_PERF_SEL_CLIENT58_REQ = 0xba,
+ TCC_PERF_SEL_CLIENT59_REQ = 0xbb,
+ TCC_PERF_SEL_CLIENT60_REQ = 0xbc,
+ TCC_PERF_SEL_CLIENT61_REQ = 0xbd,
+ TCC_PERF_SEL_CLIENT62_REQ = 0xbe,
+ TCC_PERF_SEL_CLIENT63_REQ = 0xbf,
+ TCC_PERF_SEL_CLIENT64_REQ = 0xc0,
+ TCC_PERF_SEL_CLIENT65_REQ = 0xc1,
+ TCC_PERF_SEL_CLIENT66_REQ = 0xc2,
+ TCC_PERF_SEL_CLIENT67_REQ = 0xc3,
+ TCC_PERF_SEL_CLIENT68_REQ = 0xc4,
+ TCC_PERF_SEL_CLIENT69_REQ = 0xc5,
+ TCC_PERF_SEL_CLIENT70_REQ = 0xc6,
+ TCC_PERF_SEL_CLIENT71_REQ = 0xc7,
+ TCC_PERF_SEL_CLIENT72_REQ = 0xc8,
+ TCC_PERF_SEL_CLIENT73_REQ = 0xc9,
+ TCC_PERF_SEL_CLIENT74_REQ = 0xca,
+ TCC_PERF_SEL_CLIENT75_REQ = 0xcb,
+ TCC_PERF_SEL_CLIENT76_REQ = 0xcc,
+ TCC_PERF_SEL_CLIENT77_REQ = 0xcd,
+ TCC_PERF_SEL_CLIENT78_REQ = 0xce,
+ TCC_PERF_SEL_CLIENT79_REQ = 0xcf,
+ TCC_PERF_SEL_CLIENT80_REQ = 0xd0,
+ TCC_PERF_SEL_CLIENT81_REQ = 0xd1,
+ TCC_PERF_SEL_CLIENT82_REQ = 0xd2,
+ TCC_PERF_SEL_CLIENT83_REQ = 0xd3,
+ TCC_PERF_SEL_CLIENT84_REQ = 0xd4,
+ TCC_PERF_SEL_CLIENT85_REQ = 0xd5,
+ TCC_PERF_SEL_CLIENT86_REQ = 0xd6,
+ TCC_PERF_SEL_CLIENT87_REQ = 0xd7,
+ TCC_PERF_SEL_CLIENT88_REQ = 0xd8,
+ TCC_PERF_SEL_CLIENT89_REQ = 0xd9,
+ TCC_PERF_SEL_CLIENT90_REQ = 0xda,
+ TCC_PERF_SEL_CLIENT91_REQ = 0xdb,
+ TCC_PERF_SEL_CLIENT92_REQ = 0xdc,
+ TCC_PERF_SEL_CLIENT93_REQ = 0xdd,
+ TCC_PERF_SEL_CLIENT94_REQ = 0xde,
+ TCC_PERF_SEL_CLIENT95_REQ = 0xdf,
+ TCC_PERF_SEL_CLIENT96_REQ = 0xe0,
+ TCC_PERF_SEL_CLIENT97_REQ = 0xe1,
+ TCC_PERF_SEL_CLIENT98_REQ = 0xe2,
+ TCC_PERF_SEL_CLIENT99_REQ = 0xe3,
+ TCC_PERF_SEL_CLIENT100_REQ = 0xe4,
+ TCC_PERF_SEL_CLIENT101_REQ = 0xe5,
+ TCC_PERF_SEL_CLIENT102_REQ = 0xe6,
+ TCC_PERF_SEL_CLIENT103_REQ = 0xe7,
+ TCC_PERF_SEL_CLIENT104_REQ = 0xe8,
+ TCC_PERF_SEL_CLIENT105_REQ = 0xe9,
+ TCC_PERF_SEL_CLIENT106_REQ = 0xea,
+ TCC_PERF_SEL_CLIENT107_REQ = 0xeb,
+ TCC_PERF_SEL_CLIENT108_REQ = 0xec,
+ TCC_PERF_SEL_CLIENT109_REQ = 0xed,
+ TCC_PERF_SEL_CLIENT110_REQ = 0xee,
+ TCC_PERF_SEL_CLIENT111_REQ = 0xef,
+ TCC_PERF_SEL_CLIENT112_REQ = 0xf0,
+ TCC_PERF_SEL_CLIENT113_REQ = 0xf1,
+ TCC_PERF_SEL_CLIENT114_REQ = 0xf2,
+ TCC_PERF_SEL_CLIENT115_REQ = 0xf3,
+ TCC_PERF_SEL_CLIENT116_REQ = 0xf4,
+ TCC_PERF_SEL_CLIENT117_REQ = 0xf5,
+ TCC_PERF_SEL_CLIENT118_REQ = 0xf6,
+ TCC_PERF_SEL_CLIENT119_REQ = 0xf7,
+ TCC_PERF_SEL_CLIENT120_REQ = 0xf8,
+ TCC_PERF_SEL_CLIENT121_REQ = 0xf9,
+ TCC_PERF_SEL_CLIENT122_REQ = 0xfa,
+ TCC_PERF_SEL_CLIENT123_REQ = 0xfb,
+ TCC_PERF_SEL_CLIENT124_REQ = 0xfc,
+ TCC_PERF_SEL_CLIENT125_REQ = 0xfd,
+ TCC_PERF_SEL_CLIENT126_REQ = 0xfe,
+ TCC_PERF_SEL_CLIENT127_REQ = 0xff,
+} TCC_PERF_SEL;
+typedef enum TCA_PERF_SEL {
+ TCA_PERF_SEL_NONE = 0x0,
+ TCA_PERF_SEL_CYCLE = 0x1,
+ TCA_PERF_SEL_BUSY = 0x2,
+ TCA_PERF_SEL_FORCED_HOLE_TCC0 = 0x3,
+ TCA_PERF_SEL_FORCED_HOLE_TCC1 = 0x4,
+ TCA_PERF_SEL_FORCED_HOLE_TCC2 = 0x5,
+ TCA_PERF_SEL_FORCED_HOLE_TCC3 = 0x6,
+ TCA_PERF_SEL_FORCED_HOLE_TCC4 = 0x7,
+ TCA_PERF_SEL_FORCED_HOLE_TCC5 = 0x8,
+ TCA_PERF_SEL_FORCED_HOLE_TCC6 = 0x9,
+ TCA_PERF_SEL_FORCED_HOLE_TCC7 = 0xa,
+ TCA_PERF_SEL_REQ_TCC0 = 0xb,
+ TCA_PERF_SEL_REQ_TCC1 = 0xc,
+ TCA_PERF_SEL_REQ_TCC2 = 0xd,
+ TCA_PERF_SEL_REQ_TCC3 = 0xe,
+ TCA_PERF_SEL_REQ_TCC4 = 0xf,
+ TCA_PERF_SEL_REQ_TCC5 = 0x10,
+ TCA_PERF_SEL_REQ_TCC6 = 0x11,
+ TCA_PERF_SEL_REQ_TCC7 = 0x12,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC0 = 0x13,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC1 = 0x14,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC2 = 0x15,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC3 = 0x16,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC4 = 0x17,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC5 = 0x18,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC6 = 0x19,
+ TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC7 = 0x1a,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC0 = 0x1b,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC1 = 0x1c,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC2 = 0x1d,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC3 = 0x1e,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC4 = 0x1f,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC5 = 0x20,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC6 = 0x21,
+ TCA_PERF_SEL_CROSSBAR_STALL_TCC7 = 0x22,
+} TCA_PERF_SEL;
+typedef enum TA_TC_ADDR_MODES {
+ TA_TC_ADDR_MODE_DEFAULT = 0x0,
+ TA_TC_ADDR_MODE_COMP0 = 0x1,
+ TA_TC_ADDR_MODE_COMP1 = 0x2,
+ TA_TC_ADDR_MODE_COMP2 = 0x3,
+ TA_TC_ADDR_MODE_COMP3 = 0x4,
+ TA_TC_ADDR_MODE_UNALIGNED = 0x5,
+ TA_TC_ADDR_MODE_BORDER_COLOR = 0x6,
+} TA_TC_ADDR_MODES;
+typedef enum TA_PERFCOUNT_SEL {
+ TA_PERF_SEL_NULL = 0x0,
+ TA_PERF_SEL_sh_fifo_busy = 0x1,
+ TA_PERF_SEL_sh_fifo_cmd_busy = 0x2,
+ TA_PERF_SEL_sh_fifo_addr_busy = 0x3,
+ TA_PERF_SEL_sh_fifo_data_busy = 0x4,
+ TA_PERF_SEL_sh_fifo_data_sfifo_busy = 0x5,
+ TA_PERF_SEL_sh_fifo_data_tfifo_busy = 0x6,
+ TA_PERF_SEL_gradient_busy = 0x7,
+ TA_PERF_SEL_gradient_fifo_busy = 0x8,
+ TA_PERF_SEL_lod_busy = 0x9,
+ TA_PERF_SEL_lod_fifo_busy = 0xa,
+ TA_PERF_SEL_addresser_busy = 0xb,
+ TA_PERF_SEL_addresser_fifo_busy = 0xc,
+ TA_PERF_SEL_aligner_busy = 0xd,
+ TA_PERF_SEL_write_path_busy = 0xe,
+ TA_PERF_SEL_ta_busy = 0xf,
+ TA_PERF_SEL_sq_ta_cmd_cycles = 0x10,
+ TA_PERF_SEL_sp_ta_addr_cycles = 0x11,
+ TA_PERF_SEL_sp_ta_data_cycles = 0x12,
+ TA_PERF_SEL_ta_fa_data_state_cycles = 0x13,
+ TA_PERF_SEL_sh_fifo_addr_waiting_on_cmd_cycles = 0x14,
+ TA_PERF_SEL_sh_fifo_cmd_waiting_on_addr_cycles = 0x15,
+ TA_PERF_SEL_sh_fifo_addr_starved_while_busy_cycles= 0x16,
+ TA_PERF_SEL_sh_fifo_cmd_starved_while_busy_cycles= 0x17,
+ TA_PERF_SEL_sh_fifo_data_waiting_on_data_state_cycles= 0x18,
+ TA_PERF_SEL_sh_fifo_data_state_waiting_on_data_cycles= 0x19,
+ TA_PERF_SEL_sh_fifo_data_starved_while_busy_cycles= 0x1a,
+ TA_PERF_SEL_sh_fifo_data_state_starved_while_busy_cycles= 0x1b,
+ TA_PERF_SEL_RESERVED_28 = 0x1c,
+ TA_PERF_SEL_RESERVED_29 = 0x1d,
+ TA_PERF_SEL_sh_fifo_addr_cycles = 0x1e,
+ TA_PERF_SEL_sh_fifo_data_cycles = 0x1f,
+ TA_PERF_SEL_total_wavefronts = 0x20,
+ TA_PERF_SEL_gradient_cycles = 0x21,
+ TA_PERF_SEL_walker_cycles = 0x22,
+ TA_PERF_SEL_aligner_cycles = 0x23,
+ TA_PERF_SEL_image_wavefronts = 0x24,
+ TA_PERF_SEL_image_read_wavefronts = 0x25,
+ TA_PERF_SEL_image_write_wavefronts = 0x26,
+ TA_PERF_SEL_image_atomic_wavefronts = 0x27,
+ TA_PERF_SEL_image_total_cycles = 0x28,
+ TA_PERF_SEL_RESERVED_41 = 0x29,
+ TA_PERF_SEL_RESERVED_42 = 0x2a,
+ TA_PERF_SEL_RESERVED_43 = 0x2b,
+ TA_PERF_SEL_buffer_wavefronts = 0x2c,
+ TA_PERF_SEL_buffer_read_wavefronts = 0x2d,
+ TA_PERF_SEL_buffer_write_wavefronts = 0x2e,
+ TA_PERF_SEL_buffer_atomic_wavefronts = 0x2f,
+ TA_PERF_SEL_buffer_coalescable_wavefronts = 0x30,
+ TA_PERF_SEL_buffer_total_cycles = 0x31,
+ TA_PERF_SEL_buffer_coalescable_addr_multicycled_cycles= 0x32,
+ TA_PERF_SEL_buffer_coalescable_clamp_16kdword_multicycled_cycles= 0x33,
+ TA_PERF_SEL_buffer_coalesced_read_cycles = 0x34,
+ TA_PERF_SEL_buffer_coalesced_write_cycles = 0x35,
+ TA_PERF_SEL_addr_stalled_by_tc_cycles = 0x36,
+ TA_PERF_SEL_addr_stalled_by_td_cycles = 0x37,
+ TA_PERF_SEL_data_stalled_by_tc_cycles = 0x38,
+ TA_PERF_SEL_addresser_stalled_by_aligner_only_cycles= 0x39,
+ TA_PERF_SEL_addresser_stalled_cycles = 0x3a,
+ TA_PERF_SEL_aniso_stalled_by_addresser_only_cycles= 0x3b,
+ TA_PERF_SEL_aniso_stalled_cycles = 0x3c,
+ TA_PERF_SEL_deriv_stalled_by_aniso_only_cycles = 0x3d,
+ TA_PERF_SEL_deriv_stalled_cycles = 0x3e,
+ TA_PERF_SEL_aniso_gt1_cycle_quads = 0x3f,
+ TA_PERF_SEL_color_1_cycle_pixels = 0x40,
+ TA_PERF_SEL_color_2_cycle_pixels = 0x41,
+ TA_PERF_SEL_color_3_cycle_pixels = 0x42,
+ TA_PERF_SEL_color_4_cycle_pixels = 0x43,
+ TA_PERF_SEL_mip_1_cycle_pixels = 0x44,
+ TA_PERF_SEL_mip_2_cycle_pixels = 0x45,
+ TA_PERF_SEL_vol_1_cycle_pixels = 0x46,
+ TA_PERF_SEL_vol_2_cycle_pixels = 0x47,
+ TA_PERF_SEL_bilin_point_1_cycle_pixels = 0x48,
+ TA_PERF_SEL_mipmap_lod_0_samples = 0x49,
+ TA_PERF_SEL_mipmap_lod_1_samples = 0x4a,
+ TA_PERF_SEL_mipmap_lod_2_samples = 0x4b,
+ TA_PERF_SEL_mipmap_lod_3_samples = 0x4c,
+ TA_PERF_SEL_mipmap_lod_4_samples = 0x4d,
+ TA_PERF_SEL_mipmap_lod_5_samples = 0x4e,
+ TA_PERF_SEL_mipmap_lod_6_samples = 0x4f,
+ TA_PERF_SEL_mipmap_lod_7_samples = 0x50,
+ TA_PERF_SEL_mipmap_lod_8_samples = 0x51,
+ TA_PERF_SEL_mipmap_lod_9_samples = 0x52,
+ TA_PERF_SEL_mipmap_lod_10_samples = 0x53,
+ TA_PERF_SEL_mipmap_lod_11_samples = 0x54,
+ TA_PERF_SEL_mipmap_lod_12_samples = 0x55,
+ TA_PERF_SEL_mipmap_lod_13_samples = 0x56,
+ TA_PERF_SEL_mipmap_lod_14_samples = 0x57,
+ TA_PERF_SEL_mipmap_invalid_samples = 0x58,
+ TA_PERF_SEL_aniso_1_cycle_quads = 0x59,
+ TA_PERF_SEL_aniso_2_cycle_quads = 0x5a,
+ TA_PERF_SEL_aniso_4_cycle_quads = 0x5b,
+ TA_PERF_SEL_aniso_6_cycle_quads = 0x5c,
+ TA_PERF_SEL_aniso_8_cycle_quads = 0x5d,
+ TA_PERF_SEL_aniso_10_cycle_quads = 0x5e,
+ TA_PERF_SEL_aniso_12_cycle_quads = 0x5f,
+ TA_PERF_SEL_aniso_14_cycle_quads = 0x60,
+ TA_PERF_SEL_aniso_16_cycle_quads = 0x61,
+ TA_PERF_SEL_write_path_input_cycles = 0x62,
+ TA_PERF_SEL_write_path_output_cycles = 0x63,
+ TA_PERF_SEL_flat_wavefronts = 0x64,
+ TA_PERF_SEL_flat_read_wavefronts = 0x65,
+ TA_PERF_SEL_flat_write_wavefronts = 0x66,
+ TA_PERF_SEL_flat_atomic_wavefronts = 0x67,
+ TA_PERF_SEL_flat_coalesceable_wavefronts = 0x68,
+ TA_PERF_SEL_reg_sclk_vld = 0x69,
+ TA_PERF_SEL_local_cg_dyn_sclk_grp0_en = 0x6a,
+ TA_PERF_SEL_local_cg_dyn_sclk_grp1_en = 0x6b,
+ TA_PERF_SEL_local_cg_dyn_sclk_grp1_mems_en = 0x6c,
+ TA_PERF_SEL_local_cg_dyn_sclk_grp4_en = 0x6d,
+ TA_PERF_SEL_local_cg_dyn_sclk_grp5_en = 0x6e,
+ TA_PERF_SEL_xnack_on_phase0 = 0x6f,
+ TA_PERF_SEL_xnack_on_phase1 = 0x70,
+ TA_PERF_SEL_xnack_on_phase2 = 0x71,
+ TA_PERF_SEL_xnack_on_phase3 = 0x72,
+ TA_PERF_SEL_first_xnack_on_phase0 = 0x73,
+ TA_PERF_SEL_first_xnack_on_phase1 = 0x74,
+ TA_PERF_SEL_first_xnack_on_phase2 = 0x75,
+ TA_PERF_SEL_first_xnack_on_phase3 = 0x76,
+} TA_PERFCOUNT_SEL;
+typedef enum TD_PERFCOUNT_SEL {
+ TD_PERF_SEL_none = 0x0,
+ TD_PERF_SEL_td_busy = 0x1,
+ TD_PERF_SEL_input_busy = 0x2,
+ TD_PERF_SEL_output_busy = 0x3,
+ TD_PERF_SEL_lerp_busy = 0x4,
+ TD_PERF_SEL_reg_sclk_vld = 0x5,
+ TD_PERF_SEL_local_cg_dyn_sclk_grp0_en = 0x6,
+ TD_PERF_SEL_local_cg_dyn_sclk_grp1_en = 0x7,
+ TD_PERF_SEL_local_cg_dyn_sclk_grp4_en = 0x8,
+ TD_PERF_SEL_local_cg_dyn_sclk_grp5_en = 0x9,
+ TD_PERF_SEL_tc_td_fifo_full = 0xa,
+ TD_PERF_SEL_constant_state_full = 0xb,
+ TD_PERF_SEL_sample_state_full = 0xc,
+ TD_PERF_SEL_output_fifo_full = 0xd,
+ TD_PERF_SEL_RESERVED_14 = 0xe,
+ TD_PERF_SEL_tc_stall = 0xf,
+ TD_PERF_SEL_pc_stall = 0x10,
+ TD_PERF_SEL_gds_stall = 0x11,
+ TD_PERF_SEL_RESERVED_18 = 0x12,
+ TD_PERF_SEL_RESERVED_19 = 0x13,
+ TD_PERF_SEL_gather4_wavefront = 0x14,
+ TD_PERF_SEL_sample_c_wavefront = 0x15,
+ TD_PERF_SEL_load_wavefront = 0x16,
+ TD_PERF_SEL_atomic_wavefront = 0x17,
+ TD_PERF_SEL_store_wavefront = 0x18,
+ TD_PERF_SEL_ldfptr_wavefront = 0x19,
+ TD_PERF_SEL_RESERVED_26 = 0x1a,
+ TD_PERF_SEL_RESERVED_27 = 0x1b,
+ TD_PERF_SEL_d16_en_wavefront = 0x1c,
+ TD_PERF_SEL_bicubic_filter_wavefront = 0x1d,
+ TD_PERF_SEL_bypass_filter_wavefront = 0x1e,
+ TD_PERF_SEL_min_max_filter_wavefront = 0x1f,
+ TD_PERF_SEL_coalescable_wavefront = 0x20,
+ TD_PERF_SEL_coalesced_phase = 0x21,
+ TD_PERF_SEL_four_phase_wavefront = 0x22,
+ TD_PERF_SEL_eight_phase_wavefront = 0x23,
+ TD_PERF_SEL_sixteen_phase_wavefront = 0x24,
+ TD_PERF_SEL_four_phase_forward_wavefront = 0x25,
+ TD_PERF_SEL_write_ack_wavefront = 0x26,
+ TD_PERF_SEL_RESERVED_39 = 0x27,
+ TD_PERF_SEL_user_defined_border = 0x28,
+ TD_PERF_SEL_white_border = 0x29,
+ TD_PERF_SEL_opaque_black_border = 0x2a,
+ TD_PERF_SEL_RESERVED_43 = 0x2b,
+ TD_PERF_SEL_RESERVED_44 = 0x2c,
+ TD_PERF_SEL_nack = 0x2d,
+ TD_PERF_SEL_td_sp_traffic = 0x2e,
+ TD_PERF_SEL_consume_gds_traffic = 0x2f,
+ TD_PERF_SEL_addresscmd_poison = 0x30,
+ TD_PERF_SEL_data_poison = 0x31,
+ TD_PERF_SEL_start_cycle_0 = 0x32,
+ TD_PERF_SEL_start_cycle_1 = 0x33,
+ TD_PERF_SEL_start_cycle_2 = 0x34,
+ TD_PERF_SEL_start_cycle_3 = 0x35,
+ TD_PERF_SEL_null_cycle_output = 0x36,
+ TD_PERF_SEL_d16_data_packed = 0x37,
+} TD_PERFCOUNT_SEL;
+typedef enum TCP_PERFCOUNT_SELECT {
+ TCP_PERF_SEL_TA_TCP_ADDR_STARVE_CYCLES = 0x0,
+ TCP_PERF_SEL_TA_TCP_DATA_STARVE_CYCLES = 0x1,
+ TCP_PERF_SEL_TCP_TA_ADDR_STALL_CYCLES = 0x2,
+ TCP_PERF_SEL_TCP_TA_DATA_STALL_CYCLES = 0x3,
+ TCP_PERF_SEL_TD_TCP_STALL_CYCLES = 0x4,
+ TCP_PERF_SEL_TCR_TCP_STALL_CYCLES = 0x5,
+ TCP_PERF_SEL_LOD_STALL_CYCLES = 0x6,
+ TCP_PERF_SEL_READ_TAGCONFLICT_STALL_CYCLES = 0x7,
+ TCP_PERF_SEL_WRITE_TAGCONFLICT_STALL_CYCLES = 0x8,
+ TCP_PERF_SEL_ATOMIC_TAGCONFLICT_STALL_CYCLES = 0x9,
+ TCP_PERF_SEL_ALLOC_STALL_CYCLES = 0xa,
+ TCP_PERF_SEL_LFIFO_STALL_CYCLES = 0xb,
+ TCP_PERF_SEL_RFIFO_STALL_CYCLES = 0xc,
+ TCP_PERF_SEL_TCR_RDRET_STALL = 0xd,
+ TCP_PERF_SEL_WRITE_CONFLICT_STALL = 0xe,
+ TCP_PERF_SEL_HOLE_READ_STALL = 0xf,
+ TCP_PERF_SEL_READCONFLICT_STALL_CYCLES = 0x10,
+ TCP_PERF_SEL_PENDING_STALL_CYCLES = 0x11,
+ TCP_PERF_SEL_READFIFO_STALL_CYCLES = 0x12,
+ TCP_PERF_SEL_TCP_LATENCY = 0x13,
+ TCP_PERF_SEL_TCC_READ_REQ_LATENCY = 0x14,
+ TCP_PERF_SEL_TCC_WRITE_REQ_LATENCY = 0x15,
+ TCP_PERF_SEL_TCC_WRITE_REQ_HOLE_LATENCY = 0x16,
+ TCP_PERF_SEL_TCC_READ_REQ = 0x17,
+ TCP_PERF_SEL_TCC_WRITE_REQ = 0x18,
+ TCP_PERF_SEL_TCC_ATOMIC_WITH_RET_REQ = 0x19,
+ TCP_PERF_SEL_TCC_ATOMIC_WITHOUT_RET_REQ = 0x1a,
+ TCP_PERF_SEL_TOTAL_LOCAL_READ = 0x1b,
+ TCP_PERF_SEL_TOTAL_GLOBAL_READ = 0x1c,
+ TCP_PERF_SEL_TOTAL_LOCAL_WRITE = 0x1d,
+ TCP_PERF_SEL_TOTAL_GLOBAL_WRITE = 0x1e,
+ TCP_PERF_SEL_TOTAL_ATOMIC_WITH_RET = 0x1f,
+ TCP_PERF_SEL_TOTAL_ATOMIC_WITHOUT_RET = 0x20,
+ TCP_PERF_SEL_TOTAL_WBINVL1 = 0x21,
+ TCP_PERF_SEL_IMG_READ_FMT_1 = 0x22,
+ TCP_PERF_SEL_IMG_READ_FMT_8 = 0x23,
+ TCP_PERF_SEL_IMG_READ_FMT_16 = 0x24,
+ TCP_PERF_SEL_IMG_READ_FMT_32 = 0x25,
+ TCP_PERF_SEL_IMG_READ_FMT_32_AS_8 = 0x26,
+ TCP_PERF_SEL_IMG_READ_FMT_32_AS_16 = 0x27,
+ TCP_PERF_SEL_IMG_READ_FMT_32_AS_128 = 0x28,
+ TCP_PERF_SEL_IMG_READ_FMT_64_2_CYCLE = 0x29,
+ TCP_PERF_SEL_IMG_READ_FMT_64_1_CYCLE = 0x2a,
+ TCP_PERF_SEL_IMG_READ_FMT_96 = 0x2b,
+ TCP_PERF_SEL_IMG_READ_FMT_128_4_CYCLE = 0x2c,
+ TCP_PERF_SEL_IMG_READ_FMT_128_1_CYCLE = 0x2d,
+ TCP_PERF_SEL_IMG_READ_FMT_BC1 = 0x2e,
+ TCP_PERF_SEL_IMG_READ_FMT_BC2 = 0x2f,
+ TCP_PERF_SEL_IMG_READ_FMT_BC3 = 0x30,
+ TCP_PERF_SEL_IMG_READ_FMT_BC4 = 0x31,
+ TCP_PERF_SEL_IMG_READ_FMT_BC5 = 0x32,
+ TCP_PERF_SEL_IMG_READ_FMT_BC6 = 0x33,
+ TCP_PERF_SEL_IMG_READ_FMT_BC7 = 0x34,
+ TCP_PERF_SEL_IMG_READ_FMT_I8 = 0x35,
+ TCP_PERF_SEL_IMG_READ_FMT_I16 = 0x36,
+ TCP_PERF_SEL_IMG_READ_FMT_I32 = 0x37,
+ TCP_PERF_SEL_IMG_READ_FMT_I32_AS_8 = 0x38,
+ TCP_PERF_SEL_IMG_READ_FMT_I32_AS_16 = 0x39,
+ TCP_PERF_SEL_IMG_READ_FMT_D8 = 0x3a,
+ TCP_PERF_SEL_IMG_READ_FMT_D16 = 0x3b,
+ TCP_PERF_SEL_IMG_READ_FMT_D32 = 0x3c,
+ TCP_PERF_SEL_IMG_WRITE_FMT_8 = 0x3d,
+ TCP_PERF_SEL_IMG_WRITE_FMT_16 = 0x3e,
+ TCP_PERF_SEL_IMG_WRITE_FMT_32 = 0x3f,
+ TCP_PERF_SEL_IMG_WRITE_FMT_64 = 0x40,
+ TCP_PERF_SEL_IMG_WRITE_FMT_128 = 0x41,
+ TCP_PERF_SEL_IMG_WRITE_FMT_D8 = 0x42,
+ TCP_PERF_SEL_IMG_WRITE_FMT_D16 = 0x43,
+ TCP_PERF_SEL_IMG_WRITE_FMT_D32 = 0x44,
+ TCP_PERF_SEL_IMG_ATOMIC_WITH_RET_FMT_32 = 0x45,
+ TCP_PERF_SEL_IMG_ATOMIC_WITHOUT_RET_FMT_32 = 0x46,
+ TCP_PERF_SEL_IMG_ATOMIC_WITH_RET_FMT_64 = 0x47,
+ TCP_PERF_SEL_IMG_ATOMIC_WITHOUT_RET_FMT_64 = 0x48,
+ TCP_PERF_SEL_BUF_READ_FMT_8 = 0x49,
+ TCP_PERF_SEL_BUF_READ_FMT_16 = 0x4a,
+ TCP_PERF_SEL_BUF_READ_FMT_32 = 0x4b,
+ TCP_PERF_SEL_BUF_WRITE_FMT_8 = 0x4c,
+ TCP_PERF_SEL_BUF_WRITE_FMT_16 = 0x4d,
+ TCP_PERF_SEL_BUF_WRITE_FMT_32 = 0x4e,
+ TCP_PERF_SEL_BUF_ATOMIC_WITH_RET_FMT_32 = 0x4f,
+ TCP_PERF_SEL_BUF_ATOMIC_WITHOUT_RET_FMT_32 = 0x50,
+ TCP_PERF_SEL_BUF_ATOMIC_WITH_RET_FMT_64 = 0x51,
+ TCP_PERF_SEL_BUF_ATOMIC_WITHOUT_RET_FMT_64 = 0x52,
+ TCP_PERF_SEL_ARR_LINEAR_GENERAL = 0x53,
+ TCP_PERF_SEL_ARR_LINEAR_ALIGNED = 0x54,
+ TCP_PERF_SEL_ARR_1D_THIN1 = 0x55,
+ TCP_PERF_SEL_ARR_1D_THICK = 0x56,
+ TCP_PERF_SEL_ARR_2D_THIN1 = 0x57,
+ TCP_PERF_SEL_ARR_2D_THICK = 0x58,
+ TCP_PERF_SEL_ARR_2D_XTHICK = 0x59,
+ TCP_PERF_SEL_ARR_3D_THIN1 = 0x5a,
+ TCP_PERF_SEL_ARR_3D_THICK = 0x5b,
+ TCP_PERF_SEL_ARR_3D_XTHICK = 0x5c,
+ TCP_PERF_SEL_DIM_1D = 0x5d,
+ TCP_PERF_SEL_DIM_2D = 0x5e,
+ TCP_PERF_SEL_DIM_3D = 0x5f,
+ TCP_PERF_SEL_DIM_1D_ARRAY = 0x60,
+ TCP_PERF_SEL_DIM_2D_ARRAY = 0x61,
+ TCP_PERF_SEL_DIM_2D_MSAA = 0x62,
+ TCP_PERF_SEL_DIM_2D_ARRAY_MSAA = 0x63,
+ TCP_PERF_SEL_DIM_CUBE_ARRAY = 0x64,
+ TCP_PERF_SEL_CP_TCP_INVALIDATE = 0x65,
+ TCP_PERF_SEL_TA_TCP_STATE_READ = 0x66,
+ TCP_PERF_SEL_TAGRAM0_REQ = 0x67,
+ TCP_PERF_SEL_TAGRAM1_REQ = 0x68,
+ TCP_PERF_SEL_TAGRAM2_REQ = 0x69,
+ TCP_PERF_SEL_TAGRAM3_REQ = 0x6a,
+ TCP_PERF_SEL_GATE_EN1 = 0x6b,
+ TCP_PERF_SEL_GATE_EN2 = 0x6c,
+ TCP_PERF_SEL_CORE_REG_SCLK_VLD = 0x6d,
+ TCP_PERF_SEL_TCC_REQ = 0x6e,
+ TCP_PERF_SEL_TCC_NON_READ_REQ = 0x6f,
+ TCP_PERF_SEL_TCC_BYPASS_READ_REQ = 0x70,
+ TCP_PERF_SEL_TCC_MISS_EVICT_READ_REQ = 0x71,
+ TCP_PERF_SEL_TCC_VOLATILE_READ_REQ = 0x72,
+ TCP_PERF_SEL_TCC_VOLATILE_BYPASS_READ_REQ = 0x73,
+ TCP_PERF_SEL_TCC_VOLATILE_MISS_EVICT_READ_REQ = 0x74,
+ TCP_PERF_SEL_TCC_BYPASS_WRITE_REQ = 0x75,
+ TCP_PERF_SEL_TCC_MISS_EVICT_WRITE_REQ = 0x76,
+ TCP_PERF_SEL_TCC_VOLATILE_BYPASS_WRITE_REQ = 0x77,
+ TCP_PERF_SEL_TCC_VOLATILE_WRITE_REQ = 0x78,
+ TCP_PERF_SEL_TCC_VOLATILE_MISS_EVICT_WRITE_REQ = 0x79,
+ TCP_PERF_SEL_TCC_BYPASS_ATOMIC_REQ = 0x7a,
+ TCP_PERF_SEL_TCC_ATOMIC_REQ = 0x7b,
+ TCP_PERF_SEL_TCC_VOLATILE_ATOMIC_REQ = 0x7c,
+ TCP_PERF_SEL_TCC_DATA_BUS_BUSY = 0x7d,
+ TCP_PERF_SEL_TOTAL_ACCESSES = 0x7e,
+ TCP_PERF_SEL_TOTAL_READ = 0x7f,
+ TCP_PERF_SEL_TOTAL_HIT_LRU_READ = 0x80,
+ TCP_PERF_SEL_TOTAL_HIT_EVICT_READ = 0x81,
+ TCP_PERF_SEL_TOTAL_MISS_LRU_READ = 0x82,
+ TCP_PERF_SEL_TOTAL_MISS_EVICT_READ = 0x83,
+ TCP_PERF_SEL_TOTAL_NON_READ = 0x84,
+ TCP_PERF_SEL_TOTAL_WRITE = 0x85,
+ TCP_PERF_SEL_TOTAL_MISS_LRU_WRITE = 0x86,
+ TCP_PERF_SEL_TOTAL_MISS_EVICT_WRITE = 0x87,
+ TCP_PERF_SEL_TOTAL_WBINVL1_VOL = 0x88,
+ TCP_PERF_SEL_TOTAL_WRITEBACK_INVALIDATES = 0x89,
+ TCP_PERF_SEL_DISPLAY_MICROTILING = 0x8a,
+ TCP_PERF_SEL_THIN_MICROTILING = 0x8b,
+ TCP_PERF_SEL_DEPTH_MICROTILING = 0x8c,
+ TCP_PERF_SEL_ARR_PRT_THIN1 = 0x8d,
+ TCP_PERF_SEL_ARR_PRT_2D_THIN1 = 0x8e,
+ TCP_PERF_SEL_ARR_PRT_3D_THIN1 = 0x8f,
+ TCP_PERF_SEL_ARR_PRT_THICK = 0x90,
+ TCP_PERF_SEL_ARR_PRT_2D_THICK = 0x91,
+ TCP_PERF_SEL_ARR_PRT_3D_THICK = 0x92,
+ TCP_PERF_SEL_CP_TCP_INVALIDATE_VOL = 0x93,
+ TCP_PERF_SEL_SQ_TCP_INVALIDATE_VOL = 0x94,
+ TCP_PERF_SEL_UNALIGNED = 0x95,
+ TCP_PERF_SEL_ROTATED_MICROTILING = 0x96,
+ TCP_PERF_SEL_THICK_MICROTILING = 0x97,
+ TCP_PERF_SEL_ATC = 0x98,
+ TCP_PERF_SEL_POWER_STALL = 0x99,
+ TCP_PERF_SEL_RESERVED_154 = 0x9a,
+ TCP_PERF_SEL_TCC_LRU_REQ = 0x9b,
+ TCP_PERF_SEL_TCC_STREAM_REQ = 0x9c,
+ TCP_PERF_SEL_TCC_NC_READ_REQ = 0x9d,
+ TCP_PERF_SEL_TCC_NC_WRITE_REQ = 0x9e,
+ TCP_PERF_SEL_TCC_NC_ATOMIC_REQ = 0x9f,
+ TCP_PERF_SEL_TCC_UC_READ_REQ = 0xa0,
+ TCP_PERF_SEL_TCC_UC_WRITE_REQ = 0xa1,
+ TCP_PERF_SEL_TCC_UC_ATOMIC_REQ = 0xa2,
+ TCP_PERF_SEL_TCC_CC_READ_REQ = 0xa3,
+ TCP_PERF_SEL_TCC_CC_WRITE_REQ = 0xa4,
+ TCP_PERF_SEL_TCC_CC_ATOMIC_REQ = 0xa5,
+ TCP_PERF_SEL_TCC_DCC_REQ = 0xa6,
+ TCP_PERF_SEL_TCC_PHYSICAL_REQ = 0xa7,
+ TCP_PERF_SEL_UNORDERED_MTYPE_STALL = 0xa8,
+ TCP_PERF_SEL_VOLATILE = 0xa9,
+ TCP_PERF_SEL_TC_TA_XNACK_STALL = 0xaa,
+ TCP_PERF_SEL_ATCL1_SERIALIZATION_STALL = 0xab,
+ TCP_PERF_SEL_SHOOTDOWN = 0xac,
+ TCP_PERF_SEL_GATCL1_TRANSLATION_MISS = 0xad,
+ TCP_PERF_SEL_GATCL1_PERMISSION_MISS = 0xae,
+ TCP_PERF_SEL_GATCL1_REQUEST = 0xaf,
+ TCP_PERF_SEL_GATCL1_STALL_INFLIGHT_MAX = 0xb0,
+ TCP_PERF_SEL_GATCL1_STALL_LRU_INFLIGHT = 0xb1,
+ TCP_PERF_SEL_GATCL1_LFIFO_FULL = 0xb2,
+ TCP_PERF_SEL_GATCL1_STALL_LFIFO_NOT_RES = 0xb3,
+ TCP_PERF_SEL_GATCL1_STALL_ATCL2_REQ_OUT_OF_CREDITS= 0xb4,
+ TCP_PERF_SEL_GATCL1_ATCL2_INFLIGHT = 0xb5,
+ TCP_PERF_SEL_GATCL1_STALL_MISSFIFO_FULL = 0xb6,
+ TCP_PERF_SEL_IMG_READ_FMT_ETC2_RGB = 0xb7,
+ TCP_PERF_SEL_IMG_READ_FMT_ETC2_RGBA = 0xb8,
+ TCP_PERF_SEL_IMG_READ_FMT_ETC2_RGBA1 = 0xb9,
+ TCP_PERF_SEL_IMG_READ_FMT_ETC2_R = 0xba,
+ TCP_PERF_SEL_IMG_READ_FMT_ETC2_RG = 0xbb,
+ TCP_PERF_SEL_IMG_READ_FMT_8_AS_32 = 0xbc,
+ TCP_PERF_SEL_IMG_READ_FMT_8_AS_64 = 0xbd,
+ TCP_PERF_SEL_IMG_READ_FMT_16_AS_64 = 0xbe,
+ TCP_PERF_SEL_IMG_READ_FMT_16_AS_128 = 0xbf,
+ TCP_PERF_SEL_IMG_WRITE_FMT_8_AS_32 = 0xc0,
+ TCP_PERF_SEL_IMG_WRITE_FMT_8_AS_64 = 0xc1,
+ TCP_PERF_SEL_IMG_WRITE_FMT_16_AS_64 = 0xc2,
+ TCP_PERF_SEL_IMG_WRITE_FMT_16_AS_128 = 0xc3,
+} TCP_PERFCOUNT_SELECT;
+typedef enum TCP_CACHE_POLICIES {
+ TCP_CACHE_POLICY_MISS_LRU = 0x0,
+ TCP_CACHE_POLICY_MISS_EVICT = 0x1,
+ TCP_CACHE_POLICY_HIT_LRU = 0x2,
+ TCP_CACHE_POLICY_HIT_EVICT = 0x3,
+} TCP_CACHE_POLICIES;
+typedef enum TCP_CACHE_STORE_POLICIES {
+ TCP_CACHE_STORE_POLICY_WT_LRU = 0x0,
+ TCP_CACHE_STORE_POLICY_WT_EVICT = 0x1,
+} TCP_CACHE_STORE_POLICIES;
+typedef enum TCP_WATCH_MODES {
+ TCP_WATCH_MODE_READ = 0x0,
+ TCP_WATCH_MODE_NONREAD = 0x1,
+ TCP_WATCH_MODE_ATOMIC = 0x2,
+ TCP_WATCH_MODE_ALL = 0x3,
+} TCP_WATCH_MODES;
+typedef enum TCP_DSM_DATA_SEL {
+ TCP_DSM_DISABLE = 0x0,
+ TCP_DSM_SEL0 = 0x1,
+ TCP_DSM_SEL1 = 0x2,
+ TCP_DSM_SEL_BOTH = 0x3,
+} TCP_DSM_DATA_SEL;
+typedef enum TCP_DSM_SINGLE_WRITE {
+ TCP_DSM_SINGLE_WRITE_EN = 0x1,
+} TCP_DSM_SINGLE_WRITE;
+typedef enum VGT_OUT_PRIM_TYPE {
+ VGT_OUT_POINT = 0x0,
+ VGT_OUT_LINE = 0x1,
+ VGT_OUT_TRI = 0x2,
+ VGT_OUT_RECT_V0 = 0x3,
+ VGT_OUT_RECT_V1 = 0x4,
+ VGT_OUT_RECT_V2 = 0x5,
+ VGT_OUT_RECT_V3 = 0x6,
+ VGT_OUT_RESERVED = 0x7,
+ VGT_TE_QUAD = 0x8,
+ VGT_TE_PRIM_INDEX_LINE = 0x9,
+ VGT_TE_PRIM_INDEX_TRI = 0xa,
+ VGT_TE_PRIM_INDEX_QUAD = 0xb,
+ VGT_OUT_LINE_ADJ = 0xc,
+ VGT_OUT_TRI_ADJ = 0xd,
+ VGT_OUT_PATCH = 0xe,
+} VGT_OUT_PRIM_TYPE;
+typedef enum VGT_DI_PRIM_TYPE {
+ DI_PT_NONE = 0x0,
+ DI_PT_POINTLIST = 0x1,
+ DI_PT_LINELIST = 0x2,
+ DI_PT_LINESTRIP = 0x3,
+ DI_PT_TRILIST = 0x4,
+ DI_PT_TRIFAN = 0x5,
+ DI_PT_TRISTRIP = 0x6,
+ DI_PT_UNUSED_0 = 0x7,
+ DI_PT_UNUSED_1 = 0x8,
+ DI_PT_PATCH = 0x9,
+ DI_PT_LINELIST_ADJ = 0xa,
+ DI_PT_LINESTRIP_ADJ = 0xb,
+ DI_PT_TRILIST_ADJ = 0xc,
+ DI_PT_TRISTRIP_ADJ = 0xd,
+ DI_PT_UNUSED_3 = 0xe,
+ DI_PT_UNUSED_4 = 0xf,
+ DI_PT_TRI_WITH_WFLAGS = 0x10,
+ DI_PT_RECTLIST = 0x11,
+ DI_PT_LINELOOP = 0x12,
+ DI_PT_QUADLIST = 0x13,
+ DI_PT_QUADSTRIP = 0x14,
+ DI_PT_POLYGON = 0x15,
+ DI_PT_2D_COPY_RECT_LIST_V0 = 0x16,
+ DI_PT_2D_COPY_RECT_LIST_V1 = 0x17,
+ DI_PT_2D_COPY_RECT_LIST_V2 = 0x18,
+ DI_PT_2D_COPY_RECT_LIST_V3 = 0x19,
+ DI_PT_2D_FILL_RECT_LIST = 0x1a,
+ DI_PT_2D_LINE_STRIP = 0x1b,
+ DI_PT_2D_TRI_STRIP = 0x1c,
+} VGT_DI_PRIM_TYPE;
+typedef enum VGT_DI_SOURCE_SELECT {
+ DI_SRC_SEL_DMA = 0x0,
+ DI_SRC_SEL_IMMEDIATE = 0x1,
+ DI_SRC_SEL_AUTO_INDEX = 0x2,
+ DI_SRC_SEL_RESERVED = 0x3,
+} VGT_DI_SOURCE_SELECT;
+typedef enum VGT_DI_MAJOR_MODE_SELECT {
+ DI_MAJOR_MODE_0 = 0x0,
+ DI_MAJOR_MODE_1 = 0x1,
+} VGT_DI_MAJOR_MODE_SELECT;
+typedef enum VGT_DI_INDEX_SIZE {
+ DI_INDEX_SIZE_16_BIT = 0x0,
+ DI_INDEX_SIZE_32_BIT = 0x1,
+ DI_INDEX_SIZE_8_BIT = 0x2,
+} VGT_DI_INDEX_SIZE;
+typedef enum VGT_EVENT_TYPE {
+ Reserved_0x00 = 0x0,
+ SAMPLE_STREAMOUTSTATS1 = 0x1,
+ SAMPLE_STREAMOUTSTATS2 = 0x2,
+ SAMPLE_STREAMOUTSTATS3 = 0x3,
+ CACHE_FLUSH_TS = 0x4,
+ CONTEXT_DONE = 0x5,
+ CACHE_FLUSH = 0x6,
+ CS_PARTIAL_FLUSH = 0x7,
+ VGT_STREAMOUT_SYNC = 0x8,
+ Reserved_0x09 = 0x9,
+ VGT_STREAMOUT_RESET = 0xa,
+ END_OF_PIPE_INCR_DE = 0xb,
+ END_OF_PIPE_IB_END = 0xc,
+ RST_PIX_CNT = 0xd,
+ Reserved_0x0E = 0xe,
+ VS_PARTIAL_FLUSH = 0xf,
+ PS_PARTIAL_FLUSH = 0x10,
+ FLUSH_HS_OUTPUT = 0x11,
+ FLUSH_LS_OUTPUT = 0x12,
+ Reserved_0x13 = 0x13,
+ CACHE_FLUSH_AND_INV_TS_EVENT = 0x14,
+ ZPASS_DONE = 0x15,
+ CACHE_FLUSH_AND_INV_EVENT = 0x16,
+ PERFCOUNTER_START = 0x17,
+ PERFCOUNTER_STOP = 0x18,
+ PIPELINESTAT_START = 0x19,
+ PIPELINESTAT_STOP = 0x1a,
+ PERFCOUNTER_SAMPLE = 0x1b,
+ FLUSH_ES_OUTPUT = 0x1c,
+ FLUSH_GS_OUTPUT = 0x1d,
+ SAMPLE_PIPELINESTAT = 0x1e,
+ SO_VGTSTREAMOUT_FLUSH = 0x1f,
+ SAMPLE_STREAMOUTSTATS = 0x20,
+ RESET_VTX_CNT = 0x21,
+ BLOCK_CONTEXT_DONE = 0x22,
+ CS_CONTEXT_DONE = 0x23,
+ VGT_FLUSH = 0x24,
+ TGID_ROLLOVER = 0x25,
+ SQ_NON_EVENT = 0x26,
+ SC_SEND_DB_VPZ = 0x27,
+ BOTTOM_OF_PIPE_TS = 0x28,
+ FLUSH_SX_TS = 0x29,
+ DB_CACHE_FLUSH_AND_INV = 0x2a,
+ FLUSH_AND_INV_DB_DATA_TS = 0x2b,
+ FLUSH_AND_INV_DB_META = 0x2c,
+ FLUSH_AND_INV_CB_DATA_TS = 0x2d,
+ FLUSH_AND_INV_CB_META = 0x2e,
+ CS_DONE = 0x2f,
+ PS_DONE = 0x30,
+ FLUSH_AND_INV_CB_PIXEL_DATA = 0x31,
+ SX_CB_RAT_ACK_REQUEST = 0x32,
+ THREAD_TRACE_START = 0x33,
+ THREAD_TRACE_STOP = 0x34,
+ THREAD_TRACE_MARKER = 0x35,
+ THREAD_TRACE_FLUSH = 0x36,
+ THREAD_TRACE_FINISH = 0x37,
+ PIXEL_PIPE_STAT_CONTROL = 0x38,
+ PIXEL_PIPE_STAT_DUMP = 0x39,
+ PIXEL_PIPE_STAT_RESET = 0x3a,
+ CONTEXT_SUSPEND = 0x3b,
+ OFFCHIP_HS_DEALLOC = 0x3c,
+} VGT_EVENT_TYPE;
+typedef enum VGT_DMA_SWAP_MODE {
+ VGT_DMA_SWAP_NONE = 0x0,
+ VGT_DMA_SWAP_16_BIT = 0x1,
+ VGT_DMA_SWAP_32_BIT = 0x2,
+ VGT_DMA_SWAP_WORD = 0x3,
+} VGT_DMA_SWAP_MODE;
+typedef enum VGT_INDEX_TYPE_MODE {
+ VGT_INDEX_16 = 0x0,
+ VGT_INDEX_32 = 0x1,
+ VGT_INDEX_8 = 0x2,
+} VGT_INDEX_TYPE_MODE;
+typedef enum VGT_DMA_BUF_TYPE {
+ VGT_DMA_BUF_MEM = 0x0,
+ VGT_DMA_BUF_RING = 0x1,
+ VGT_DMA_BUF_SETUP = 0x2,
+ VGT_DMA_PTR_UPDATE = 0x3,
+} VGT_DMA_BUF_TYPE;
+typedef enum VGT_OUTPATH_SELECT {
+ VGT_OUTPATH_VTX_REUSE = 0x0,
+ VGT_OUTPATH_TESS_EN = 0x1,
+ VGT_OUTPATH_PASSTHRU = 0x2,
+ VGT_OUTPATH_GS_BLOCK = 0x3,
+ VGT_OUTPATH_HS_BLOCK = 0x4,
+} VGT_OUTPATH_SELECT;
+typedef enum VGT_GRP_PRIM_TYPE {
+ VGT_GRP_3D_POINT = 0x0,
+ VGT_GRP_3D_LINE = 0x1,
+ VGT_GRP_3D_TRI = 0x2,
+ VGT_GRP_3D_RECT = 0x3,
+ VGT_GRP_3D_QUAD = 0x4,
+ VGT_GRP_2D_COPY_RECT_V0 = 0x5,
+ VGT_GRP_2D_COPY_RECT_V1 = 0x6,
+ VGT_GRP_2D_COPY_RECT_V2 = 0x7,
+ VGT_GRP_2D_COPY_RECT_V3 = 0x8,
+ VGT_GRP_2D_FILL_RECT = 0x9,
+ VGT_GRP_2D_LINE = 0xa,
+ VGT_GRP_2D_TRI = 0xb,
+ VGT_GRP_PRIM_INDEX_LINE = 0xc,
+ VGT_GRP_PRIM_INDEX_TRI = 0xd,
+ VGT_GRP_PRIM_INDEX_QUAD = 0xe,
+ VGT_GRP_3D_LINE_ADJ = 0xf,
+ VGT_GRP_3D_TRI_ADJ = 0x10,
+ VGT_GRP_3D_PATCH = 0x11,
+} VGT_GRP_PRIM_TYPE;
+typedef enum VGT_GRP_PRIM_ORDER {
+ VGT_GRP_LIST = 0x0,
+ VGT_GRP_STRIP = 0x1,
+ VGT_GRP_FAN = 0x2,
+ VGT_GRP_LOOP = 0x3,
+ VGT_GRP_POLYGON = 0x4,
+} VGT_GRP_PRIM_ORDER;
+typedef enum VGT_GROUP_CONV_SEL {
+ VGT_GRP_INDEX_16 = 0x0,
+ VGT_GRP_INDEX_32 = 0x1,
+ VGT_GRP_UINT_16 = 0x2,
+ VGT_GRP_UINT_32 = 0x3,
+ VGT_GRP_SINT_16 = 0x4,
+ VGT_GRP_SINT_32 = 0x5,
+ VGT_GRP_FLOAT_32 = 0x6,
+ VGT_GRP_AUTO_PRIM = 0x7,
+ VGT_GRP_FIX_1_23_TO_FLOAT = 0x8,
+} VGT_GROUP_CONV_SEL;
+typedef enum VGT_GS_MODE_TYPE {
+ GS_OFF = 0x0,
+ GS_SCENARIO_A = 0x1,
+ GS_SCENARIO_B = 0x2,
+ GS_SCENARIO_G = 0x3,
+ GS_SCENARIO_C = 0x4,
+ SPRITE_EN = 0x5,
+} VGT_GS_MODE_TYPE;
+typedef enum VGT_GS_CUT_MODE {
+ GS_CUT_1024 = 0x0,
+ GS_CUT_512 = 0x1,
+ GS_CUT_256 = 0x2,
+ GS_CUT_128 = 0x3,
+} VGT_GS_CUT_MODE;
+typedef enum VGT_GS_OUTPRIM_TYPE {
+ POINTLIST = 0x0,
+ LINESTRIP = 0x1,
+ TRISTRIP = 0x2,
+} VGT_GS_OUTPRIM_TYPE;
+typedef enum VGT_CACHE_INVALID_MODE {
+ VC_ONLY = 0x0,
+ TC_ONLY = 0x1,
+ VC_AND_TC = 0x2,
+} VGT_CACHE_INVALID_MODE;
+typedef enum VGT_TESS_TYPE {
+ TESS_ISOLINE = 0x0,
+ TESS_TRIANGLE = 0x1,
+ TESS_QUAD = 0x2,
+} VGT_TESS_TYPE;
+typedef enum VGT_TESS_PARTITION {
+ PART_INTEGER = 0x0,
+ PART_POW2 = 0x1,
+ PART_FRAC_ODD = 0x2,
+ PART_FRAC_EVEN = 0x3,
+} VGT_TESS_PARTITION;
+typedef enum VGT_TESS_TOPOLOGY {
+ OUTPUT_POINT = 0x0,
+ OUTPUT_LINE = 0x1,
+ OUTPUT_TRIANGLE_CW = 0x2,
+ OUTPUT_TRIANGLE_CCW = 0x3,
+} VGT_TESS_TOPOLOGY;
+typedef enum VGT_RDREQ_POLICY {
+ VGT_POLICY_LRU = 0x0,
+ VGT_POLICY_STREAM = 0x1,
+} VGT_RDREQ_POLICY;
+typedef enum VGT_DIST_MODE {
+ NO_DIST = 0x0,
+ PATCHES = 0x1,
+ DONUTS = 0x2,
+} VGT_DIST_MODE;
+typedef enum VGT_STAGES_LS_EN {
+ LS_STAGE_OFF = 0x0,
+ LS_STAGE_ON = 0x1,
+ CS_STAGE_ON = 0x2,
+ RESERVED_LS = 0x3,
+} VGT_STAGES_LS_EN;
+typedef enum VGT_STAGES_HS_EN {
+ HS_STAGE_OFF = 0x0,
+ HS_STAGE_ON = 0x1,
+} VGT_STAGES_HS_EN;
+typedef enum VGT_STAGES_ES_EN {
+ ES_STAGE_OFF = 0x0,
+ ES_STAGE_DS = 0x1,
+ ES_STAGE_REAL = 0x2,
+ RESERVED_ES = 0x3,
+} VGT_STAGES_ES_EN;
+typedef enum VGT_STAGES_GS_EN {
+ GS_STAGE_OFF = 0x0,
+ GS_STAGE_ON = 0x1,
+} VGT_STAGES_GS_EN;
+typedef enum VGT_STAGES_VS_EN {
+ VS_STAGE_REAL = 0x0,
+ VS_STAGE_DS = 0x1,
+ VS_STAGE_COPY_SHADER = 0x2,
+ RESERVED_VS = 0x3,
+} VGT_STAGES_VS_EN;
+typedef enum VGT_PERFCOUNT_SELECT {
+ vgt_perf_VGT_SPI_ESTHREAD_EVENT_WINDOW_ACTIVE = 0x0,
+ vgt_perf_VGT_SPI_ESVERT_VALID = 0x1,
+ vgt_perf_VGT_SPI_ESVERT_EOV = 0x2,
+ vgt_perf_VGT_SPI_ESVERT_STALLED = 0x3,
+ vgt_perf_VGT_SPI_ESVERT_STARVED_BUSY = 0x4,
+ vgt_perf_VGT_SPI_ESVERT_STARVED_IDLE = 0x5,
+ vgt_perf_VGT_SPI_ESVERT_STATIC = 0x6,
+ vgt_perf_VGT_SPI_ESTHREAD_IS_EVENT = 0x7,
+ vgt_perf_VGT_SPI_ESTHREAD_SEND = 0x8,
+ vgt_perf_VGT_SPI_GSPRIM_VALID = 0x9,
+ vgt_perf_VGT_SPI_GSPRIM_EOV = 0xa,
+ vgt_perf_VGT_SPI_GSPRIM_CONT = 0xb,
+ vgt_perf_VGT_SPI_GSPRIM_STALLED = 0xc,
+ vgt_perf_VGT_SPI_GSPRIM_STARVED_BUSY = 0xd,
+ vgt_perf_VGT_SPI_GSPRIM_STARVED_IDLE = 0xe,
+ vgt_perf_VGT_SPI_GSPRIM_STATIC = 0xf,
+ vgt_perf_VGT_SPI_GSTHREAD_EVENT_WINDOW_ACTIVE = 0x10,
+ vgt_perf_VGT_SPI_GSTHREAD_IS_EVENT = 0x11,
+ vgt_perf_VGT_SPI_GSTHREAD_SEND = 0x12,
+ vgt_perf_VGT_SPI_VSTHREAD_EVENT_WINDOW_ACTIVE = 0x13,
+ vgt_perf_VGT_SPI_VSVERT_SEND = 0x14,
+ vgt_perf_VGT_SPI_VSVERT_EOV = 0x15,
+ vgt_perf_VGT_SPI_VSVERT_STALLED = 0x16,
+ vgt_perf_VGT_SPI_VSVERT_STARVED_BUSY = 0x17,
+ vgt_perf_VGT_SPI_VSVERT_STARVED_IDLE = 0x18,
+ vgt_perf_VGT_SPI_VSVERT_STATIC = 0x19,
+ vgt_perf_VGT_SPI_VSTHREAD_IS_EVENT = 0x1a,
+ vgt_perf_VGT_SPI_VSTHREAD_SEND = 0x1b,
+ vgt_perf_VGT_PA_EVENT_WINDOW_ACTIVE = 0x1c,
+ vgt_perf_VGT_PA_CLIPV_SEND = 0x1d,
+ vgt_perf_VGT_PA_CLIPV_FIRSTVERT = 0x1e,
+ vgt_perf_VGT_PA_CLIPV_STALLED = 0x1f,
+ vgt_perf_VGT_PA_CLIPV_STARVED_BUSY = 0x20,
+ vgt_perf_VGT_PA_CLIPV_STARVED_IDLE = 0x21,
+ vgt_perf_VGT_PA_CLIPV_STATIC = 0x22,
+ vgt_perf_VGT_PA_CLIPP_SEND = 0x23,
+ vgt_perf_VGT_PA_CLIPP_EOP = 0x24,
+ vgt_perf_VGT_PA_CLIPP_IS_EVENT = 0x25,
+ vgt_perf_VGT_PA_CLIPP_NULL_PRIM = 0x26,
+ vgt_perf_VGT_PA_CLIPP_NEW_VTX_VECT = 0x27,
+ vgt_perf_VGT_PA_CLIPP_STALLED = 0x28,
+ vgt_perf_VGT_PA_CLIPP_STARVED_BUSY = 0x29,
+ vgt_perf_VGT_PA_CLIPP_STARVED_IDLE = 0x2a,
+ vgt_perf_VGT_PA_CLIPP_STATIC = 0x2b,
+ vgt_perf_VGT_PA_CLIPS_SEND = 0x2c,
+ vgt_perf_VGT_PA_CLIPS_STALLED = 0x2d,
+ vgt_perf_VGT_PA_CLIPS_STARVED_BUSY = 0x2e,
+ vgt_perf_VGT_PA_CLIPS_STARVED_IDLE = 0x2f,
+ vgt_perf_VGT_PA_CLIPS_STATIC = 0x30,
+ vgt_perf_vsvert_ds_send = 0x31,
+ vgt_perf_vsvert_api_send = 0x32,
+ vgt_perf_hs_tif_stall = 0x33,
+ vgt_perf_hs_input_stall = 0x34,
+ vgt_perf_hs_interface_stall = 0x35,
+ vgt_perf_hs_tfm_stall = 0x36,
+ vgt_perf_te11_starved = 0x37,
+ vgt_perf_gs_event_stall = 0x38,
+ vgt_perf_vgt_pa_clipp_send_not_event = 0x39,
+ vgt_perf_vgt_pa_clipp_valid_prim = 0x3a,
+ vgt_perf_reused_es_indices = 0x3b,
+ vgt_perf_vs_cache_hits = 0x3c,
+ vgt_perf_gs_cache_hits = 0x3d,
+ vgt_perf_ds_cache_hits = 0x3e,
+ vgt_perf_total_cache_hits = 0x3f,
+ vgt_perf_vgt_busy = 0x40,
+ vgt_perf_vgt_gs_busy = 0x41,
+ vgt_perf_esvert_stalled_es_tbl = 0x42,
+ vgt_perf_esvert_stalled_gs_tbl = 0x43,
+ vgt_perf_esvert_stalled_gs_event = 0x44,
+ vgt_perf_esvert_stalled_gsprim = 0x45,
+ vgt_perf_gsprim_stalled_es_tbl = 0x46,
+ vgt_perf_gsprim_stalled_gs_tbl = 0x47,
+ vgt_perf_gsprim_stalled_gs_event = 0x48,
+ vgt_perf_gsprim_stalled_esvert = 0x49,
+ vgt_perf_esthread_stalled_es_rb_full = 0x4a,
+ vgt_perf_esthread_stalled_spi_bp = 0x4b,
+ vgt_perf_counters_avail_stalled = 0x4c,
+ vgt_perf_gs_rb_space_avail_stalled = 0x4d,
+ vgt_perf_gs_issue_rtr_stalled = 0x4e,
+ vgt_perf_gsthread_stalled = 0x4f,
+ vgt_perf_strmout_stalled = 0x50,
+ vgt_perf_wait_for_es_done_stalled = 0x51,
+ vgt_perf_cm_stalled_by_gog = 0x52,
+ vgt_perf_cm_reading_stalled = 0x53,
+ vgt_perf_cm_stalled_by_gsfetch_done = 0x54,
+ vgt_perf_gog_vs_tbl_stalled = 0x55,
+ vgt_perf_gog_out_indx_stalled = 0x56,
+ vgt_perf_gog_out_prim_stalled = 0x57,
+ vgt_perf_waveid_stalled = 0x58,
+ vgt_perf_gog_busy = 0x59,
+ vgt_perf_reused_vs_indices = 0x5a,
+ vgt_perf_sclk_reg_vld_event = 0x5b,
+ vgt_perf_vs_conflicting_indices = 0x5c,
+ vgt_perf_sclk_core_vld_event = 0x5d,
+ vgt_perf_hswave_stalled = 0x5e,
+ vgt_perf_sclk_gs_vld_event = 0x5f,
+ vgt_perf_VGT_SPI_LSVERT_VALID = 0x60,
+ vgt_perf_VGT_SPI_LSVERT_EOV = 0x61,
+ vgt_perf_VGT_SPI_LSVERT_STALLED = 0x62,
+ vgt_perf_VGT_SPI_LSVERT_STARVED_BUSY = 0x63,
+ vgt_perf_VGT_SPI_LSVERT_STARVED_IDLE = 0x64,
+ vgt_perf_VGT_SPI_LSVERT_STATIC = 0x65,
+ vgt_perf_VGT_SPI_LSWAVE_EVENT_WINDOW_ACTIVE = 0x66,
+ vgt_perf_VGT_SPI_LSWAVE_IS_EVENT = 0x67,
+ vgt_perf_VGT_SPI_LSWAVE_SEND = 0x68,
+ vgt_perf_VGT_SPI_HSVERT_VALID = 0x69,
+ vgt_perf_VGT_SPI_HSVERT_EOV = 0x6a,
+ vgt_perf_VGT_SPI_HSVERT_STALLED = 0x6b,
+ vgt_perf_VGT_SPI_HSVERT_STARVED_BUSY = 0x6c,
+ vgt_perf_VGT_SPI_HSVERT_STARVED_IDLE = 0x6d,
+ vgt_perf_VGT_SPI_HSVERT_STATIC = 0x6e,
+ vgt_perf_VGT_SPI_HSWAVE_EVENT_WINDOW_ACTIVE = 0x6f,
+ vgt_perf_VGT_SPI_HSWAVE_IS_EVENT = 0x70,
+ vgt_perf_VGT_SPI_HSWAVE_SEND = 0x71,
+ vgt_perf_ds_prims = 0x72,
+ vgt_perf_ls_thread_groups = 0x73,
+ vgt_perf_hs_thread_groups = 0x74,
+ vgt_perf_es_thread_groups = 0x75,
+ vgt_perf_vs_thread_groups = 0x76,
+ vgt_perf_ls_done_latency = 0x77,
+ vgt_perf_hs_done_latency = 0x78,
+ vgt_perf_es_done_latency = 0x79,
+ vgt_perf_gs_done_latency = 0x7a,
+ vgt_perf_vgt_hs_busy = 0x7b,
+ vgt_perf_vgt_te11_busy = 0x7c,
+ vgt_perf_ls_flush = 0x7d,
+ vgt_perf_hs_flush = 0x7e,
+ vgt_perf_es_flush = 0x7f,
+ vgt_perf_vgt_pa_clipp_eopg = 0x80,
+ vgt_perf_ls_done = 0x81,
+ vgt_perf_hs_done = 0x82,
+ vgt_perf_es_done = 0x83,
+ vgt_perf_gs_done = 0x84,
+ vgt_perf_vsfetch_done = 0x85,
+ vgt_perf_gs_done_received = 0x86,
+ vgt_perf_es_ring_high_water_mark = 0x87,
+ vgt_perf_gs_ring_high_water_mark = 0x88,
+ vgt_perf_vs_table_high_water_mark = 0x89,
+ vgt_perf_hs_tgs_active_high_water_mark = 0x8a,
+ vgt_perf_pa_clipp_dealloc = 0x8b,
+ vgt_perf_cut_mem_flush_stalled = 0x8c,
+ vgt_perf_vsvert_work_received = 0x8d,
+ vgt_perf_vgt_pa_clipp_starved_after_work = 0x8e,
+ vgt_perf_te11_con_starved_after_work = 0x8f,
+ vgt_perf_hs_waiting_on_ls_done_stall = 0x90,
+ vgt_spi_vsvert_valid = 0x91,
+} VGT_PERFCOUNT_SELECT;
+typedef enum IA_PERFCOUNT_SELECT {
+ ia_perf_GRP_INPUT_EVENT_WINDOW_ACTIVE = 0x0,
+ ia_perf_dma_data_fifo_full = 0x1,
+ ia_perf_RESERVED1 = 0x2,
+ ia_perf_RESERVED2 = 0x3,
+ ia_perf_RESERVED3 = 0x4,
+ ia_perf_RESERVED4 = 0x5,
+ ia_perf_RESERVED5 = 0x6,
+ ia_perf_MC_LAT_BIN_0 = 0x7,
+ ia_perf_MC_LAT_BIN_1 = 0x8,
+ ia_perf_MC_LAT_BIN_2 = 0x9,
+ ia_perf_MC_LAT_BIN_3 = 0xa,
+ ia_perf_MC_LAT_BIN_4 = 0xb,
+ ia_perf_MC_LAT_BIN_5 = 0xc,
+ ia_perf_MC_LAT_BIN_6 = 0xd,
+ ia_perf_MC_LAT_BIN_7 = 0xe,
+ ia_perf_ia_busy = 0xf,
+ ia_perf_ia_sclk_reg_vld_event = 0x10,
+ ia_perf_RESERVED6 = 0x11,
+ ia_perf_ia_sclk_core_vld_event = 0x12,
+ ia_perf_RESERVED7 = 0x13,
+ ia_perf_ia_dma_return = 0x14,
+ ia_perf_ia_stalled = 0x15,
+ ia_perf_shift_starved_pipe0_event = 0x16,
+ ia_perf_shift_starved_pipe1_event = 0x17,
+} IA_PERFCOUNT_SELECT;
+typedef enum WD_PERFCOUNT_SELECT {
+ wd_perf_RBIU_FIFOS_EVENT_WINDOW_ACTIVE = 0x0,
+ wd_perf_RBIU_DR_FIFO_STARVED = 0x1,
+ wd_perf_RBIU_DR_FIFO_STALLED = 0x2,
+ wd_perf_RBIU_DI_FIFO_STARVED = 0x3,
+ wd_perf_RBIU_DI_FIFO_STALLED = 0x4,
+ wd_perf_wd_busy = 0x5,
+ wd_perf_wd_sclk_reg_vld_event = 0x6,
+ wd_perf_wd_sclk_input_vld_event = 0x7,
+ wd_perf_wd_sclk_core_vld_event = 0x8,
+ wd_perf_wd_stalled = 0x9,
+ wd_perf_inside_tf_bin_0 = 0xa,
+ wd_perf_inside_tf_bin_1 = 0xb,
+ wd_perf_inside_tf_bin_2 = 0xc,
+ wd_perf_inside_tf_bin_3 = 0xd,
+ wd_perf_inside_tf_bin_4 = 0xe,
+ wd_perf_inside_tf_bin_5 = 0xf,
+ wd_perf_inside_tf_bin_6 = 0x10,
+ wd_perf_inside_tf_bin_7 = 0x11,
+ wd_perf_inside_tf_bin_8 = 0x12,
+ wd_perf_tfreq_lat_bin_0 = 0x13,
+ wd_perf_tfreq_lat_bin_1 = 0x14,
+ wd_perf_tfreq_lat_bin_2 = 0x15,
+ wd_perf_tfreq_lat_bin_3 = 0x16,
+ wd_perf_tfreq_lat_bin_4 = 0x17,
+ wd_perf_tfreq_lat_bin_5 = 0x18,
+ wd_perf_tfreq_lat_bin_6 = 0x19,
+ wd_perf_tfreq_lat_bin_7 = 0x1a,
+ wd_starved_on_hs_done = 0x1b,
+ wd_perf_se0_hs_done_latency = 0x1c,
+ wd_perf_se1_hs_done_latency = 0x1d,
+ wd_perf_se2_hs_done_latency = 0x1e,
+ wd_perf_se3_hs_done_latency = 0x1f,
+ wd_perf_hs_done_se0 = 0x20,
+ wd_perf_hs_done_se1 = 0x21,
+ wd_perf_hs_done_se2 = 0x22,
+ wd_perf_hs_done_se3 = 0x23,
+ wd_perf_null_patches = 0x24,
+} WD_PERFCOUNT_SELECT;
+typedef enum WD_IA_DRAW_TYPE {
+ WD_IA_DRAW_TYPE_DI_MM0 = 0x0,
+ WD_IA_DRAW_TYPE_DI_MM1 = 0x1,
+ WD_IA_DRAW_TYPE_EVENT_INIT = 0x2,
+ WD_IA_DRAW_TYPE_EVENT_ADDR = 0x3,
+ WD_IA_DRAW_TYPE_MIN_INDX = 0x4,
+ WD_IA_DRAW_TYPE_MAX_INDX = 0x5,
+ WD_IA_DRAW_TYPE_INDX_OFF = 0x6,
+ WD_IA_DRAW_TYPE_IMM_DATA = 0x7,
+} WD_IA_DRAW_TYPE;
+typedef enum WD_IA_DRAW_SOURCE {
+ WD_IA_DRAW_SOURCE_DMA = 0x0,
+ WD_IA_DRAW_SOURCE_IMMD = 0x1,
+ WD_IA_DRAW_SOURCE_AUTO = 0x2,
+ WD_IA_DRAW_SOURCE_OPAQ = 0x3,
+} WD_IA_DRAW_SOURCE;
+#define GSTHREADID_SIZE 0x2
+typedef enum DebugBlockId {
+ DBG_BLOCK_ID_RESERVED = 0x0,
+ DBG_BLOCK_ID_DBG = 0x1,
+ DBG_BLOCK_ID_VMC = 0x2,
+ DBG_BLOCK_ID_PDMA = 0x3,
+ DBG_BLOCK_ID_CG = 0x4,
+ DBG_BLOCK_ID_SRBM = 0x5,
+ DBG_BLOCK_ID_GRBM = 0x6,
+ DBG_BLOCK_ID_RLC = 0x7,
+ DBG_BLOCK_ID_CSC = 0x8,
+ DBG_BLOCK_ID_SEM = 0x9,
+ DBG_BLOCK_ID_IH = 0xa,
+ DBG_BLOCK_ID_SC = 0xb,
+ DBG_BLOCK_ID_SQ = 0xc,
+ DBG_BLOCK_ID_UVDU = 0xd,
+ DBG_BLOCK_ID_SQA = 0xe,
+ DBG_BLOCK_ID_SDMA0 = 0xf,
+ DBG_BLOCK_ID_SDMA1 = 0x10,
+ DBG_BLOCK_ID_SPIM = 0x11,
+ DBG_BLOCK_ID_GDS = 0x12,
+ DBG_BLOCK_ID_VC0 = 0x13,
+ DBG_BLOCK_ID_VC1 = 0x14,
+ DBG_BLOCK_ID_PA0 = 0x15,
+ DBG_BLOCK_ID_PA1 = 0x16,
+ DBG_BLOCK_ID_CP0 = 0x17,
+ DBG_BLOCK_ID_CP1 = 0x18,
+ DBG_BLOCK_ID_CP2 = 0x19,
+ DBG_BLOCK_ID_XBR = 0x1a,
+ DBG_BLOCK_ID_UVDM = 0x1b,
+ DBG_BLOCK_ID_VGT0 = 0x1c,
+ DBG_BLOCK_ID_VGT1 = 0x1d,
+ DBG_BLOCK_ID_IA = 0x1e,
+ DBG_BLOCK_ID_SXM0 = 0x1f,
+ DBG_BLOCK_ID_SXM1 = 0x20,
+ DBG_BLOCK_ID_SCT0 = 0x21,
+ DBG_BLOCK_ID_SCT1 = 0x22,
+ DBG_BLOCK_ID_SPM0 = 0x23,
+ DBG_BLOCK_ID_SPM1 = 0x24,
+ DBG_BLOCK_ID_UNUSED0 = 0x25,
+ DBG_BLOCK_ID_UNUSED1 = 0x26,
+ DBG_BLOCK_ID_TCAA = 0x27,
+ DBG_BLOCK_ID_TCAB = 0x28,
+ DBG_BLOCK_ID_TCCA = 0x29,
+ DBG_BLOCK_ID_TCCB = 0x2a,
+ DBG_BLOCK_ID_MCC0 = 0x2b,
+ DBG_BLOCK_ID_MCC1 = 0x2c,
+ DBG_BLOCK_ID_MCC2 = 0x2d,
+ DBG_BLOCK_ID_MCC3 = 0x2e,
+ DBG_BLOCK_ID_SXS0 = 0x2f,
+ DBG_BLOCK_ID_SXS1 = 0x30,
+ DBG_BLOCK_ID_SXS2 = 0x31,
+ DBG_BLOCK_ID_SXS3 = 0x32,
+ DBG_BLOCK_ID_SXS4 = 0x33,
+ DBG_BLOCK_ID_SXS5 = 0x34,
+ DBG_BLOCK_ID_SXS6 = 0x35,
+ DBG_BLOCK_ID_SXS7 = 0x36,
+ DBG_BLOCK_ID_SXS8 = 0x37,
+ DBG_BLOCK_ID_SXS9 = 0x38,
+ DBG_BLOCK_ID_BCI0 = 0x39,
+ DBG_BLOCK_ID_BCI1 = 0x3a,
+ DBG_BLOCK_ID_BCI2 = 0x3b,
+ DBG_BLOCK_ID_BCI3 = 0x3c,
+ DBG_BLOCK_ID_MCB = 0x3d,
+ DBG_BLOCK_ID_UNUSED6 = 0x3e,
+ DBG_BLOCK_ID_SQA00 = 0x3f,
+ DBG_BLOCK_ID_SQA01 = 0x40,
+ DBG_BLOCK_ID_SQA02 = 0x41,
+ DBG_BLOCK_ID_SQA10 = 0x42,
+ DBG_BLOCK_ID_SQA11 = 0x43,
+ DBG_BLOCK_ID_SQA12 = 0x44,
+ DBG_BLOCK_ID_UNUSED7 = 0x45,
+ DBG_BLOCK_ID_UNUSED8 = 0x46,
+ DBG_BLOCK_ID_SQB00 = 0x47,
+ DBG_BLOCK_ID_SQB01 = 0x48,
+ DBG_BLOCK_ID_SQB10 = 0x49,
+ DBG_BLOCK_ID_SQB11 = 0x4a,
+ DBG_BLOCK_ID_SQ00 = 0x4b,
+ DBG_BLOCK_ID_SQ01 = 0x4c,
+ DBG_BLOCK_ID_SQ10 = 0x4d,
+ DBG_BLOCK_ID_SQ11 = 0x4e,
+ DBG_BLOCK_ID_CB00 = 0x4f,
+ DBG_BLOCK_ID_CB01 = 0x50,
+ DBG_BLOCK_ID_CB02 = 0x51,
+ DBG_BLOCK_ID_CB03 = 0x52,
+ DBG_BLOCK_ID_CB04 = 0x53,
+ DBG_BLOCK_ID_UNUSED9 = 0x54,
+ DBG_BLOCK_ID_UNUSED10 = 0x55,
+ DBG_BLOCK_ID_UNUSED11 = 0x56,
+ DBG_BLOCK_ID_CB10 = 0x57,
+ DBG_BLOCK_ID_CB11 = 0x58,
+ DBG_BLOCK_ID_CB12 = 0x59,
+ DBG_BLOCK_ID_CB13 = 0x5a,
+ DBG_BLOCK_ID_CB14 = 0x5b,
+ DBG_BLOCK_ID_UNUSED12 = 0x5c,
+ DBG_BLOCK_ID_UNUSED13 = 0x5d,
+ DBG_BLOCK_ID_UNUSED14 = 0x5e,
+ DBG_BLOCK_ID_TCP0 = 0x5f,
+ DBG_BLOCK_ID_TCP1 = 0x60,
+ DBG_BLOCK_ID_TCP2 = 0x61,
+ DBG_BLOCK_ID_TCP3 = 0x62,
+ DBG_BLOCK_ID_TCP4 = 0x63,
+ DBG_BLOCK_ID_TCP5 = 0x64,
+ DBG_BLOCK_ID_TCP6 = 0x65,
+ DBG_BLOCK_ID_TCP7 = 0x66,
+ DBG_BLOCK_ID_TCP8 = 0x67,
+ DBG_BLOCK_ID_TCP9 = 0x68,
+ DBG_BLOCK_ID_TCP10 = 0x69,
+ DBG_BLOCK_ID_TCP11 = 0x6a,
+ DBG_BLOCK_ID_TCP12 = 0x6b,
+ DBG_BLOCK_ID_TCP13 = 0x6c,
+ DBG_BLOCK_ID_TCP14 = 0x6d,
+ DBG_BLOCK_ID_TCP15 = 0x6e,
+ DBG_BLOCK_ID_TCP16 = 0x6f,
+ DBG_BLOCK_ID_TCP17 = 0x70,
+ DBG_BLOCK_ID_TCP18 = 0x71,
+ DBG_BLOCK_ID_TCP19 = 0x72,
+ DBG_BLOCK_ID_TCP20 = 0x73,
+ DBG_BLOCK_ID_TCP21 = 0x74,
+ DBG_BLOCK_ID_TCP22 = 0x75,
+ DBG_BLOCK_ID_TCP23 = 0x76,
+ DBG_BLOCK_ID_TCP_RESERVED0 = 0x77,
+ DBG_BLOCK_ID_TCP_RESERVED1 = 0x78,
+ DBG_BLOCK_ID_TCP_RESERVED2 = 0x79,
+ DBG_BLOCK_ID_TCP_RESERVED3 = 0x7a,
+ DBG_BLOCK_ID_TCP_RESERVED4 = 0x7b,
+ DBG_BLOCK_ID_TCP_RESERVED5 = 0x7c,
+ DBG_BLOCK_ID_TCP_RESERVED6 = 0x7d,
+ DBG_BLOCK_ID_TCP_RESERVED7 = 0x7e,
+ DBG_BLOCK_ID_DB00 = 0x7f,
+ DBG_BLOCK_ID_DB01 = 0x80,
+ DBG_BLOCK_ID_DB02 = 0x81,
+ DBG_BLOCK_ID_DB03 = 0x82,
+ DBG_BLOCK_ID_DB04 = 0x83,
+ DBG_BLOCK_ID_UNUSED15 = 0x84,
+ DBG_BLOCK_ID_UNUSED16 = 0x85,
+ DBG_BLOCK_ID_UNUSED17 = 0x86,
+ DBG_BLOCK_ID_DB10 = 0x87,
+ DBG_BLOCK_ID_DB11 = 0x88,
+ DBG_BLOCK_ID_DB12 = 0x89,
+ DBG_BLOCK_ID_DB13 = 0x8a,
+ DBG_BLOCK_ID_DB14 = 0x8b,
+ DBG_BLOCK_ID_UNUSED18 = 0x8c,
+ DBG_BLOCK_ID_UNUSED19 = 0x8d,
+ DBG_BLOCK_ID_UNUSED20 = 0x8e,
+ DBG_BLOCK_ID_TCC0 = 0x8f,
+ DBG_BLOCK_ID_TCC1 = 0x90,
+ DBG_BLOCK_ID_TCC2 = 0x91,
+ DBG_BLOCK_ID_TCC3 = 0x92,
+ DBG_BLOCK_ID_TCC4 = 0x93,
+ DBG_BLOCK_ID_TCC5 = 0x94,
+ DBG_BLOCK_ID_TCC6 = 0x95,
+ DBG_BLOCK_ID_TCC7 = 0x96,
+ DBG_BLOCK_ID_SPS00 = 0x97,
+ DBG_BLOCK_ID_SPS01 = 0x98,
+ DBG_BLOCK_ID_SPS02 = 0x99,
+ DBG_BLOCK_ID_SPS10 = 0x9a,
+ DBG_BLOCK_ID_SPS11 = 0x9b,
+ DBG_BLOCK_ID_SPS12 = 0x9c,
+ DBG_BLOCK_ID_UNUSED21 = 0x9d,
+ DBG_BLOCK_ID_UNUSED22 = 0x9e,
+ DBG_BLOCK_ID_TA00 = 0x9f,
+ DBG_BLOCK_ID_TA01 = 0xa0,
+ DBG_BLOCK_ID_TA02 = 0xa1,
+ DBG_BLOCK_ID_TA03 = 0xa2,
+ DBG_BLOCK_ID_TA04 = 0xa3,
+ DBG_BLOCK_ID_TA05 = 0xa4,
+ DBG_BLOCK_ID_TA06 = 0xa5,
+ DBG_BLOCK_ID_TA07 = 0xa6,
+ DBG_BLOCK_ID_TA08 = 0xa7,
+ DBG_BLOCK_ID_TA09 = 0xa8,
+ DBG_BLOCK_ID_TA0A = 0xa9,
+ DBG_BLOCK_ID_TA0B = 0xaa,
+ DBG_BLOCK_ID_UNUSED23 = 0xab,
+ DBG_BLOCK_ID_UNUSED24 = 0xac,
+ DBG_BLOCK_ID_UNUSED25 = 0xad,
+ DBG_BLOCK_ID_UNUSED26 = 0xae,
+ DBG_BLOCK_ID_TA10 = 0xaf,
+ DBG_BLOCK_ID_TA11 = 0xb0,
+ DBG_BLOCK_ID_TA12 = 0xb1,
+ DBG_BLOCK_ID_TA13 = 0xb2,
+ DBG_BLOCK_ID_TA14 = 0xb3,
+ DBG_BLOCK_ID_TA15 = 0xb4,
+ DBG_BLOCK_ID_TA16 = 0xb5,
+ DBG_BLOCK_ID_TA17 = 0xb6,
+ DBG_BLOCK_ID_TA18 = 0xb7,
+ DBG_BLOCK_ID_TA19 = 0xb8,
+ DBG_BLOCK_ID_TA1A = 0xb9,
+ DBG_BLOCK_ID_TA1B = 0xba,
+ DBG_BLOCK_ID_UNUSED27 = 0xbb,
+ DBG_BLOCK_ID_UNUSED28 = 0xbc,
+ DBG_BLOCK_ID_UNUSED29 = 0xbd,
+ DBG_BLOCK_ID_UNUSED30 = 0xbe,
+ DBG_BLOCK_ID_TD00 = 0xbf,
+ DBG_BLOCK_ID_TD01 = 0xc0,
+ DBG_BLOCK_ID_TD02 = 0xc1,
+ DBG_BLOCK_ID_TD03 = 0xc2,
+ DBG_BLOCK_ID_TD04 = 0xc3,
+ DBG_BLOCK_ID_TD05 = 0xc4,
+ DBG_BLOCK_ID_TD06 = 0xc5,
+ DBG_BLOCK_ID_TD07 = 0xc6,
+ DBG_BLOCK_ID_TD08 = 0xc7,
+ DBG_BLOCK_ID_TD09 = 0xc8,
+ DBG_BLOCK_ID_TD0A = 0xc9,
+ DBG_BLOCK_ID_TD0B = 0xca,
+ DBG_BLOCK_ID_UNUSED31 = 0xcb,
+ DBG_BLOCK_ID_UNUSED32 = 0xcc,
+ DBG_BLOCK_ID_UNUSED33 = 0xcd,
+ DBG_BLOCK_ID_UNUSED34 = 0xce,
+ DBG_BLOCK_ID_TD10 = 0xcf,
+ DBG_BLOCK_ID_TD11 = 0xd0,
+ DBG_BLOCK_ID_TD12 = 0xd1,
+ DBG_BLOCK_ID_TD13 = 0xd2,
+ DBG_BLOCK_ID_TD14 = 0xd3,
+ DBG_BLOCK_ID_TD15 = 0xd4,
+ DBG_BLOCK_ID_TD16 = 0xd5,
+ DBG_BLOCK_ID_TD17 = 0xd6,
+ DBG_BLOCK_ID_TD18 = 0xd7,
+ DBG_BLOCK_ID_TD19 = 0xd8,
+ DBG_BLOCK_ID_TD1A = 0xd9,
+ DBG_BLOCK_ID_TD1B = 0xda,
+ DBG_BLOCK_ID_UNUSED35 = 0xdb,
+ DBG_BLOCK_ID_UNUSED36 = 0xdc,
+ DBG_BLOCK_ID_UNUSED37 = 0xdd,
+ DBG_BLOCK_ID_UNUSED38 = 0xde,
+ DBG_BLOCK_ID_LDS00 = 0xdf,
+ DBG_BLOCK_ID_LDS01 = 0xe0,
+ DBG_BLOCK_ID_LDS02 = 0xe1,
+ DBG_BLOCK_ID_LDS03 = 0xe2,
+ DBG_BLOCK_ID_LDS04 = 0xe3,
+ DBG_BLOCK_ID_LDS05 = 0xe4,
+ DBG_BLOCK_ID_LDS06 = 0xe5,
+ DBG_BLOCK_ID_LDS07 = 0xe6,
+ DBG_BLOCK_ID_LDS08 = 0xe7,
+ DBG_BLOCK_ID_LDS09 = 0xe8,
+ DBG_BLOCK_ID_LDS0A = 0xe9,
+ DBG_BLOCK_ID_LDS0B = 0xea,
+ DBG_BLOCK_ID_UNUSED39 = 0xeb,
+ DBG_BLOCK_ID_UNUSED40 = 0xec,
+ DBG_BLOCK_ID_UNUSED41 = 0xed,
+ DBG_BLOCK_ID_UNUSED42 = 0xee,
+ DBG_BLOCK_ID_LDS10 = 0xef,
+ DBG_BLOCK_ID_LDS11 = 0xf0,
+ DBG_BLOCK_ID_LDS12 = 0xf1,
+ DBG_BLOCK_ID_LDS13 = 0xf2,
+ DBG_BLOCK_ID_LDS14 = 0xf3,
+ DBG_BLOCK_ID_LDS15 = 0xf4,
+ DBG_BLOCK_ID_LDS16 = 0xf5,
+ DBG_BLOCK_ID_LDS17 = 0xf6,
+ DBG_BLOCK_ID_LDS18 = 0xf7,
+ DBG_BLOCK_ID_LDS19 = 0xf8,
+ DBG_BLOCK_ID_LDS1A = 0xf9,
+ DBG_BLOCK_ID_LDS1B = 0xfa,
+ DBG_BLOCK_ID_UNUSED43 = 0xfb,
+ DBG_BLOCK_ID_UNUSED44 = 0xfc,
+ DBG_BLOCK_ID_UNUSED45 = 0xfd,
+ DBG_BLOCK_ID_UNUSED46 = 0xfe,
+} DebugBlockId;
+typedef enum DebugBlockId_BY2 {
+ DBG_BLOCK_ID_RESERVED_BY2 = 0x0,
+ DBG_BLOCK_ID_VMC_BY2 = 0x1,
+ DBG_BLOCK_ID_UNUSED0_BY2 = 0x2,
+ DBG_BLOCK_ID_GRBM_BY2 = 0x3,
+ DBG_BLOCK_ID_CSC_BY2 = 0x4,
+ DBG_BLOCK_ID_IH_BY2 = 0x5,
+ DBG_BLOCK_ID_SQ_BY2 = 0x6,
+ DBG_BLOCK_ID_UVD_BY2 = 0x7,
+ DBG_BLOCK_ID_SDMA0_BY2 = 0x8,
+ DBG_BLOCK_ID_SPIM_BY2 = 0x9,
+ DBG_BLOCK_ID_VC0_BY2 = 0xa,
+ DBG_BLOCK_ID_PA_BY2 = 0xb,
+ DBG_BLOCK_ID_CP0_BY2 = 0xc,
+ DBG_BLOCK_ID_CP2_BY2 = 0xd,
+ DBG_BLOCK_ID_PC0_BY2 = 0xe,
+ DBG_BLOCK_ID_BCI0_BY2 = 0xf,
+ DBG_BLOCK_ID_SXM0_BY2 = 0x10,
+ DBG_BLOCK_ID_SCT0_BY2 = 0x11,
+ DBG_BLOCK_ID_SPM0_BY2 = 0x12,
+ DBG_BLOCK_ID_BCI2_BY2 = 0x13,
+ DBG_BLOCK_ID_TCA_BY2 = 0x14,
+ DBG_BLOCK_ID_TCCA_BY2 = 0x15,
+ DBG_BLOCK_ID_MCC_BY2 = 0x16,
+ DBG_BLOCK_ID_MCC2_BY2 = 0x17,
+ DBG_BLOCK_ID_MCD_BY2 = 0x18,
+ DBG_BLOCK_ID_MCD2_BY2 = 0x19,
+ DBG_BLOCK_ID_MCD4_BY2 = 0x1a,
+ DBG_BLOCK_ID_MCB_BY2 = 0x1b,
+ DBG_BLOCK_ID_SQA_BY2 = 0x1c,
+ DBG_BLOCK_ID_SQA02_BY2 = 0x1d,
+ DBG_BLOCK_ID_SQA11_BY2 = 0x1e,
+ DBG_BLOCK_ID_UNUSED8_BY2 = 0x1f,
+ DBG_BLOCK_ID_SQB_BY2 = 0x20,
+ DBG_BLOCK_ID_SQB10_BY2 = 0x21,
+ DBG_BLOCK_ID_UNUSED10_BY2 = 0x22,
+ DBG_BLOCK_ID_UNUSED12_BY2 = 0x23,
+ DBG_BLOCK_ID_CB_BY2 = 0x24,
+ DBG_BLOCK_ID_CB02_BY2 = 0x25,
+ DBG_BLOCK_ID_CB10_BY2 = 0x26,
+ DBG_BLOCK_ID_CB12_BY2 = 0x27,
+ DBG_BLOCK_ID_SXS_BY2 = 0x28,
+ DBG_BLOCK_ID_SXS2_BY2 = 0x29,
+ DBG_BLOCK_ID_SXS4_BY2 = 0x2a,
+ DBG_BLOCK_ID_SXS6_BY2 = 0x2b,
+ DBG_BLOCK_ID_DB_BY2 = 0x2c,
+ DBG_BLOCK_ID_DB02_BY2 = 0x2d,
+ DBG_BLOCK_ID_DB10_BY2 = 0x2e,
+ DBG_BLOCK_ID_DB12_BY2 = 0x2f,
+ DBG_BLOCK_ID_TCP_BY2 = 0x30,
+ DBG_BLOCK_ID_TCP2_BY2 = 0x31,
+ DBG_BLOCK_ID_TCP4_BY2 = 0x32,
+ DBG_BLOCK_ID_TCP6_BY2 = 0x33,
+ DBG_BLOCK_ID_TCP8_BY2 = 0x34,
+ DBG_BLOCK_ID_TCP10_BY2 = 0x35,
+ DBG_BLOCK_ID_TCP12_BY2 = 0x36,
+ DBG_BLOCK_ID_TCP14_BY2 = 0x37,
+ DBG_BLOCK_ID_TCP16_BY2 = 0x38,
+ DBG_BLOCK_ID_TCP18_BY2 = 0x39,
+ DBG_BLOCK_ID_TCP20_BY2 = 0x3a,
+ DBG_BLOCK_ID_TCP22_BY2 = 0x3b,
+ DBG_BLOCK_ID_TCP_RESERVED0_BY2 = 0x3c,
+ DBG_BLOCK_ID_TCP_RESERVED2_BY2 = 0x3d,
+ DBG_BLOCK_ID_TCP_RESERVED4_BY2 = 0x3e,
+ DBG_BLOCK_ID_TCP_RESERVED6_BY2 = 0x3f,
+ DBG_BLOCK_ID_TCC_BY2 = 0x40,
+ DBG_BLOCK_ID_TCC2_BY2 = 0x41,
+ DBG_BLOCK_ID_TCC4_BY2 = 0x42,
+ DBG_BLOCK_ID_TCC6_BY2 = 0x43,
+ DBG_BLOCK_ID_SPS_BY2 = 0x44,
+ DBG_BLOCK_ID_SPS02_BY2 = 0x45,
+ DBG_BLOCK_ID_SPS11_BY2 = 0x46,
+ DBG_BLOCK_ID_UNUSED14_BY2 = 0x47,
+ DBG_BLOCK_ID_TA_BY2 = 0x48,
+ DBG_BLOCK_ID_TA02_BY2 = 0x49,
+ DBG_BLOCK_ID_TA04_BY2 = 0x4a,
+ DBG_BLOCK_ID_TA06_BY2 = 0x4b,
+ DBG_BLOCK_ID_TA08_BY2 = 0x4c,
+ DBG_BLOCK_ID_TA0A_BY2 = 0x4d,
+ DBG_BLOCK_ID_UNUSED20_BY2 = 0x4e,
+ DBG_BLOCK_ID_UNUSED22_BY2 = 0x4f,
+ DBG_BLOCK_ID_TA10_BY2 = 0x50,
+ DBG_BLOCK_ID_TA12_BY2 = 0x51,
+ DBG_BLOCK_ID_TA14_BY2 = 0x52,
+ DBG_BLOCK_ID_TA16_BY2 = 0x53,
+ DBG_BLOCK_ID_TA18_BY2 = 0x54,
+ DBG_BLOCK_ID_TA1A_BY2 = 0x55,
+ DBG_BLOCK_ID_UNUSED24_BY2 = 0x56,
+ DBG_BLOCK_ID_UNUSED26_BY2 = 0x57,
+ DBG_BLOCK_ID_TD_BY2 = 0x58,
+ DBG_BLOCK_ID_TD02_BY2 = 0x59,
+ DBG_BLOCK_ID_TD04_BY2 = 0x5a,
+ DBG_BLOCK_ID_TD06_BY2 = 0x5b,
+ DBG_BLOCK_ID_TD08_BY2 = 0x5c,
+ DBG_BLOCK_ID_TD0A_BY2 = 0x5d,
+ DBG_BLOCK_ID_UNUSED28_BY2 = 0x5e,
+ DBG_BLOCK_ID_UNUSED30_BY2 = 0x5f,
+ DBG_BLOCK_ID_TD10_BY2 = 0x60,
+ DBG_BLOCK_ID_TD12_BY2 = 0x61,
+ DBG_BLOCK_ID_TD14_BY2 = 0x62,
+ DBG_BLOCK_ID_TD16_BY2 = 0x63,
+ DBG_BLOCK_ID_TD18_BY2 = 0x64,
+ DBG_BLOCK_ID_TD1A_BY2 = 0x65,
+ DBG_BLOCK_ID_UNUSED32_BY2 = 0x66,
+ DBG_BLOCK_ID_UNUSED34_BY2 = 0x67,
+ DBG_BLOCK_ID_LDS_BY2 = 0x68,
+ DBG_BLOCK_ID_LDS02_BY2 = 0x69,
+ DBG_BLOCK_ID_LDS04_BY2 = 0x6a,
+ DBG_BLOCK_ID_LDS06_BY2 = 0x6b,
+ DBG_BLOCK_ID_LDS08_BY2 = 0x6c,
+ DBG_BLOCK_ID_LDS0A_BY2 = 0x6d,
+ DBG_BLOCK_ID_UNUSED36_BY2 = 0x6e,
+ DBG_BLOCK_ID_UNUSED38_BY2 = 0x6f,
+ DBG_BLOCK_ID_LDS10_BY2 = 0x70,
+ DBG_BLOCK_ID_LDS12_BY2 = 0x71,
+ DBG_BLOCK_ID_LDS14_BY2 = 0x72,
+ DBG_BLOCK_ID_LDS16_BY2 = 0x73,
+ DBG_BLOCK_ID_LDS18_BY2 = 0x74,
+ DBG_BLOCK_ID_LDS1A_BY2 = 0x75,
+ DBG_BLOCK_ID_UNUSED40_BY2 = 0x76,
+ DBG_BLOCK_ID_UNUSED42_BY2 = 0x77,
+} DebugBlockId_BY2;
+typedef enum DebugBlockId_BY4 {
+ DBG_BLOCK_ID_RESERVED_BY4 = 0x0,
+ DBG_BLOCK_ID_UNUSED0_BY4 = 0x1,
+ DBG_BLOCK_ID_CSC_BY4 = 0x2,
+ DBG_BLOCK_ID_SQ_BY4 = 0x3,
+ DBG_BLOCK_ID_SDMA0_BY4 = 0x4,
+ DBG_BLOCK_ID_VC0_BY4 = 0x5,
+ DBG_BLOCK_ID_CP0_BY4 = 0x6,
+ DBG_BLOCK_ID_UNUSED1_BY4 = 0x7,
+ DBG_BLOCK_ID_SXM0_BY4 = 0x8,
+ DBG_BLOCK_ID_SPM0_BY4 = 0x9,
+ DBG_BLOCK_ID_TCAA_BY4 = 0xa,
+ DBG_BLOCK_ID_MCC_BY4 = 0xb,
+ DBG_BLOCK_ID_MCD_BY4 = 0xc,
+ DBG_BLOCK_ID_MCD4_BY4 = 0xd,
+ DBG_BLOCK_ID_SQA_BY4 = 0xe,
+ DBG_BLOCK_ID_SQA11_BY4 = 0xf,
+ DBG_BLOCK_ID_SQB_BY4 = 0x10,
+ DBG_BLOCK_ID_UNUSED10_BY4 = 0x11,
+ DBG_BLOCK_ID_CB_BY4 = 0x12,
+ DBG_BLOCK_ID_CB10_BY4 = 0x13,
+ DBG_BLOCK_ID_SXS_BY4 = 0x14,
+ DBG_BLOCK_ID_SXS4_BY4 = 0x15,
+ DBG_BLOCK_ID_DB_BY4 = 0x16,
+ DBG_BLOCK_ID_DB10_BY4 = 0x17,
+ DBG_BLOCK_ID_TCP_BY4 = 0x18,
+ DBG_BLOCK_ID_TCP4_BY4 = 0x19,
+ DBG_BLOCK_ID_TCP8_BY4 = 0x1a,
+ DBG_BLOCK_ID_TCP12_BY4 = 0x1b,
+ DBG_BLOCK_ID_TCP16_BY4 = 0x1c,
+ DBG_BLOCK_ID_TCP20_BY4 = 0x1d,
+ DBG_BLOCK_ID_TCP_RESERVED0_BY4 = 0x1e,
+ DBG_BLOCK_ID_TCP_RESERVED4_BY4 = 0x1f,
+ DBG_BLOCK_ID_TCC_BY4 = 0x20,
+ DBG_BLOCK_ID_TCC4_BY4 = 0x21,
+ DBG_BLOCK_ID_SPS_BY4 = 0x22,
+ DBG_BLOCK_ID_SPS11_BY4 = 0x23,
+ DBG_BLOCK_ID_TA_BY4 = 0x24,
+ DBG_BLOCK_ID_TA04_BY4 = 0x25,
+ DBG_BLOCK_ID_TA08_BY4 = 0x26,
+ DBG_BLOCK_ID_UNUSED20_BY4 = 0x27,
+ DBG_BLOCK_ID_TA10_BY4 = 0x28,
+ DBG_BLOCK_ID_TA14_BY4 = 0x29,
+ DBG_BLOCK_ID_TA18_BY4 = 0x2a,
+ DBG_BLOCK_ID_UNUSED24_BY4 = 0x2b,
+ DBG_BLOCK_ID_TD_BY4 = 0x2c,
+ DBG_BLOCK_ID_TD04_BY4 = 0x2d,
+ DBG_BLOCK_ID_TD08_BY4 = 0x2e,
+ DBG_BLOCK_ID_UNUSED28_BY4 = 0x2f,
+ DBG_BLOCK_ID_TD10_BY4 = 0x30,
+ DBG_BLOCK_ID_TD14_BY4 = 0x31,
+ DBG_BLOCK_ID_TD18_BY4 = 0x32,
+ DBG_BLOCK_ID_UNUSED32_BY4 = 0x33,
+ DBG_BLOCK_ID_LDS_BY4 = 0x34,
+ DBG_BLOCK_ID_LDS04_BY4 = 0x35,
+ DBG_BLOCK_ID_LDS08_BY4 = 0x36,
+ DBG_BLOCK_ID_UNUSED36_BY4 = 0x37,
+ DBG_BLOCK_ID_LDS10_BY4 = 0x38,
+ DBG_BLOCK_ID_LDS14_BY4 = 0x39,
+ DBG_BLOCK_ID_LDS18_BY4 = 0x3a,
+ DBG_BLOCK_ID_UNUSED40_BY4 = 0x3b,
+} DebugBlockId_BY4;
+typedef enum DebugBlockId_BY8 {
+ DBG_BLOCK_ID_RESERVED_BY8 = 0x0,
+ DBG_BLOCK_ID_CSC_BY8 = 0x1,
+ DBG_BLOCK_ID_SDMA0_BY8 = 0x2,
+ DBG_BLOCK_ID_CP0_BY8 = 0x3,
+ DBG_BLOCK_ID_SXM0_BY8 = 0x4,
+ DBG_BLOCK_ID_TCA_BY8 = 0x5,
+ DBG_BLOCK_ID_MCD_BY8 = 0x6,
+ DBG_BLOCK_ID_SQA_BY8 = 0x7,
+ DBG_BLOCK_ID_SQB_BY8 = 0x8,
+ DBG_BLOCK_ID_CB_BY8 = 0x9,
+ DBG_BLOCK_ID_SXS_BY8 = 0xa,
+ DBG_BLOCK_ID_DB_BY8 = 0xb,
+ DBG_BLOCK_ID_TCP_BY8 = 0xc,
+ DBG_BLOCK_ID_TCP8_BY8 = 0xd,
+ DBG_BLOCK_ID_TCP16_BY8 = 0xe,
+ DBG_BLOCK_ID_TCP_RESERVED0_BY8 = 0xf,
+ DBG_BLOCK_ID_TCC_BY8 = 0x10,
+ DBG_BLOCK_ID_SPS_BY8 = 0x11,
+ DBG_BLOCK_ID_TA_BY8 = 0x12,
+ DBG_BLOCK_ID_TA08_BY8 = 0x13,
+ DBG_BLOCK_ID_TA10_BY8 = 0x14,
+ DBG_BLOCK_ID_TA18_BY8 = 0x15,
+ DBG_BLOCK_ID_TD_BY8 = 0x16,
+ DBG_BLOCK_ID_TD08_BY8 = 0x17,
+ DBG_BLOCK_ID_TD10_BY8 = 0x18,
+ DBG_BLOCK_ID_TD18_BY8 = 0x19,
+ DBG_BLOCK_ID_LDS_BY8 = 0x1a,
+ DBG_BLOCK_ID_LDS08_BY8 = 0x1b,
+ DBG_BLOCK_ID_LDS10_BY8 = 0x1c,
+ DBG_BLOCK_ID_LDS18_BY8 = 0x1d,
+} DebugBlockId_BY8;
+typedef enum DebugBlockId_BY16 {
+ DBG_BLOCK_ID_RESERVED_BY16 = 0x0,
+ DBG_BLOCK_ID_SDMA0_BY16 = 0x1,
+ DBG_BLOCK_ID_SXM_BY16 = 0x2,
+ DBG_BLOCK_ID_MCD_BY16 = 0x3,
+ DBG_BLOCK_ID_SQB_BY16 = 0x4,
+ DBG_BLOCK_ID_SXS_BY16 = 0x5,
+ DBG_BLOCK_ID_TCP_BY16 = 0x6,
+ DBG_BLOCK_ID_TCP16_BY16 = 0x7,
+ DBG_BLOCK_ID_TCC_BY16 = 0x8,
+ DBG_BLOCK_ID_TA_BY16 = 0x9,
+ DBG_BLOCK_ID_TA10_BY16 = 0xa,
+ DBG_BLOCK_ID_TD_BY16 = 0xb,
+ DBG_BLOCK_ID_TD10_BY16 = 0xc,
+ DBG_BLOCK_ID_LDS_BY16 = 0xd,
+ DBG_BLOCK_ID_LDS10_BY16 = 0xe,
+} DebugBlockId_BY16;
+typedef enum SurfaceEndian {
+ ENDIAN_NONE = 0x0,
+ ENDIAN_8IN16 = 0x1,
+ ENDIAN_8IN32 = 0x2,
+ ENDIAN_8IN64 = 0x3,
+} SurfaceEndian;
+typedef enum ArrayMode {
+ ARRAY_LINEAR_GENERAL = 0x0,
+ ARRAY_LINEAR_ALIGNED = 0x1,
+ ARRAY_1D_TILED_THIN1 = 0x2,
+ ARRAY_1D_TILED_THICK = 0x3,
+ ARRAY_2D_TILED_THIN1 = 0x4,
+ ARRAY_PRT_TILED_THIN1 = 0x5,
+ ARRAY_PRT_2D_TILED_THIN1 = 0x6,
+ ARRAY_2D_TILED_THICK = 0x7,
+ ARRAY_2D_TILED_XTHICK = 0x8,
+ ARRAY_PRT_TILED_THICK = 0x9,
+ ARRAY_PRT_2D_TILED_THICK = 0xa,
+ ARRAY_PRT_3D_TILED_THIN1 = 0xb,
+ ARRAY_3D_TILED_THIN1 = 0xc,
+ ARRAY_3D_TILED_THICK = 0xd,
+ ARRAY_3D_TILED_XTHICK = 0xe,
+ ARRAY_PRT_3D_TILED_THICK = 0xf,
+} ArrayMode;
+typedef enum PipeTiling {
+ CONFIG_1_PIPE = 0x0,
+ CONFIG_2_PIPE = 0x1,
+ CONFIG_4_PIPE = 0x2,
+ CONFIG_8_PIPE = 0x3,
+} PipeTiling;
+typedef enum BankTiling {
+ CONFIG_4_BANK = 0x0,
+ CONFIG_8_BANK = 0x1,
+} BankTiling;
+typedef enum GroupInterleave {
+ CONFIG_256B_GROUP = 0x0,
+ CONFIG_512B_GROUP = 0x1,
+} GroupInterleave;
+typedef enum RowTiling {
+ CONFIG_1KB_ROW = 0x0,
+ CONFIG_2KB_ROW = 0x1,
+ CONFIG_4KB_ROW = 0x2,
+ CONFIG_8KB_ROW = 0x3,
+ CONFIG_1KB_ROW_OPT = 0x4,
+ CONFIG_2KB_ROW_OPT = 0x5,
+ CONFIG_4KB_ROW_OPT = 0x6,
+ CONFIG_8KB_ROW_OPT = 0x7,
+} RowTiling;
+typedef enum BankSwapBytes {
+ CONFIG_128B_SWAPS = 0x0,
+ CONFIG_256B_SWAPS = 0x1,
+ CONFIG_512B_SWAPS = 0x2,
+ CONFIG_1KB_SWAPS = 0x3,
+} BankSwapBytes;
+typedef enum SampleSplitBytes {
+ CONFIG_1KB_SPLIT = 0x0,
+ CONFIG_2KB_SPLIT = 0x1,
+ CONFIG_4KB_SPLIT = 0x2,
+ CONFIG_8KB_SPLIT = 0x3,
+} SampleSplitBytes;
+typedef enum NumPipes {
+ ADDR_CONFIG_1_PIPE = 0x0,
+ ADDR_CONFIG_2_PIPE = 0x1,
+ ADDR_CONFIG_4_PIPE = 0x2,
+ ADDR_CONFIG_8_PIPE = 0x3,
+} NumPipes;
+typedef enum PipeInterleaveSize {
+ ADDR_CONFIG_PIPE_INTERLEAVE_256B = 0x0,
+ ADDR_CONFIG_PIPE_INTERLEAVE_512B = 0x1,
+} PipeInterleaveSize;
+typedef enum BankInterleaveSize {
+ ADDR_CONFIG_BANK_INTERLEAVE_1 = 0x0,
+ ADDR_CONFIG_BANK_INTERLEAVE_2 = 0x1,
+ ADDR_CONFIG_BANK_INTERLEAVE_4 = 0x2,
+ ADDR_CONFIG_BANK_INTERLEAVE_8 = 0x3,
+} BankInterleaveSize;
+typedef enum NumShaderEngines {
+ ADDR_CONFIG_1_SHADER_ENGINE = 0x0,
+ ADDR_CONFIG_2_SHADER_ENGINE = 0x1,
+} NumShaderEngines;
+typedef enum ShaderEngineTileSize {
+ ADDR_CONFIG_SE_TILE_16 = 0x0,
+ ADDR_CONFIG_SE_TILE_32 = 0x1,
+} ShaderEngineTileSize;
+typedef enum NumGPUs {
+ ADDR_CONFIG_1_GPU = 0x0,
+ ADDR_CONFIG_2_GPU = 0x1,
+ ADDR_CONFIG_4_GPU = 0x2,
+} NumGPUs;
+typedef enum MultiGPUTileSize {
+ ADDR_CONFIG_GPU_TILE_16 = 0x0,
+ ADDR_CONFIG_GPU_TILE_32 = 0x1,
+ ADDR_CONFIG_GPU_TILE_64 = 0x2,
+ ADDR_CONFIG_GPU_TILE_128 = 0x3,
+} MultiGPUTileSize;
+typedef enum RowSize {
+ ADDR_CONFIG_1KB_ROW = 0x0,
+ ADDR_CONFIG_2KB_ROW = 0x1,
+ ADDR_CONFIG_4KB_ROW = 0x2,
+} RowSize;
+typedef enum NumLowerPipes {
+ ADDR_CONFIG_1_LOWER_PIPES = 0x0,
+ ADDR_CONFIG_2_LOWER_PIPES = 0x1,
+} NumLowerPipes;
+typedef enum ColorTransform {
+ DCC_CT_AUTO = 0x0,
+ DCC_CT_NONE = 0x1,
+ ABGR_TO_A_BG_G_RB = 0x2,
+ BGRA_TO_BG_G_RB_A = 0x3,
+} ColorTransform;
+typedef enum CompareRef {
+ REF_NEVER = 0x0,
+ REF_LESS = 0x1,
+ REF_EQUAL = 0x2,
+ REF_LEQUAL = 0x3,
+ REF_GREATER = 0x4,
+ REF_NOTEQUAL = 0x5,
+ REF_GEQUAL = 0x6,
+ REF_ALWAYS = 0x7,
+} CompareRef;
+typedef enum ReadSize {
+ READ_256_BITS = 0x0,
+ READ_512_BITS = 0x1,
+} ReadSize;
+typedef enum DepthFormat {
+ DEPTH_INVALID = 0x0,
+ DEPTH_16 = 0x1,
+ DEPTH_X8_24 = 0x2,
+ DEPTH_8_24 = 0x3,
+ DEPTH_X8_24_FLOAT = 0x4,
+ DEPTH_8_24_FLOAT = 0x5,
+ DEPTH_32_FLOAT = 0x6,
+ DEPTH_X24_8_32_FLOAT = 0x7,
+} DepthFormat;
+typedef enum ZFormat {
+ Z_INVALID = 0x0,
+ Z_16 = 0x1,
+ Z_24 = 0x2,
+ Z_32_FLOAT = 0x3,
+} ZFormat;
+typedef enum StencilFormat {
+ STENCIL_INVALID = 0x0,
+ STENCIL_8 = 0x1,
+} StencilFormat;
+typedef enum CmaskMode {
+ CMASK_CLEAR_NONE = 0x0,
+ CMASK_CLEAR_ONE = 0x1,
+ CMASK_CLEAR_ALL = 0x2,
+ CMASK_ANY_EXPANDED = 0x3,
+ CMASK_ALPHA0_FRAG1 = 0x4,
+ CMASK_ALPHA0_FRAG2 = 0x5,
+ CMASK_ALPHA0_FRAG4 = 0x6,
+ CMASK_ALPHA0_FRAGS = 0x7,
+ CMASK_ALPHA1_FRAG1 = 0x8,
+ CMASK_ALPHA1_FRAG2 = 0x9,
+ CMASK_ALPHA1_FRAG4 = 0xa,
+ CMASK_ALPHA1_FRAGS = 0xb,
+ CMASK_ALPHAX_FRAG1 = 0xc,
+ CMASK_ALPHAX_FRAG2 = 0xd,
+ CMASK_ALPHAX_FRAG4 = 0xe,
+ CMASK_ALPHAX_FRAGS = 0xf,
+} CmaskMode;
+typedef enum QuadExportFormat {
+ EXPORT_UNUSED = 0x0,
+ EXPORT_32_R = 0x1,
+ EXPORT_32_GR = 0x2,
+ EXPORT_32_AR = 0x3,
+ EXPORT_FP16_ABGR = 0x4,
+ EXPORT_UNSIGNED16_ABGR = 0x5,
+ EXPORT_SIGNED16_ABGR = 0x6,
+ EXPORT_32_ABGR = 0x7,
+ EXPORT_32BPP_8PIX = 0x8,
+ EXPORT_16_16_UNSIGNED_8PIX = 0x9,
+ EXPORT_16_16_SIGNED_8PIX = 0xa,
+ EXPORT_16_16_FLOAT_8PIX = 0xb,
+} QuadExportFormat;
+typedef enum QuadExportFormatOld {
+ EXPORT_4P_32BPC_ABGR = 0x0,
+ EXPORT_4P_16BPC_ABGR = 0x1,
+ EXPORT_4P_32BPC_GR = 0x2,
+ EXPORT_4P_32BPC_AR = 0x3,
+ EXPORT_2P_32BPC_ABGR = 0x4,
+ EXPORT_8P_32BPC_R = 0x5,
+} QuadExportFormatOld;
+typedef enum ColorFormat {
+ COLOR_INVALID = 0x0,
+ COLOR_8 = 0x1,
+ COLOR_16 = 0x2,
+ COLOR_8_8 = 0x3,
+ COLOR_32 = 0x4,
+ COLOR_16_16 = 0x5,
+ COLOR_10_11_11 = 0x6,
+ COLOR_11_11_10 = 0x7,
+ COLOR_10_10_10_2 = 0x8,
+ COLOR_2_10_10_10 = 0x9,
+ COLOR_8_8_8_8 = 0xa,
+ COLOR_32_32 = 0xb,
+ COLOR_16_16_16_16 = 0xc,
+ COLOR_RESERVED_13 = 0xd,
+ COLOR_32_32_32_32 = 0xe,
+ COLOR_RESERVED_15 = 0xf,
+ COLOR_5_6_5 = 0x10,
+ COLOR_1_5_5_5 = 0x11,
+ COLOR_5_5_5_1 = 0x12,
+ COLOR_4_4_4_4 = 0x13,
+ COLOR_8_24 = 0x14,
+ COLOR_24_8 = 0x15,
+ COLOR_X24_8_32_FLOAT = 0x16,
+ COLOR_RESERVED_23 = 0x17,
+ COLOR_RESERVED_24 = 0x18,
+ COLOR_RESERVED_25 = 0x19,
+ COLOR_RESERVED_26 = 0x1a,
+ COLOR_RESERVED_27 = 0x1b,
+ COLOR_RESERVED_28 = 0x1c,
+ COLOR_RESERVED_29 = 0x1d,
+ COLOR_RESERVED_30 = 0x1e,
+} ColorFormat;
+typedef enum SurfaceFormat {
+ FMT_INVALID = 0x0,
+ FMT_8 = 0x1,
+ FMT_16 = 0x2,
+ FMT_8_8 = 0x3,
+ FMT_32 = 0x4,
+ FMT_16_16 = 0x5,
+ FMT_10_11_11 = 0x6,
+ FMT_11_11_10 = 0x7,
+ FMT_10_10_10_2 = 0x8,
+ FMT_2_10_10_10 = 0x9,
+ FMT_8_8_8_8 = 0xa,
+ FMT_32_32 = 0xb,
+ FMT_16_16_16_16 = 0xc,
+ FMT_32_32_32 = 0xd,
+ FMT_32_32_32_32 = 0xe,
+ FMT_RESERVED_4 = 0xf,
+ FMT_5_6_5 = 0x10,
+ FMT_1_5_5_5 = 0x11,
+ FMT_5_5_5_1 = 0x12,
+ FMT_4_4_4_4 = 0x13,
+ FMT_8_24 = 0x14,
+ FMT_24_8 = 0x15,
+ FMT_X24_8_32_FLOAT = 0x16,
+ FMT_RESERVED_33 = 0x17,
+ FMT_11_11_10_FLOAT = 0x18,
+ FMT_16_FLOAT = 0x19,
+ FMT_32_FLOAT = 0x1a,
+ FMT_16_16_FLOAT = 0x1b,
+ FMT_8_24_FLOAT = 0x1c,
+ FMT_24_8_FLOAT = 0x1d,
+ FMT_32_32_FLOAT = 0x1e,
+ FMT_10_11_11_FLOAT = 0x1f,
+ FMT_16_16_16_16_FLOAT = 0x20,
+ FMT_3_3_2 = 0x21,
+ FMT_6_5_5 = 0x22,
+ FMT_32_32_32_32_FLOAT = 0x23,
+ FMT_RESERVED_36 = 0x24,
+ FMT_1 = 0x25,
+ FMT_1_REVERSED = 0x26,
+ FMT_GB_GR = 0x27,
+ FMT_BG_RG = 0x28,
+ FMT_32_AS_8 = 0x29,
+ FMT_32_AS_8_8 = 0x2a,
+ FMT_5_9_9_9_SHAREDEXP = 0x2b,
+ FMT_8_8_8 = 0x2c,
+ FMT_16_16_16 = 0x2d,
+ FMT_16_16_16_FLOAT = 0x2e,
+ FMT_4_4 = 0x2f,
+ FMT_32_32_32_FLOAT = 0x30,
+ FMT_BC1 = 0x31,
+ FMT_BC2 = 0x32,
+ FMT_BC3 = 0x33,
+ FMT_BC4 = 0x34,
+ FMT_BC5 = 0x35,
+ FMT_BC6 = 0x36,
+ FMT_BC7 = 0x37,
+ FMT_32_AS_32_32_32_32 = 0x38,
+ FMT_APC3 = 0x39,
+ FMT_APC4 = 0x3a,
+ FMT_APC5 = 0x3b,
+ FMT_APC6 = 0x3c,
+ FMT_APC7 = 0x3d,
+ FMT_CTX1 = 0x3e,
+ FMT_RESERVED_63 = 0x3f,
+} SurfaceFormat;
+typedef enum BUF_DATA_FORMAT {
+ BUF_DATA_FORMAT_INVALID = 0x0,
+ BUF_DATA_FORMAT_8 = 0x1,
+ BUF_DATA_FORMAT_16 = 0x2,
+ BUF_DATA_FORMAT_8_8 = 0x3,
+ BUF_DATA_FORMAT_32 = 0x4,
+ BUF_DATA_FORMAT_16_16 = 0x5,
+ BUF_DATA_FORMAT_10_11_11 = 0x6,
+ BUF_DATA_FORMAT_11_11_10 = 0x7,
+ BUF_DATA_FORMAT_10_10_10_2 = 0x8,
+ BUF_DATA_FORMAT_2_10_10_10 = 0x9,
+ BUF_DATA_FORMAT_8_8_8_8 = 0xa,
+ BUF_DATA_FORMAT_32_32 = 0xb,
+ BUF_DATA_FORMAT_16_16_16_16 = 0xc,
+ BUF_DATA_FORMAT_32_32_32 = 0xd,
+ BUF_DATA_FORMAT_32_32_32_32 = 0xe,
+ BUF_DATA_FORMAT_RESERVED_15 = 0xf,
+} BUF_DATA_FORMAT;
+typedef enum IMG_DATA_FORMAT {
+ IMG_DATA_FORMAT_INVALID = 0x0,
+ IMG_DATA_FORMAT_8 = 0x1,
+ IMG_DATA_FORMAT_16 = 0x2,
+ IMG_DATA_FORMAT_8_8 = 0x3,
+ IMG_DATA_FORMAT_32 = 0x4,
+ IMG_DATA_FORMAT_16_16 = 0x5,
+ IMG_DATA_FORMAT_10_11_11 = 0x6,
+ IMG_DATA_FORMAT_11_11_10 = 0x7,
+ IMG_DATA_FORMAT_10_10_10_2 = 0x8,
+ IMG_DATA_FORMAT_2_10_10_10 = 0x9,
+ IMG_DATA_FORMAT_8_8_8_8 = 0xa,
+ IMG_DATA_FORMAT_32_32 = 0xb,
+ IMG_DATA_FORMAT_16_16_16_16 = 0xc,
+ IMG_DATA_FORMAT_32_32_32 = 0xd,
+ IMG_DATA_FORMAT_32_32_32_32 = 0xe,
+ IMG_DATA_FORMAT_16_AS_32_32 = 0xf,
+ IMG_DATA_FORMAT_5_6_5 = 0x10,
+ IMG_DATA_FORMAT_1_5_5_5 = 0x11,
+ IMG_DATA_FORMAT_5_5_5_1 = 0x12,
+ IMG_DATA_FORMAT_4_4_4_4 = 0x13,
+ IMG_DATA_FORMAT_8_24 = 0x14,
+ IMG_DATA_FORMAT_24_8 = 0x15,
+ IMG_DATA_FORMAT_X24_8_32 = 0x16,
+ IMG_DATA_FORMAT_8_AS_8_8_8_8 = 0x17,
+ IMG_DATA_FORMAT_ETC2_RGB = 0x18,
+ IMG_DATA_FORMAT_ETC2_RGBA = 0x19,
+ IMG_DATA_FORMAT_ETC2_R = 0x1a,
+ IMG_DATA_FORMAT_ETC2_RG = 0x1b,
+ IMG_DATA_FORMAT_ETC2_RGBA1 = 0x1c,
+ IMG_DATA_FORMAT_RESERVED_29 = 0x1d,
+ IMG_DATA_FORMAT_RESERVED_30 = 0x1e,
+ IMG_DATA_FORMAT_RESERVED_31 = 0x1f,
+ IMG_DATA_FORMAT_GB_GR = 0x20,
+ IMG_DATA_FORMAT_BG_RG = 0x21,
+ IMG_DATA_FORMAT_5_9_9_9 = 0x22,
+ IMG_DATA_FORMAT_BC1 = 0x23,
+ IMG_DATA_FORMAT_BC2 = 0x24,
+ IMG_DATA_FORMAT_BC3 = 0x25,
+ IMG_DATA_FORMAT_BC4 = 0x26,
+ IMG_DATA_FORMAT_BC5 = 0x27,
+ IMG_DATA_FORMAT_BC6 = 0x28,
+ IMG_DATA_FORMAT_BC7 = 0x29,
+ IMG_DATA_FORMAT_16_AS_16_16_16_16 = 0x2a,
+ IMG_DATA_FORMAT_16_AS_32_32_32_32 = 0x2b,
+ IMG_DATA_FORMAT_FMASK8_S2_F1 = 0x2c,
+ IMG_DATA_FORMAT_FMASK8_S4_F1 = 0x2d,
+ IMG_DATA_FORMAT_FMASK8_S8_F1 = 0x2e,
+ IMG_DATA_FORMAT_FMASK8_S2_F2 = 0x2f,
+ IMG_DATA_FORMAT_FMASK8_S4_F2 = 0x30,
+ IMG_DATA_FORMAT_FMASK8_S4_F4 = 0x31,
+ IMG_DATA_FORMAT_FMASK16_S16_F1 = 0x32,
+ IMG_DATA_FORMAT_FMASK16_S8_F2 = 0x33,
+ IMG_DATA_FORMAT_FMASK32_S16_F2 = 0x34,
+ IMG_DATA_FORMAT_FMASK32_S8_F4 = 0x35,
+ IMG_DATA_FORMAT_FMASK32_S8_F8 = 0x36,
+ IMG_DATA_FORMAT_FMASK64_S16_F4 = 0x37,
+ IMG_DATA_FORMAT_FMASK64_S16_F8 = 0x38,
+ IMG_DATA_FORMAT_4_4 = 0x39,
+ IMG_DATA_FORMAT_6_5_5 = 0x3a,
+ IMG_DATA_FORMAT_1 = 0x3b,
+ IMG_DATA_FORMAT_1_REVERSED = 0x3c,
+ IMG_DATA_FORMAT_8_AS_32 = 0x3d,
+ IMG_DATA_FORMAT_8_AS_32_32 = 0x3e,
+ IMG_DATA_FORMAT_32_AS_32_32_32_32 = 0x3f,
+} IMG_DATA_FORMAT;
+typedef enum BUF_NUM_FORMAT {
+ BUF_NUM_FORMAT_UNORM = 0x0,
+ BUF_NUM_FORMAT_SNORM = 0x1,
+ BUF_NUM_FORMAT_USCALED = 0x2,
+ BUF_NUM_FORMAT_SSCALED = 0x3,
+ BUF_NUM_FORMAT_UINT = 0x4,
+ BUF_NUM_FORMAT_SINT = 0x5,
+ BUF_NUM_FORMAT_RESERVED_6 = 0x6,
+ BUF_NUM_FORMAT_FLOAT = 0x7,
+} BUF_NUM_FORMAT;
+typedef enum IMG_NUM_FORMAT {
+ IMG_NUM_FORMAT_UNORM = 0x0,
+ IMG_NUM_FORMAT_SNORM = 0x1,
+ IMG_NUM_FORMAT_USCALED = 0x2,
+ IMG_NUM_FORMAT_SSCALED = 0x3,
+ IMG_NUM_FORMAT_UINT = 0x4,
+ IMG_NUM_FORMAT_SINT = 0x5,
+ IMG_NUM_FORMAT_RESERVED_6 = 0x6,
+ IMG_NUM_FORMAT_FLOAT = 0x7,
+ IMG_NUM_FORMAT_RESERVED_8 = 0x8,
+ IMG_NUM_FORMAT_SRGB = 0x9,
+ IMG_NUM_FORMAT_RESERVED_10 = 0xa,
+ IMG_NUM_FORMAT_RESERVED_11 = 0xb,
+ IMG_NUM_FORMAT_RESERVED_12 = 0xc,
+ IMG_NUM_FORMAT_RESERVED_13 = 0xd,
+ IMG_NUM_FORMAT_RESERVED_14 = 0xe,
+ IMG_NUM_FORMAT_RESERVED_15 = 0xf,
+} IMG_NUM_FORMAT;
+typedef enum TileType {
+ ARRAY_COLOR_TILE = 0x0,
+ ARRAY_DEPTH_TILE = 0x1,
+} TileType;
+typedef enum NonDispTilingOrder {
+ ADDR_SURF_MICRO_TILING_DISPLAY = 0x0,
+ ADDR_SURF_MICRO_TILING_NON_DISPLAY = 0x1,
+} NonDispTilingOrder;
+typedef enum MicroTileMode {
+ ADDR_SURF_DISPLAY_MICRO_TILING = 0x0,
+ ADDR_SURF_THIN_MICRO_TILING = 0x1,
+ ADDR_SURF_DEPTH_MICRO_TILING = 0x2,
+ ADDR_SURF_ROTATED_MICRO_TILING = 0x3,
+ ADDR_SURF_THICK_MICRO_TILING = 0x4,
+} MicroTileMode;
+typedef enum TileSplit {
+ ADDR_SURF_TILE_SPLIT_64B = 0x0,
+ ADDR_SURF_TILE_SPLIT_128B = 0x1,
+ ADDR_SURF_TILE_SPLIT_256B = 0x2,
+ ADDR_SURF_TILE_SPLIT_512B = 0x3,
+ ADDR_SURF_TILE_SPLIT_1KB = 0x4,
+ ADDR_SURF_TILE_SPLIT_2KB = 0x5,
+ ADDR_SURF_TILE_SPLIT_4KB = 0x6,
+} TileSplit;
+typedef enum SampleSplit {
+ ADDR_SURF_SAMPLE_SPLIT_1 = 0x0,
+ ADDR_SURF_SAMPLE_SPLIT_2 = 0x1,
+ ADDR_SURF_SAMPLE_SPLIT_4 = 0x2,
+ ADDR_SURF_SAMPLE_SPLIT_8 = 0x3,
+} SampleSplit;
+typedef enum PipeConfig {
+ ADDR_SURF_P2 = 0x0,
+ ADDR_SURF_P2_RESERVED0 = 0x1,
+ ADDR_SURF_P2_RESERVED1 = 0x2,
+ ADDR_SURF_P2_RESERVED2 = 0x3,
+ ADDR_SURF_P4_8x16 = 0x4,
+ ADDR_SURF_P4_16x16 = 0x5,
+ ADDR_SURF_P4_16x32 = 0x6,
+ ADDR_SURF_P4_32x32 = 0x7,
+ ADDR_SURF_P8_16x16_8x16 = 0x8,
+ ADDR_SURF_P8_16x32_8x16 = 0x9,
+ ADDR_SURF_P8_32x32_8x16 = 0xa,
+ ADDR_SURF_P8_16x32_16x16 = 0xb,
+ ADDR_SURF_P8_32x32_16x16 = 0xc,
+ ADDR_SURF_P8_32x32_16x32 = 0xd,
+ ADDR_SURF_P8_32x64_32x32 = 0xe,
+ ADDR_SURF_P8_RESERVED0 = 0xf,
+ ADDR_SURF_P16_32x32_8x16 = 0x10,
+ ADDR_SURF_P16_32x32_16x16 = 0x11,
+} PipeConfig;
+typedef enum NumBanks {
+ ADDR_SURF_2_BANK = 0x0,
+ ADDR_SURF_4_BANK = 0x1,
+ ADDR_SURF_8_BANK = 0x2,
+ ADDR_SURF_16_BANK = 0x3,
+} NumBanks;
+typedef enum BankWidth {
+ ADDR_SURF_BANK_WIDTH_1 = 0x0,
+ ADDR_SURF_BANK_WIDTH_2 = 0x1,
+ ADDR_SURF_BANK_WIDTH_4 = 0x2,
+ ADDR_SURF_BANK_WIDTH_8 = 0x3,
+} BankWidth;
+typedef enum BankHeight {
+ ADDR_SURF_BANK_HEIGHT_1 = 0x0,
+ ADDR_SURF_BANK_HEIGHT_2 = 0x1,
+ ADDR_SURF_BANK_HEIGHT_4 = 0x2,
+ ADDR_SURF_BANK_HEIGHT_8 = 0x3,
+} BankHeight;
+typedef enum BankWidthHeight {
+ ADDR_SURF_BANK_WH_1 = 0x0,
+ ADDR_SURF_BANK_WH_2 = 0x1,
+ ADDR_SURF_BANK_WH_4 = 0x2,
+ ADDR_SURF_BANK_WH_8 = 0x3,
+} BankWidthHeight;
+typedef enum MacroTileAspect {
+ ADDR_SURF_MACRO_ASPECT_1 = 0x0,
+ ADDR_SURF_MACRO_ASPECT_2 = 0x1,
+ ADDR_SURF_MACRO_ASPECT_4 = 0x2,
+ ADDR_SURF_MACRO_ASPECT_8 = 0x3,
+} MacroTileAspect;
+typedef enum GATCL1RequestType {
+ GATCL1_TYPE_NORMAL = 0x0,
+ GATCL1_TYPE_SHOOTDOWN = 0x1,
+ GATCL1_TYPE_BYPASS = 0x2,
+} GATCL1RequestType;
+typedef enum TCC_CACHE_POLICIES {
+ TCC_CACHE_POLICY_LRU = 0x0,
+ TCC_CACHE_POLICY_STREAM = 0x1,
+} TCC_CACHE_POLICIES;
+typedef enum MTYPE {
+ MTYPE_NC_NV = 0x0,
+ MTYPE_NC = 0x1,
+ MTYPE_CC = 0x2,
+ MTYPE_UC = 0x3,
+} MTYPE;
+typedef enum PERFMON_COUNTER_MODE {
+ PERFMON_COUNTER_MODE_ACCUM = 0x0,
+ PERFMON_COUNTER_MODE_ACTIVE_CYCLES = 0x1,
+ PERFMON_COUNTER_MODE_MAX = 0x2,
+ PERFMON_COUNTER_MODE_DIRTY = 0x3,
+ PERFMON_COUNTER_MODE_SAMPLE = 0x4,
+ PERFMON_COUNTER_MODE_CYCLES_SINCE_FIRST_EVENT = 0x5,
+ PERFMON_COUNTER_MODE_CYCLES_SINCE_LAST_EVENT = 0x6,
+ PERFMON_COUNTER_MODE_CYCLES_GE_HI = 0x7,
+ PERFMON_COUNTER_MODE_CYCLES_EQ_HI = 0x8,
+ PERFMON_COUNTER_MODE_INACTIVE_CYCLES = 0x9,
+ PERFMON_COUNTER_MODE_RESERVED = 0xf,
+} PERFMON_COUNTER_MODE;
+typedef enum PERFMON_SPM_MODE {
+ PERFMON_SPM_MODE_OFF = 0x0,
+ PERFMON_SPM_MODE_16BIT_CLAMP = 0x1,
+ PERFMON_SPM_MODE_16BIT_NO_CLAMP = 0x2,
+ PERFMON_SPM_MODE_32BIT_CLAMP = 0x3,
+ PERFMON_SPM_MODE_32BIT_NO_CLAMP = 0x4,
+ PERFMON_SPM_MODE_RESERVED_5 = 0x5,
+ PERFMON_SPM_MODE_RESERVED_6 = 0x6,
+ PERFMON_SPM_MODE_RESERVED_7 = 0x7,
+ PERFMON_SPM_MODE_TEST_MODE_0 = 0x8,
+ PERFMON_SPM_MODE_TEST_MODE_1 = 0x9,
+ PERFMON_SPM_MODE_TEST_MODE_2 = 0xa,
+} PERFMON_SPM_MODE;
+typedef enum SurfaceTiling {
+ ARRAY_LINEAR = 0x0,
+ ARRAY_TILED = 0x1,
+} SurfaceTiling;
+typedef enum SurfaceArray {
+ ARRAY_1D = 0x0,
+ ARRAY_2D = 0x1,
+ ARRAY_3D = 0x2,
+ ARRAY_3D_SLICE = 0x3,
+} SurfaceArray;
+typedef enum ColorArray {
+ ARRAY_2D_ALT_COLOR = 0x0,
+ ARRAY_2D_COLOR = 0x1,
+ ARRAY_3D_SLICE_COLOR = 0x3,
+} ColorArray;
+typedef enum DepthArray {
+ ARRAY_2D_ALT_DEPTH = 0x0,
+ ARRAY_2D_DEPTH = 0x1,
+} DepthArray;
+typedef enum ENUM_NUM_SIMD_PER_CU {
+ NUM_SIMD_PER_CU = 0x4,
+} ENUM_NUM_SIMD_PER_CU;
+typedef enum MEM_PWR_FORCE_CTRL {
+ NO_FORCE_REQUEST = 0x0,
+ FORCE_LIGHT_SLEEP_REQUEST = 0x1,
+ FORCE_DEEP_SLEEP_REQUEST = 0x2,
+ FORCE_SHUT_DOWN_REQUEST = 0x3,
+} MEM_PWR_FORCE_CTRL;
+typedef enum MEM_PWR_FORCE_CTRL2 {
+ NO_FORCE_REQ = 0x0,
+ FORCE_LIGHT_SLEEP_REQ = 0x1,
+} MEM_PWR_FORCE_CTRL2;
+typedef enum MEM_PWR_DIS_CTRL {
+ ENABLE_MEM_PWR_CTRL = 0x0,
+ DISABLE_MEM_PWR_CTRL = 0x1,
+} MEM_PWR_DIS_CTRL;
+typedef enum MEM_PWR_SEL_CTRL {
+ DYNAMIC_SHUT_DOWN_ENABLE = 0x0,
+ DYNAMIC_DEEP_SLEEP_ENABLE = 0x1,
+ DYNAMIC_LIGHT_SLEEP_ENABLE = 0x2,
+} MEM_PWR_SEL_CTRL;
+typedef enum MEM_PWR_SEL_CTRL2 {
+ DYNAMIC_DEEP_SLEEP_EN = 0x0,
+ DYNAMIC_LIGHT_SLEEP_EN = 0x1,
+} MEM_PWR_SEL_CTRL2;
+
+#endif /* GFX_8_1_ENUM_H */
--- /dev/null
+/*
+ * GFX_8_1 Register documentation
+ *
+ * Copyright (C) 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GFX_8_1_SH_MASK_H
+#define GFX_8_1_SH_MASK_H
+
+#define CB_BLEND_RED__BLEND_RED_MASK 0xffffffff
+#define CB_BLEND_RED__BLEND_RED__SHIFT 0x0
+#define CB_BLEND_GREEN__BLEND_GREEN_MASK 0xffffffff
+#define CB_BLEND_GREEN__BLEND_GREEN__SHIFT 0x0
+#define CB_BLEND_BLUE__BLEND_BLUE_MASK 0xffffffff
+#define CB_BLEND_BLUE__BLEND_BLUE__SHIFT 0x0
+#define CB_BLEND_ALPHA__BLEND_ALPHA_MASK 0xffffffff
+#define CB_BLEND_ALPHA__BLEND_ALPHA__SHIFT 0x0
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_MRT_SHARING_DISABLE_MASK 0x2
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_MRT_SHARING_DISABLE__SHIFT 0x1
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_WATERMARK_MASK 0x7c
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_WATERMARK__SHIFT 0x2
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD_MASK 0x1
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD__SHIFT 0x0
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE_MASK 0x8
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE__SHIFT 0x3
+#define CB_COLOR_CONTROL__MODE_MASK 0x70
+#define CB_COLOR_CONTROL__MODE__SHIFT 0x4
+#define CB_COLOR_CONTROL__ROP3_MASK 0xff0000
+#define CB_COLOR_CONTROL__ROP3__SHIFT 0x10
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND0_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND0_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND0_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND0_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND1_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND1_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND1_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND1_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND2_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND2_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND2_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND2_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND3_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND3_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND3_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND3_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND4_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND4_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND4_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND4_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND5_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND5_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND5_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND5_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND6_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND6_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND6_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND6_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND_MASK 0x1f
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND__SHIFT 0x0
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN_MASK 0xe0
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN__SHIFT 0x5
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND_MASK 0x1f00
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND__SHIFT 0x8
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d
+#define CB_BLEND7_CONTROL__ENABLE_MASK 0x40000000
+#define CB_BLEND7_CONTROL__ENABLE__SHIFT 0x1e
+#define CB_BLEND7_CONTROL__DISABLE_ROP3_MASK 0x80000000
+#define CB_BLEND7_CONTROL__DISABLE_ROP3__SHIFT 0x1f
+#define CB_COLOR0_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR0_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR1_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR1_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR2_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR2_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR3_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR3_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR4_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR4_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR5_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR5_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR6_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR6_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR7_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR7_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR0_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR0_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR0_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR0_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR1_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR1_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR1_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR1_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR2_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR2_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR2_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR2_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR3_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR3_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR3_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR3_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR4_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR4_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR4_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR4_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR5_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR5_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR5_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR5_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR6_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR6_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR6_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR6_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR7_PITCH__TILE_MAX_MASK 0x7ff
+#define CB_COLOR7_PITCH__TILE_MAX__SHIFT 0x0
+#define CB_COLOR7_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000
+#define CB_COLOR7_PITCH__FMASK_TILE_MAX__SHIFT 0x14
+#define CB_COLOR0_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR0_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR1_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR1_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR2_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR2_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR3_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR3_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR4_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR4_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR5_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR5_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR6_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR6_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR7_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR7_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR0_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR0_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR0_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR0_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR1_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR1_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR1_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR1_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR2_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR2_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR2_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR2_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR3_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR3_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR3_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR3_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR4_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR4_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR4_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR4_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR5_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR5_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR5_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR5_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR6_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR6_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR6_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR6_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR7_VIEW__SLICE_START_MASK 0x7ff
+#define CB_COLOR7_VIEW__SLICE_START__SHIFT 0x0
+#define CB_COLOR7_VIEW__SLICE_MAX_MASK 0xffe000
+#define CB_COLOR7_VIEW__SLICE_MAX__SHIFT 0xd
+#define CB_COLOR0_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR0_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR0_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR0_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR0_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR0_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR0_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR0_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR0_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR0_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR0_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR0_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR0_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR0_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR0_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR0_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR0_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR0_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR0_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR0_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR0_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR0_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR0_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR0_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR0_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR0_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR0_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR0_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR0_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR0_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR1_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR1_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR1_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR1_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR1_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR1_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR1_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR1_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR1_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR1_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR1_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR1_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR1_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR1_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR1_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR1_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR1_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR1_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR1_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR1_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR1_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR1_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR1_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR1_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR1_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR1_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR1_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR1_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR1_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR1_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR2_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR2_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR2_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR2_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR2_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR2_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR2_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR2_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR2_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR2_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR2_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR2_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR2_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR2_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR2_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR2_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR2_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR2_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR2_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR2_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR2_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR2_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR2_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR2_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR2_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR2_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR2_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR2_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR2_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR2_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR3_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR3_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR3_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR3_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR3_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR3_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR3_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR3_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR3_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR3_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR3_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR3_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR3_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR3_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR3_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR3_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR3_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR3_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR3_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR3_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR3_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR3_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR3_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR3_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR3_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR3_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR3_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR3_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR3_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR3_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR4_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR4_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR4_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR4_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR4_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR4_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR4_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR4_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR4_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR4_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR4_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR4_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR4_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR4_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR4_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR4_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR4_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR4_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR4_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR4_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR4_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR4_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR4_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR4_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR4_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR4_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR4_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR4_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR4_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR4_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR5_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR5_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR5_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR5_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR5_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR5_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR5_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR5_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR5_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR5_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR5_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR5_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR5_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR5_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR5_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR5_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR5_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR5_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR5_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR5_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR5_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR5_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR5_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR5_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR5_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR5_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR5_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR5_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR5_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR5_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR6_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR6_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR6_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR6_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR6_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR6_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR6_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR6_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR6_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR6_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR6_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR6_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR6_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR6_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR6_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR6_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR6_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR6_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR6_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR6_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR6_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR6_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR6_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR6_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR6_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR6_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR6_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR6_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR6_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR6_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR7_INFO__ENDIAN_MASK 0x3
+#define CB_COLOR7_INFO__ENDIAN__SHIFT 0x0
+#define CB_COLOR7_INFO__FORMAT_MASK 0x7c
+#define CB_COLOR7_INFO__FORMAT__SHIFT 0x2
+#define CB_COLOR7_INFO__LINEAR_GENERAL_MASK 0x80
+#define CB_COLOR7_INFO__LINEAR_GENERAL__SHIFT 0x7
+#define CB_COLOR7_INFO__NUMBER_TYPE_MASK 0x700
+#define CB_COLOR7_INFO__NUMBER_TYPE__SHIFT 0x8
+#define CB_COLOR7_INFO__COMP_SWAP_MASK 0x1800
+#define CB_COLOR7_INFO__COMP_SWAP__SHIFT 0xb
+#define CB_COLOR7_INFO__FAST_CLEAR_MASK 0x2000
+#define CB_COLOR7_INFO__FAST_CLEAR__SHIFT 0xd
+#define CB_COLOR7_INFO__COMPRESSION_MASK 0x4000
+#define CB_COLOR7_INFO__COMPRESSION__SHIFT 0xe
+#define CB_COLOR7_INFO__BLEND_CLAMP_MASK 0x8000
+#define CB_COLOR7_INFO__BLEND_CLAMP__SHIFT 0xf
+#define CB_COLOR7_INFO__BLEND_BYPASS_MASK 0x10000
+#define CB_COLOR7_INFO__BLEND_BYPASS__SHIFT 0x10
+#define CB_COLOR7_INFO__SIMPLE_FLOAT_MASK 0x20000
+#define CB_COLOR7_INFO__SIMPLE_FLOAT__SHIFT 0x11
+#define CB_COLOR7_INFO__ROUND_MODE_MASK 0x40000
+#define CB_COLOR7_INFO__ROUND_MODE__SHIFT 0x12
+#define CB_COLOR7_INFO__CMASK_IS_LINEAR_MASK 0x80000
+#define CB_COLOR7_INFO__CMASK_IS_LINEAR__SHIFT 0x13
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17
+#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000
+#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a
+#define CB_COLOR7_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK 0x8000000
+#define CB_COLOR7_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT 0x1b
+#define CB_COLOR7_INFO__DCC_ENABLE_MASK 0x10000000
+#define CB_COLOR7_INFO__DCC_ENABLE__SHIFT 0x1c
+#define CB_COLOR7_INFO__CMASK_ADDR_TYPE_MASK 0x60000000
+#define CB_COLOR7_INFO__CMASK_ADDR_TYPE__SHIFT 0x1d
+#define CB_COLOR0_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR0_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR0_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR0_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR0_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR0_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR0_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR0_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR1_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR1_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR1_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR1_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR1_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR1_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR1_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR1_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR2_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR2_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR2_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR2_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR2_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR2_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR2_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR2_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR3_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR3_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR3_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR3_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR3_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR3_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR3_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR3_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR4_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR4_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR4_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR4_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR4_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR4_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR4_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR4_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR5_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR5_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR5_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR5_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR5_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR5_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR5_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR5_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR6_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR6_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR6_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR6_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR6_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR6_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR6_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR6_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR7_ATTRIB__TILE_MODE_INDEX_MASK 0x1f
+#define CB_COLOR7_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0
+#define CB_COLOR7_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0
+#define CB_COLOR7_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5
+#define CB_COLOR7_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00
+#define CB_COLOR7_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa
+#define CB_COLOR7_ATTRIB__NUM_SAMPLES_MASK 0x7000
+#define CB_COLOR7_ATTRIB__NUM_SAMPLES__SHIFT 0xc
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS_MASK 0x18000
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11
+#define CB_COLOR0_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR0_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR0_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR0_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR0_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR0_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR0_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR0_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR0_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR0_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR0_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR0_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR0_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR0_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR0_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR0_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR0_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR0_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR1_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR1_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR1_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR1_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR1_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR1_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR1_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR1_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR1_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR1_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR1_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR1_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR1_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR1_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR1_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR1_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR1_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR1_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR2_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR2_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR2_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR2_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR2_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR2_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR2_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR2_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR2_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR2_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR2_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR2_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR2_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR2_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR2_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR2_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR2_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR2_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR3_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR3_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR3_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR3_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR3_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR3_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR3_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR3_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR3_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR3_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR3_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR3_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR3_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR3_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR3_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR3_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR3_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR3_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR4_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR4_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR4_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR4_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR4_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR4_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR4_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR4_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR4_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR4_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR4_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR4_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR4_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR4_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR4_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR4_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR4_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR4_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR5_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR5_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR5_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR5_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR5_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR5_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR5_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR5_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR5_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR5_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR5_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR5_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR5_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR5_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR5_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR5_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR5_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR5_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR6_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR6_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR6_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR6_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR6_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR6_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR6_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR6_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR6_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR6_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR6_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR6_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR6_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR6_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR6_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR6_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR6_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR6_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR7_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK 0x1
+#define CB_COLOR7_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT 0x0
+#define CB_COLOR7_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK 0x2
+#define CB_COLOR7_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT 0x1
+#define CB_COLOR7_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK 0xc
+#define CB_COLOR7_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT 0x2
+#define CB_COLOR7_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK 0x10
+#define CB_COLOR7_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT 0x4
+#define CB_COLOR7_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK 0x60
+#define CB_COLOR7_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT 0x5
+#define CB_COLOR7_DCC_CONTROL__COLOR_TRANSFORM_MASK 0x180
+#define CB_COLOR7_DCC_CONTROL__COLOR_TRANSFORM__SHIFT 0x7
+#define CB_COLOR7_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK 0x200
+#define CB_COLOR7_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT 0x9
+#define CB_COLOR7_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK 0x3c00
+#define CB_COLOR7_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT 0xa
+#define CB_COLOR7_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK 0x3c000
+#define CB_COLOR7_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT 0xe
+#define CB_COLOR0_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR0_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR1_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR1_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR2_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR2_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR3_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR3_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR4_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR4_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR5_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR5_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR6_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR6_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR7_CMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR7_CMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR0_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR0_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR1_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR1_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR2_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR2_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR3_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR3_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR4_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR4_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR5_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR5_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR6_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR6_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR7_CMASK_SLICE__TILE_MAX_MASK 0x3fff
+#define CB_COLOR7_CMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR0_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR0_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR1_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR1_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR2_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR2_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR3_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR3_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR4_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR4_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR5_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR5_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR6_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR6_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR7_FMASK__BASE_256B_MASK 0xffffffff
+#define CB_COLOR7_FMASK__BASE_256B__SHIFT 0x0
+#define CB_COLOR0_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR0_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR1_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR1_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR2_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR2_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR3_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR3_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR4_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR4_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR5_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR5_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR6_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR6_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR7_FMASK_SLICE__TILE_MAX_MASK 0x3fffff
+#define CB_COLOR7_FMASK_SLICE__TILE_MAX__SHIFT 0x0
+#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff
+#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0
+#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff
+#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0
+#define CB_COLOR0_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR0_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR1_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR1_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR2_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR2_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR3_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR3_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR4_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR4_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR5_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR5_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR6_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR6_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_COLOR7_DCC_BASE__BASE_256B_MASK 0xffffffff
+#define CB_COLOR7_DCC_BASE__BASE_256B__SHIFT 0x0
+#define CB_TARGET_MASK__TARGET0_ENABLE_MASK 0xf
+#define CB_TARGET_MASK__TARGET0_ENABLE__SHIFT 0x0
+#define CB_TARGET_MASK__TARGET1_ENABLE_MASK 0xf0
+#define CB_TARGET_MASK__TARGET1_ENABLE__SHIFT 0x4
+#define CB_TARGET_MASK__TARGET2_ENABLE_MASK 0xf00
+#define CB_TARGET_MASK__TARGET2_ENABLE__SHIFT 0x8
+#define CB_TARGET_MASK__TARGET3_ENABLE_MASK 0xf000
+#define CB_TARGET_MASK__TARGET3_ENABLE__SHIFT 0xc
+#define CB_TARGET_MASK__TARGET4_ENABLE_MASK 0xf0000
+#define CB_TARGET_MASK__TARGET4_ENABLE__SHIFT 0x10
+#define CB_TARGET_MASK__TARGET5_ENABLE_MASK 0xf00000
+#define CB_TARGET_MASK__TARGET5_ENABLE__SHIFT 0x14
+#define CB_TARGET_MASK__TARGET6_ENABLE_MASK 0xf000000
+#define CB_TARGET_MASK__TARGET6_ENABLE__SHIFT 0x18
+#define CB_TARGET_MASK__TARGET7_ENABLE_MASK 0xf0000000
+#define CB_TARGET_MASK__TARGET7_ENABLE__SHIFT 0x1c
+#define CB_SHADER_MASK__OUTPUT0_ENABLE_MASK 0xf
+#define CB_SHADER_MASK__OUTPUT0_ENABLE__SHIFT 0x0
+#define CB_SHADER_MASK__OUTPUT1_ENABLE_MASK 0xf0
+#define CB_SHADER_MASK__OUTPUT1_ENABLE__SHIFT 0x4
+#define CB_SHADER_MASK__OUTPUT2_ENABLE_MASK 0xf00
+#define CB_SHADER_MASK__OUTPUT2_ENABLE__SHIFT 0x8
+#define CB_SHADER_MASK__OUTPUT3_ENABLE_MASK 0xf000
+#define CB_SHADER_MASK__OUTPUT3_ENABLE__SHIFT 0xc
+#define CB_SHADER_MASK__OUTPUT4_ENABLE_MASK 0xf0000
+#define CB_SHADER_MASK__OUTPUT4_ENABLE__SHIFT 0x10
+#define CB_SHADER_MASK__OUTPUT5_ENABLE_MASK 0xf00000
+#define CB_SHADER_MASK__OUTPUT5_ENABLE__SHIFT 0x14
+#define CB_SHADER_MASK__OUTPUT6_ENABLE_MASK 0xf000000
+#define CB_SHADER_MASK__OUTPUT6_ENABLE__SHIFT 0x18
+#define CB_SHADER_MASK__OUTPUT7_ENABLE_MASK 0xf0000000
+#define CB_SHADER_MASK__OUTPUT7_ENABLE__SHIFT 0x1c
+#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT_MASK 0xf
+#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT__SHIFT 0x0
+#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT_MASK 0x3c0
+#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT__SHIFT 0x6
+#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT_MASK 0xf000
+#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT__SHIFT 0xc
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE_MASK 0x10000
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE__SHIFT 0x10
+#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING_MASK 0x40000
+#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING__SHIFT 0x12
+#define CB_HW_CONTROL__FORCE_NEEDS_DST_MASK 0x80000
+#define CB_HW_CONTROL__FORCE_NEEDS_DST__SHIFT 0x13
+#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE_MASK 0x100000
+#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE__SHIFT 0x14
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST_MASK 0x200000
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST__SHIFT 0x15
+#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK_MASK 0x400000
+#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK__SHIFT 0x16
+#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG_MASK 0x800000
+#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG__SHIFT 0x17
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST_MASK 0x1000000
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT 0x18
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS_MASK 0x2000000
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS__SHIFT 0x19
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK 0x4000000
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT 0x1a
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED_MASK 0x8000000
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED__SHIFT 0x1b
+#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT_MASK 0x10000000
+#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT__SHIFT 0x1c
+#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT_MASK 0x20000000
+#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT__SHIFT 0x1d
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT_MASK 0x40000000
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT__SHIFT 0x1e
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE_MASK 0x80000000
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE__SHIFT 0x1f
+#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS_MASK 0x1f
+#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS__SHIFT 0x0
+#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS_MASK 0x7e0
+#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS__SHIFT 0x5
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS_MASK 0x1f800
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS__SHIFT 0xb
+#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH_MASK 0x3fe0000
+#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH__SHIFT 0x11
+#define CB_HW_CONTROL_1__CHICKEN_BITS_MASK 0xfc000000
+#define CB_HW_CONTROL_1__CHICKEN_BITS__SHIFT 0x1a
+#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH_MASK 0xff
+#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH__SHIFT 0x0
+#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH_MASK 0x7f00
+#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH__SHIFT 0x8
+#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH_MASK 0x7f8000
+#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH__SHIFT 0xf
+#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8_MASK 0xf000000
+#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8__SHIFT 0x18
+#define CB_HW_CONTROL_2__CHICKEN_BITS_MASK 0xf0000000
+#define CB_HW_CONTROL_2__CHICKEN_BITS__SHIFT 0x1c
+#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL_MASK 0x1
+#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL__SHIFT 0x0
+#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED_MASK 0x2
+#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED__SHIFT 0x1
+#define CB_HW_CONTROL_3__DISABLE_FAST_CLEAR_FETCH_OPT_MASK 0x4
+#define CB_HW_CONTROL_3__DISABLE_FAST_CLEAR_FETCH_OPT__SHIFT 0x2
+#define CB_HW_CONTROL_3__DISABLE_QUAD_MARKER_DROP_STOP_MASK 0x8
+#define CB_HW_CONTROL_3__DISABLE_QUAD_MARKER_DROP_STOP__SHIFT 0x3
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_CAM_CLR_MASK 0x10
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_CAM_CLR__SHIFT 0x4
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM_MASK 0x20
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM__SHIFT 0x5
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING_MASK 0x80
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING__SHIFT 0x7
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_TARGET_MASK_VALIDATION_MASK 0x100
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_TARGET_MASK_VALIDATION__SHIFT 0x8
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS_MASK 0x200
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS__SHIFT 0x9
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS_MASK 0x400
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS__SHIFT 0xa
+#define CB_HW_CONTROL_3__DISABLE_CMASK_LAST_QUAD_INSERTION_MASK 0x800
+#define CB_HW_CONTROL_3__DISABLE_CMASK_LAST_QUAD_INSERTION__SHIFT 0xb
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_511967_MASK 0x1000
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_511967__SHIFT 0xc
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_520657_MASK 0x2000
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_520657__SHIFT 0xd
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DEPTH_MASK 0x1f
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DEPTH__SHIFT 0x0
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DISABLE_MASK 0x20
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DISABLE__SHIFT 0x5
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_CC_POP_DISABLE_MASK 0x40
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_CC_POP_DISABLE__SHIFT 0x6
+#define CB_DCC_CONFIG__FC_RDLAT_KEYID_FIFO_DEPTH_MASK 0xff00
+#define CB_DCC_CONFIG__FC_RDLAT_KEYID_FIFO_DEPTH__SHIFT 0x8
+#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH_MASK 0x7f0000
+#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH__SHIFT 0x10
+#define CB_DCC_CONFIG__DCC_CACHE_EVICT_POINT_MASK 0xf000000
+#define CB_DCC_CONFIG__DCC_CACHE_EVICT_POINT__SHIFT 0x18
+#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS_MASK 0xf0000000
+#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS__SHIFT 0x1c
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE_MASK 0x1
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE__SHIFT 0x0
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL_MASK 0xe
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL__SHIFT 0x1
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE_MASK 0x10
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE__SHIFT 0x4
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL_MASK 0x3e0
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL__SHIFT 0x5
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE_MASK 0x400
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE__SHIFT 0xa
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL_MASK 0x800
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL__SHIFT 0xb
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE_MASK 0x1000
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE__SHIFT 0xc
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL_MASK 0xe000
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL__SHIFT 0xd
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE_MASK 0x20000
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE__SHIFT 0x11
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL_MASK 0x1c0000
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL__SHIFT 0x12
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE_MASK 0x200000
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE__SHIFT 0x15
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL_MASK 0xc00000
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL__SHIFT 0x16
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x1ff
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x7fc00
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x1ff
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x7fc00
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x1ff
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x1ff
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x1ff
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK 0xf
+#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define CB_DEBUG_BUS_1__CB_BUSY_MASK 0x1
+#define CB_DEBUG_BUS_1__CB_BUSY__SHIFT 0x0
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALID_READY_MASK 0x2
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALID_READY__SHIFT 0x1
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALID_READYB_MASK 0x4
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALID_READYB__SHIFT 0x2
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALIDB_READY_MASK 0x8
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALIDB_READY__SHIFT 0x3
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALIDB_READYB_MASK 0x10
+#define CB_DEBUG_BUS_1__DB_CB_TILE_VALIDB_READYB__SHIFT 0x4
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALID_READY_MASK 0x20
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALID_READY__SHIFT 0x5
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALID_READYB_MASK 0x40
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALID_READYB__SHIFT 0x6
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALIDB_READY_MASK 0x80
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALIDB_READY__SHIFT 0x7
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALIDB_READYB_MASK 0x100
+#define CB_DEBUG_BUS_1__DB_CB_LQUAD_VALIDB_READYB__SHIFT 0x8
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALID_READY_MASK 0x200
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALID_READY__SHIFT 0x9
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALID_READYB_MASK 0x400
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALID_READYB__SHIFT 0xa
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALIDB_READY_MASK 0x800
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALIDB_READY__SHIFT 0xb
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALIDB_READYB_MASK 0x1000
+#define CB_DEBUG_BUS_1__CB_TAP_WRREQ_VALIDB_READYB__SHIFT 0xc
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALID_READY_MASK 0x2000
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALID_READY__SHIFT 0xd
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALID_READYB_MASK 0x4000
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALID_READYB__SHIFT 0xe
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALIDB_READY_MASK 0x8000
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALIDB_READY__SHIFT 0xf
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALIDB_READYB_MASK 0x10000
+#define CB_DEBUG_BUS_1__CB_TAP_RDREQ_VALIDB_READYB__SHIFT 0x10
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALID_READY_MASK 0x20000
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALID_READY__SHIFT 0x11
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALID_READYB_MASK 0x40000
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALID_READYB__SHIFT 0x12
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALIDB_READY_MASK 0x80000
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALIDB_READY__SHIFT 0x13
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALIDB_READYB_MASK 0x100000
+#define CB_DEBUG_BUS_1__CM_FC_TILE_VALIDB_READYB__SHIFT 0x14
+#define CB_DEBUG_BUS_1__FC_CLEAR_QUAD_VALID_READY_MASK 0x200000
+#define CB_DEBUG_BUS_1__FC_CLEAR_QUAD_VALID_READY__SHIFT 0x15
+#define CB_DEBUG_BUS_1__FC_CLEAR_QUAD_VALID_READYB_MASK 0x400000
+#define CB_DEBUG_BUS_1__FC_CLEAR_QUAD_VALID_READYB__SHIFT 0x16
+#define CB_DEBUG_BUS_1__FC_CLEAR_QUAD_VALIDB_READY_MASK 0x800000
+#define CB_DEBUG_BUS_1__FC_CLEAR_QUAD_VALIDB_READY__SHIFT 0x17
+#define CB_DEBUG_BUS_2__FC_CLEAR_QUAD_VALIDB_READYB_MASK 0x1
+#define CB_DEBUG_BUS_2__FC_CLEAR_QUAD_VALIDB_READYB__SHIFT 0x0
+#define CB_DEBUG_BUS_2__FC_QUAD_RESIDENCY_STALL_MASK 0x2
+#define CB_DEBUG_BUS_2__FC_QUAD_RESIDENCY_STALL__SHIFT 0x1
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALID_READY_MASK 0x4
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALID_READY__SHIFT 0x2
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALID_READYB_MASK 0x8
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALID_READYB__SHIFT 0x3
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALIDB_READY_MASK 0x10
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALIDB_READY__SHIFT 0x4
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALIDB_READYB_MASK 0x20
+#define CB_DEBUG_BUS_2__FC_CC_QUADFRAG_VALIDB_READYB__SHIFT 0x5
+#define CB_DEBUG_BUS_2__FOP_IN_VALID_READY_MASK 0x40
+#define CB_DEBUG_BUS_2__FOP_IN_VALID_READY__SHIFT 0x6
+#define CB_DEBUG_BUS_2__FOP_IN_VALID_READYB_MASK 0x80
+#define CB_DEBUG_BUS_2__FOP_IN_VALID_READYB__SHIFT 0x7
+#define CB_DEBUG_BUS_2__FOP_IN_VALIDB_READY_MASK 0x100
+#define CB_DEBUG_BUS_2__FOP_IN_VALIDB_READY__SHIFT 0x8
+#define CB_DEBUG_BUS_2__FOP_IN_VALIDB_READYB_MASK 0x200
+#define CB_DEBUG_BUS_2__FOP_IN_VALIDB_READYB__SHIFT 0x9
+#define CB_DEBUG_BUS_2__FOP_FMASK_RAW_STALL_MASK 0x400
+#define CB_DEBUG_BUS_2__FOP_FMASK_RAW_STALL__SHIFT 0xa
+#define CB_DEBUG_BUS_2__FOP_FMASK_BYPASS_STALL_MASK 0x800
+#define CB_DEBUG_BUS_2__FOP_FMASK_BYPASS_STALL__SHIFT 0xb
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALID_READY_MASK 0x1000
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALID_READY__SHIFT 0xc
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALID_READYB_MASK 0x2000
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALID_READYB__SHIFT 0xd
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALIDB_READY_MASK 0x4000
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALIDB_READY__SHIFT 0xe
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALIDB_READYB_MASK 0x8000
+#define CB_DEBUG_BUS_2__CC_IB_TB_FRAG_VALIDB_READYB__SHIFT 0xf
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALID_READY_MASK 0x10000
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALID_READY__SHIFT 0x10
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALID_READYB_MASK 0x20000
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALID_READYB__SHIFT 0x11
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALIDB_READY_MASK 0x40000
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALIDB_READY__SHIFT 0x12
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALIDB_READYB_MASK 0x80000
+#define CB_DEBUG_BUS_2__CC_IB_SR_FRAG_VALIDB_READYB__SHIFT 0x13
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALID_READY_MASK 0x100000
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALID_READY__SHIFT 0x14
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALID_READYB_MASK 0x200000
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALID_READYB__SHIFT 0x15
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALIDB_READY_MASK 0x400000
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALIDB_READY__SHIFT 0x16
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALIDB_READYB_MASK 0x800000
+#define CB_DEBUG_BUS_2__CC_RB_BC_EVENFRAG_VALIDB_READYB__SHIFT 0x17
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALID_READY_MASK 0x1
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALID_READY__SHIFT 0x0
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALID_READYB_MASK 0x2
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALID_READYB__SHIFT 0x1
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALIDB_READY_MASK 0x4
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALIDB_READY__SHIFT 0x2
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALIDB_READYB_MASK 0x8
+#define CB_DEBUG_BUS_3__CC_RB_BC_ODDFRAG_VALIDB_READYB__SHIFT 0x3
+#define CB_DEBUG_BUS_3__CC_BC_CS_FRAG_VALID_MASK 0x10
+#define CB_DEBUG_BUS_3__CC_BC_CS_FRAG_VALID__SHIFT 0x4
+#define CB_DEBUG_BUS_3__CC_SF_FULL_MASK 0x20
+#define CB_DEBUG_BUS_3__CC_SF_FULL__SHIFT 0x5
+#define CB_DEBUG_BUS_3__CC_RB_FULL_MASK 0x40
+#define CB_DEBUG_BUS_3__CC_RB_FULL__SHIFT 0x6
+#define CB_DEBUG_BUS_3__CC_EVENFIFO_QUAD_RESIDENCY_STALL_MASK 0x80
+#define CB_DEBUG_BUS_3__CC_EVENFIFO_QUAD_RESIDENCY_STALL__SHIFT 0x7
+#define CB_DEBUG_BUS_3__CC_ODDFIFO_QUAD_RESIDENCY_STALL_MASK 0x100
+#define CB_DEBUG_BUS_3__CC_ODDFIFO_QUAD_RESIDENCY_STALL__SHIFT 0x8
+#define CB_DEBUG_BUS_3__CM_TQ_FULL_MASK 0x200
+#define CB_DEBUG_BUS_3__CM_TQ_FULL__SHIFT 0x9
+#define CB_DEBUG_BUS_3__CM_TILE_RESIDENCY_STALL_MASK 0x400
+#define CB_DEBUG_BUS_3__CM_TILE_RESIDENCY_STALL__SHIFT 0xa
+#define CB_DEBUG_BUS_3__LQUAD_NO_TILE_MASK 0x800
+#define CB_DEBUG_BUS_3__LQUAD_NO_TILE__SHIFT 0xb
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_R_MASK 0x1000
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_R__SHIFT 0xc
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_AR_MASK 0x2000
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_AR__SHIFT 0xd
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_GR_MASK 0x4000
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_GR__SHIFT 0xe
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_ABGR_MASK 0x8000
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_32_ABGR__SHIFT 0xf
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_FP16_ABGR_MASK 0x10000
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_FP16_ABGR__SHIFT 0x10
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_SIGNED16_ABGR_MASK 0x20000
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_SIGNED16_ABGR__SHIFT 0x11
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_UNSIGNED16_ABGR_MASK 0x40000
+#define CB_DEBUG_BUS_3__LQUAD_FORMAT_IS_EXPORT_UNSIGNED16_ABGR__SHIFT 0x12
+#define CB_DEBUG_BUS_3__CM_CACHE_HIT_MASK 0x80000
+#define CB_DEBUG_BUS_3__CM_CACHE_HIT__SHIFT 0x13
+#define CB_DEBUG_BUS_3__CM_CACHE_TAG_MISS_MASK 0x100000
+#define CB_DEBUG_BUS_3__CM_CACHE_TAG_MISS__SHIFT 0x14
+#define CB_DEBUG_BUS_3__CM_CACHE_SECTOR_MISS_MASK 0x200000
+#define CB_DEBUG_BUS_3__CM_CACHE_SECTOR_MISS__SHIFT 0x15
+#define CB_DEBUG_BUS_3__CM_CACHE_REEVICTION_STALL_MASK 0x400000
+#define CB_DEBUG_BUS_3__CM_CACHE_REEVICTION_STALL__SHIFT 0x16
+#define CB_DEBUG_BUS_3__CM_CACHE_EVICT_NONZERO_INFLIGHT_STALL_MASK 0x800000
+#define CB_DEBUG_BUS_3__CM_CACHE_EVICT_NONZERO_INFLIGHT_STALL__SHIFT 0x17
+#define CB_DEBUG_BUS_4__CM_CACHE_REPLACE_PENDING_EVICT_STALL_MASK 0x1
+#define CB_DEBUG_BUS_4__CM_CACHE_REPLACE_PENDING_EVICT_STALL__SHIFT 0x0
+#define CB_DEBUG_BUS_4__CM_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL_MASK 0x2
+#define CB_DEBUG_BUS_4__CM_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL__SHIFT 0x1
+#define CB_DEBUG_BUS_4__CM_CACHE_READ_OUTPUT_STALL_MASK 0x4
+#define CB_DEBUG_BUS_4__CM_CACHE_READ_OUTPUT_STALL__SHIFT 0x2
+#define CB_DEBUG_BUS_4__CM_CACHE_WRITE_OUTPUT_STALL_MASK 0x8
+#define CB_DEBUG_BUS_4__CM_CACHE_WRITE_OUTPUT_STALL__SHIFT 0x3
+#define CB_DEBUG_BUS_4__CM_CACHE_ACK_OUTPUT_STALL_MASK 0x10
+#define CB_DEBUG_BUS_4__CM_CACHE_ACK_OUTPUT_STALL__SHIFT 0x4
+#define CB_DEBUG_BUS_4__CM_CACHE_STALL_MASK 0x20
+#define CB_DEBUG_BUS_4__CM_CACHE_STALL__SHIFT 0x5
+#define CB_DEBUG_BUS_4__FC_CACHE_HIT_MASK 0x40
+#define CB_DEBUG_BUS_4__FC_CACHE_HIT__SHIFT 0x6
+#define CB_DEBUG_BUS_4__FC_CACHE_TAG_MISS_MASK 0x80
+#define CB_DEBUG_BUS_4__FC_CACHE_TAG_MISS__SHIFT 0x7
+#define CB_DEBUG_BUS_4__FC_CACHE_SECTOR_MISS_MASK 0x100
+#define CB_DEBUG_BUS_4__FC_CACHE_SECTOR_MISS__SHIFT 0x8
+#define CB_DEBUG_BUS_4__FC_CACHE_REEVICTION_STALL_MASK 0x200
+#define CB_DEBUG_BUS_4__FC_CACHE_REEVICTION_STALL__SHIFT 0x9
+#define CB_DEBUG_BUS_4__FC_CACHE_EVICT_NONZERO_INFLIGHT_STALL_MASK 0x400
+#define CB_DEBUG_BUS_4__FC_CACHE_EVICT_NONZERO_INFLIGHT_STALL__SHIFT 0xa
+#define CB_DEBUG_BUS_4__FC_CACHE_REPLACE_PENDING_EVICT_STALL_MASK 0x800
+#define CB_DEBUG_BUS_4__FC_CACHE_REPLACE_PENDING_EVICT_STALL__SHIFT 0xb
+#define CB_DEBUG_BUS_4__FC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL_MASK 0x1000
+#define CB_DEBUG_BUS_4__FC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL__SHIFT 0xc
+#define CB_DEBUG_BUS_4__FC_CACHE_READ_OUTPUT_STALL_MASK 0x2000
+#define CB_DEBUG_BUS_4__FC_CACHE_READ_OUTPUT_STALL__SHIFT 0xd
+#define CB_DEBUG_BUS_4__FC_CACHE_WRITE_OUTPUT_STALL_MASK 0x4000
+#define CB_DEBUG_BUS_4__FC_CACHE_WRITE_OUTPUT_STALL__SHIFT 0xe
+#define CB_DEBUG_BUS_4__FC_CACHE_ACK_OUTPUT_STALL_MASK 0x8000
+#define CB_DEBUG_BUS_4__FC_CACHE_ACK_OUTPUT_STALL__SHIFT 0xf
+#define CB_DEBUG_BUS_4__FC_CACHE_STALL_MASK 0x10000
+#define CB_DEBUG_BUS_4__FC_CACHE_STALL__SHIFT 0x10
+#define CB_DEBUG_BUS_4__CC_CACHE_HIT_MASK 0x20000
+#define CB_DEBUG_BUS_4__CC_CACHE_HIT__SHIFT 0x11
+#define CB_DEBUG_BUS_4__CC_CACHE_TAG_MISS_MASK 0x40000
+#define CB_DEBUG_BUS_4__CC_CACHE_TAG_MISS__SHIFT 0x12
+#define CB_DEBUG_BUS_4__CC_CACHE_SECTOR_MISS_MASK 0x80000
+#define CB_DEBUG_BUS_4__CC_CACHE_SECTOR_MISS__SHIFT 0x13
+#define CB_DEBUG_BUS_4__CC_CACHE_REEVICTION_STALL_MASK 0x100000
+#define CB_DEBUG_BUS_4__CC_CACHE_REEVICTION_STALL__SHIFT 0x14
+#define CB_DEBUG_BUS_4__CC_CACHE_EVICT_NONZERO_INFLIGHT_STALL_MASK 0x200000
+#define CB_DEBUG_BUS_4__CC_CACHE_EVICT_NONZERO_INFLIGHT_STALL__SHIFT 0x15
+#define CB_DEBUG_BUS_4__CC_CACHE_REPLACE_PENDING_EVICT_STALL_MASK 0x400000
+#define CB_DEBUG_BUS_4__CC_CACHE_REPLACE_PENDING_EVICT_STALL__SHIFT 0x16
+#define CB_DEBUG_BUS_4__CC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL_MASK 0x800000
+#define CB_DEBUG_BUS_4__CC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL__SHIFT 0x17
+#define CB_DEBUG_BUS_5__CC_CACHE_READ_OUTPUT_STALL_MASK 0x1
+#define CB_DEBUG_BUS_5__CC_CACHE_READ_OUTPUT_STALL__SHIFT 0x0
+#define CB_DEBUG_BUS_5__CC_CACHE_WRITE_OUTPUT_STALL_MASK 0x2
+#define CB_DEBUG_BUS_5__CC_CACHE_WRITE_OUTPUT_STALL__SHIFT 0x1
+#define CB_DEBUG_BUS_5__CC_CACHE_ACK_OUTPUT_STALL_MASK 0x4
+#define CB_DEBUG_BUS_5__CC_CACHE_ACK_OUTPUT_STALL__SHIFT 0x2
+#define CB_DEBUG_BUS_5__CC_CACHE_STALL_MASK 0x8
+#define CB_DEBUG_BUS_5__CC_CACHE_STALL__SHIFT 0x3
+#define CB_DEBUG_BUS_5__CC_CACHE_WA_TO_RMW_CONVERSION_MASK 0x10
+#define CB_DEBUG_BUS_5__CC_CACHE_WA_TO_RMW_CONVERSION__SHIFT 0x4
+#define CB_DEBUG_BUS_5__CM_CACHE_FLUSH_MASK 0x20
+#define CB_DEBUG_BUS_5__CM_CACHE_FLUSH__SHIFT 0x5
+#define CB_DEBUG_BUS_5__CM_CACHE_TAGS_FLUSHED_MASK 0x40
+#define CB_DEBUG_BUS_5__CM_CACHE_TAGS_FLUSHED__SHIFT 0x6
+#define CB_DEBUG_BUS_5__CM_CACHE_SECTORS_FLUSHED_MASK 0x80
+#define CB_DEBUG_BUS_5__CM_CACHE_SECTORS_FLUSHED__SHIFT 0x7
+#define CB_DEBUG_BUS_5__CM_CACHE_DIRTY_SECTORS_FLUSHED_MASK 0x100
+#define CB_DEBUG_BUS_5__CM_CACHE_DIRTY_SECTORS_FLUSHED__SHIFT 0x8
+#define CB_DEBUG_BUS_5__FC_CACHE_FLUSH_MASK 0x200
+#define CB_DEBUG_BUS_5__FC_CACHE_FLUSH__SHIFT 0x9
+#define CB_DEBUG_BUS_5__FC_CACHE_TAGS_FLUSHED_MASK 0x400
+#define CB_DEBUG_BUS_5__FC_CACHE_TAGS_FLUSHED__SHIFT 0xa
+#define CB_DEBUG_BUS_5__FC_CACHE_SECTORS_FLUSHED_MASK 0x3800
+#define CB_DEBUG_BUS_5__FC_CACHE_SECTORS_FLUSHED__SHIFT 0xb
+#define CB_DEBUG_BUS_5__FC_CACHE_DIRTY_SECTORS_FLUSHED_MASK 0x1c000
+#define CB_DEBUG_BUS_5__FC_CACHE_DIRTY_SECTORS_FLUSHED__SHIFT 0xe
+#define CB_DEBUG_BUS_5__CC_CACHE_FLUSH_MASK 0x20000
+#define CB_DEBUG_BUS_5__CC_CACHE_FLUSH__SHIFT 0x11
+#define CB_DEBUG_BUS_5__CC_CACHE_TAGS_FLUSHED_MASK 0x40000
+#define CB_DEBUG_BUS_5__CC_CACHE_TAGS_FLUSHED__SHIFT 0x12
+#define CB_DEBUG_BUS_5__CC_CACHE_SECTORS_FLUSHED_MASK 0x380000
+#define CB_DEBUG_BUS_5__CC_CACHE_SECTORS_FLUSHED__SHIFT 0x13
+#define CB_DEBUG_BUS_6__CC_CACHE_DIRTY_SECTORS_FLUSHED_MASK 0x7
+#define CB_DEBUG_BUS_6__CC_CACHE_DIRTY_SECTORS_FLUSHED__SHIFT 0x0
+#define CB_DEBUG_BUS_6__CM_MC_READ_REQUEST_MASK 0x8
+#define CB_DEBUG_BUS_6__CM_MC_READ_REQUEST__SHIFT 0x3
+#define CB_DEBUG_BUS_6__FC_MC_READ_REQUEST_MASK 0x10
+#define CB_DEBUG_BUS_6__FC_MC_READ_REQUEST__SHIFT 0x4
+#define CB_DEBUG_BUS_6__CC_MC_READ_REQUEST_MASK 0x20
+#define CB_DEBUG_BUS_6__CC_MC_READ_REQUEST__SHIFT 0x5
+#define CB_DEBUG_BUS_6__CM_MC_WRITE_REQUEST_MASK 0x40
+#define CB_DEBUG_BUS_6__CM_MC_WRITE_REQUEST__SHIFT 0x6
+#define CB_DEBUG_BUS_6__FC_MC_WRITE_REQUEST_MASK 0x80
+#define CB_DEBUG_BUS_6__FC_MC_WRITE_REQUEST__SHIFT 0x7
+#define CB_DEBUG_BUS_6__CC_MC_WRITE_REQUEST_MASK 0x100
+#define CB_DEBUG_BUS_6__CC_MC_WRITE_REQUEST__SHIFT 0x8
+#define CB_DEBUG_BUS_6__CM_MC_READ_REQUESTS_IN_FLIGHT_MASK 0x1fe00
+#define CB_DEBUG_BUS_6__CM_MC_READ_REQUESTS_IN_FLIGHT__SHIFT 0x9
+#define CB_DEBUG_BUS_7__FC_MC_READ_REQUESTS_IN_FLIGHT_MASK 0x7ff
+#define CB_DEBUG_BUS_7__FC_MC_READ_REQUESTS_IN_FLIGHT__SHIFT 0x0
+#define CB_DEBUG_BUS_7__CC_MC_READ_REQUESTS_IN_FLIGHT_MASK 0x1ff800
+#define CB_DEBUG_BUS_7__CC_MC_READ_REQUESTS_IN_FLIGHT__SHIFT 0xb
+#define CB_DEBUG_BUS_8__CM_MC_WRITE_REQUESTS_IN_FLIGHT_MASK 0xff
+#define CB_DEBUG_BUS_8__CM_MC_WRITE_REQUESTS_IN_FLIGHT__SHIFT 0x0
+#define CB_DEBUG_BUS_8__FC_MC_WRITE_REQUESTS_IN_FLIGHT_MASK 0x7ff00
+#define CB_DEBUG_BUS_8__FC_MC_WRITE_REQUESTS_IN_FLIGHT__SHIFT 0x8
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_FMASK_COMPRESSION_DISABLE_MASK 0x80000
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_FMASK_COMPRESSION_DISABLE__SHIFT 0x13
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_FMASK_DECOMPRESS_MASK 0x100000
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_FMASK_DECOMPRESS__SHIFT 0x14
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_ELIMINATE_FAST_CLEAR_MASK 0x200000
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_ELIMINATE_FAST_CLEAR__SHIFT 0x15
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_CLEAR_MASK 0x400000
+#define CB_DEBUG_BUS_8__FC_SEQUENCER_CLEAR__SHIFT 0x16
+#define CB_DEBUG_BUS_9__CC_MC_WRITE_REQUESTS_IN_FLIGHT_MASK 0x3ff
+#define CB_DEBUG_BUS_9__CC_MC_WRITE_REQUESTS_IN_FLIGHT__SHIFT 0x0
+#define CB_DEBUG_BUS_9__CC_SURFACE_SYNC_MASK 0x400
+#define CB_DEBUG_BUS_9__CC_SURFACE_SYNC__SHIFT 0xa
+#define CB_DEBUG_BUS_9__TWO_PROBE_QUAD_FRAGMENT_MASK 0x800
+#define CB_DEBUG_BUS_9__TWO_PROBE_QUAD_FRAGMENT__SHIFT 0xb
+#define CB_DEBUG_BUS_9__EXPORT_32_ABGR_QUAD_FRAGMENT_MASK 0x1000
+#define CB_DEBUG_BUS_9__EXPORT_32_ABGR_QUAD_FRAGMENT__SHIFT 0xc
+#define CB_DEBUG_BUS_9__DUAL_SOURCE_COLOR_QUAD_FRAGMENT_MASK 0x2000
+#define CB_DEBUG_BUS_9__DUAL_SOURCE_COLOR_QUAD_FRAGMENT__SHIFT 0xd
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_QUAD_MASK 0x4000
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_QUAD__SHIFT 0xe
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_PIXEL_MASK 0x78000
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_PIXEL__SHIFT 0xf
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_QUAD_FRAGMENT_MASK 0x80000
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_QUAD_FRAGMENT__SHIFT 0x13
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_TILE_MASK 0x100000
+#define CB_DEBUG_BUS_9__DEBUG_BUS_DRAWN_TILE__SHIFT 0x14
+#define CB_DEBUG_BUS_9__EVENT_ALL_MASK 0x200000
+#define CB_DEBUG_BUS_9__EVENT_ALL__SHIFT 0x15
+#define CB_DEBUG_BUS_9__EVENT_CACHE_FLUSH_TS_MASK 0x400000
+#define CB_DEBUG_BUS_9__EVENT_CACHE_FLUSH_TS__SHIFT 0x16
+#define CB_DEBUG_BUS_9__EVENT_CONTEXT_DONE_MASK 0x800000
+#define CB_DEBUG_BUS_9__EVENT_CONTEXT_DONE__SHIFT 0x17
+#define CB_DEBUG_BUS_10__EVENT_CACHE_FLUSH_MASK 0x1
+#define CB_DEBUG_BUS_10__EVENT_CACHE_FLUSH__SHIFT 0x0
+#define CB_DEBUG_BUS_10__EVENT_CACHE_FLUSH_AND_INV_TS_EVENT_MASK 0x2
+#define CB_DEBUG_BUS_10__EVENT_CACHE_FLUSH_AND_INV_TS_EVENT__SHIFT 0x1
+#define CB_DEBUG_BUS_10__EVENT_CACHE_FLUSH_AND_INV_EVENT_MASK 0x4
+#define CB_DEBUG_BUS_10__EVENT_CACHE_FLUSH_AND_INV_EVENT__SHIFT 0x2
+#define CB_DEBUG_BUS_10__EVENT_FLUSH_AND_INV_CB_DATA_TS_MASK 0x8
+#define CB_DEBUG_BUS_10__EVENT_FLUSH_AND_INV_CB_DATA_TS__SHIFT 0x3
+#define CB_DEBUG_BUS_10__EVENT_FLUSH_AND_INV_CB_META_MASK 0x10
+#define CB_DEBUG_BUS_10__EVENT_FLUSH_AND_INV_CB_META__SHIFT 0x4
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XC_MASK 0x20
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XC__SHIFT 0x5
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XD_MASK 0x40
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XD__SHIFT 0x6
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XE_MASK 0x80
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XE__SHIFT 0x7
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XF_MASK 0x100
+#define CB_DEBUG_BUS_10__CMASK_READ_DATA_0XF__SHIFT 0x8
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XC_MASK 0x200
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XC__SHIFT 0x9
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XD_MASK 0x400
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XD__SHIFT 0xa
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XE_MASK 0x800
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XE__SHIFT 0xb
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XF_MASK 0x1000
+#define CB_DEBUG_BUS_10__CMASK_WRITE_DATA_0XF__SHIFT 0xc
+#define CB_DEBUG_BUS_10__CORE_SCLK_VLD_MASK 0x2000
+#define CB_DEBUG_BUS_10__CORE_SCLK_VLD__SHIFT 0xd
+#define CB_DEBUG_BUS_10__REG_SCLK0_VLD_MASK 0x4000
+#define CB_DEBUG_BUS_10__REG_SCLK0_VLD__SHIFT 0xe
+#define CB_DEBUG_BUS_10__REG_SCLK1_VLD_MASK 0x8000
+#define CB_DEBUG_BUS_10__REG_SCLK1_VLD__SHIFT 0xf
+#define CB_DEBUG_BUS_10__MERGE_TILE_ONLY_VALID_READY_MASK 0x10000
+#define CB_DEBUG_BUS_10__MERGE_TILE_ONLY_VALID_READY__SHIFT 0x10
+#define CB_DEBUG_BUS_10__MERGE_TILE_ONLY_VALID_READYB_MASK 0x20000
+#define CB_DEBUG_BUS_10__MERGE_TILE_ONLY_VALID_READYB__SHIFT 0x11
+#define CB_DEBUG_BUS_10__FC_QUAD_RDLAT_FIFO_FULL_MASK 0x40000
+#define CB_DEBUG_BUS_10__FC_QUAD_RDLAT_FIFO_FULL__SHIFT 0x12
+#define CB_DEBUG_BUS_10__FC_TILE_RDLAT_FIFO_FULL_MASK 0x80000
+#define CB_DEBUG_BUS_10__FC_TILE_RDLAT_FIFO_FULL__SHIFT 0x13
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_1_FRAGMENT_BEFORE_UPDATE_MASK 0x100000
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_1_FRAGMENT_BEFORE_UPDATE__SHIFT 0x14
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_2_FRAGMENTS_BEFORE_UPDATE_MASK 0x200000
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_2_FRAGMENTS_BEFORE_UPDATE__SHIFT 0x15
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_3_FRAGMENTS_BEFORE_UPDATE_MASK 0x400000
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_3_FRAGMENTS_BEFORE_UPDATE__SHIFT 0x16
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_4_FRAGMENTS_BEFORE_UPDATE_MASK 0x800000
+#define CB_DEBUG_BUS_10__FOP_QUAD_HAS_4_FRAGMENTS_BEFORE_UPDATE__SHIFT 0x17
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_5_FRAGMENTS_BEFORE_UPDATE_MASK 0x1
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_5_FRAGMENTS_BEFORE_UPDATE__SHIFT 0x0
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_6_FRAGMENTS_BEFORE_UPDATE_MASK 0x2
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_6_FRAGMENTS_BEFORE_UPDATE__SHIFT 0x1
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_7_FRAGMENTS_BEFORE_UPDATE_MASK 0x4
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_7_FRAGMENTS_BEFORE_UPDATE__SHIFT 0x2
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_8_FRAGMENTS_BEFORE_UPDATE_MASK 0x8
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_8_FRAGMENTS_BEFORE_UPDATE__SHIFT 0x3
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_1_FRAGMENT_AFTER_UPDATE_MASK 0x10
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_1_FRAGMENT_AFTER_UPDATE__SHIFT 0x4
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_2_FRAGMENTS_AFTER_UPDATE_MASK 0x20
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_2_FRAGMENTS_AFTER_UPDATE__SHIFT 0x5
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_3_FRAGMENTS_AFTER_UPDATE_MASK 0x40
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_3_FRAGMENTS_AFTER_UPDATE__SHIFT 0x6
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_4_FRAGMENTS_AFTER_UPDATE_MASK 0x80
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_4_FRAGMENTS_AFTER_UPDATE__SHIFT 0x7
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_5_FRAGMENTS_AFTER_UPDATE_MASK 0x100
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_5_FRAGMENTS_AFTER_UPDATE__SHIFT 0x8
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_6_FRAGMENTS_AFTER_UPDATE_MASK 0x200
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_6_FRAGMENTS_AFTER_UPDATE__SHIFT 0x9
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_7_FRAGMENTS_AFTER_UPDATE_MASK 0x400
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_7_FRAGMENTS_AFTER_UPDATE__SHIFT 0xa
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_8_FRAGMENTS_AFTER_UPDATE_MASK 0x800
+#define CB_DEBUG_BUS_11__FOP_QUAD_HAS_8_FRAGMENTS_AFTER_UPDATE__SHIFT 0xb
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_1_FRAGMENT_MASK 0x1000
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_1_FRAGMENT__SHIFT 0xc
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_2_FRAGMENTS_MASK 0x2000
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_2_FRAGMENTS__SHIFT 0xd
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_3_FRAGMENTS_MASK 0x4000
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_3_FRAGMENTS__SHIFT 0xe
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_4_FRAGMENTS_MASK 0x8000
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_4_FRAGMENTS__SHIFT 0xf
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_5_FRAGMENTS_MASK 0x10000
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_5_FRAGMENTS__SHIFT 0x10
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_6_FRAGMENTS_MASK 0x20000
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_6_FRAGMENTS__SHIFT 0x11
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_7_FRAGMENTS_MASK 0x40000
+#define CB_DEBUG_BUS_11__FOP_QUAD_ADDED_7_FRAGMENTS__SHIFT 0x12
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_1_FRAGMENT_MASK 0x80000
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_1_FRAGMENT__SHIFT 0x13
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_2_FRAGMENTS_MASK 0x100000
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_2_FRAGMENTS__SHIFT 0x14
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_3_FRAGMENTS_MASK 0x200000
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_3_FRAGMENTS__SHIFT 0x15
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_4_FRAGMENTS_MASK 0x400000
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_4_FRAGMENTS__SHIFT 0x16
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_5_FRAGMENTS_MASK 0x800000
+#define CB_DEBUG_BUS_11__FOP_QUAD_REMOVED_5_FRAGMENTS__SHIFT 0x17
+#define CB_DEBUG_BUS_12__FOP_QUAD_REMOVED_6_FRAGMENTS_MASK 0x1
+#define CB_DEBUG_BUS_12__FOP_QUAD_REMOVED_6_FRAGMENTS__SHIFT 0x0
+#define CB_DEBUG_BUS_12__FOP_QUAD_REMOVED_7_FRAGMENTS_MASK 0x2
+#define CB_DEBUG_BUS_12__FOP_QUAD_REMOVED_7_FRAGMENTS__SHIFT 0x1
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_0_MASK 0x4
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_0__SHIFT 0x2
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_1_MASK 0x8
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_1__SHIFT 0x3
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_2_MASK 0x10
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_2__SHIFT 0x4
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_3_MASK 0x20
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_3__SHIFT 0x5
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_4_MASK 0x40
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_4__SHIFT 0x6
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_5_MASK 0x80
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_5__SHIFT 0x7
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_6_MASK 0x100
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_6__SHIFT 0x8
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_7_MASK 0x200
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_READS_FRAGMENT_7__SHIFT 0x9
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_0_MASK 0x400
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_0__SHIFT 0xa
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_1_MASK 0x800
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_1__SHIFT 0xb
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_2_MASK 0x1000
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_2__SHIFT 0xc
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_3_MASK 0x2000
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_3__SHIFT 0xd
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_4_MASK 0x4000
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_4__SHIFT 0xe
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_5_MASK 0x8000
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_5__SHIFT 0xf
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_6_MASK 0x10000
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_6__SHIFT 0x10
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_7_MASK 0x20000
+#define CB_DEBUG_BUS_12__FC_CC_QUADFRAG_WRITES_FRAGMENT_7__SHIFT 0x11
+#define CB_DEBUG_BUS_12__FC_QUAD_BLEND_OPT_DONT_READ_DST_MASK 0x40000
+#define CB_DEBUG_BUS_12__FC_QUAD_BLEND_OPT_DONT_READ_DST__SHIFT 0x12
+#define CB_DEBUG_BUS_12__FC_QUAD_BLEND_OPT_BLEND_BYPASS_MASK 0x80000
+#define CB_DEBUG_BUS_12__FC_QUAD_BLEND_OPT_BLEND_BYPASS__SHIFT 0x13
+#define CB_DEBUG_BUS_12__FC_QUAD_BLEND_OPT_DISCARD_PIXELS_MASK 0x100000
+#define CB_DEBUG_BUS_12__FC_QUAD_BLEND_OPT_DISCARD_PIXELS__SHIFT 0x14
+#define CB_DEBUG_BUS_12__FC_QUAD_KILLED_BY_EXTRA_PIXEL_EXPORT_MASK 0x200000
+#define CB_DEBUG_BUS_12__FC_QUAD_KILLED_BY_EXTRA_PIXEL_EXPORT__SHIFT 0x15
+#define CB_DEBUG_BUS_12__FC_QUAD_KILLED_BY_COLOR_INVALID_MASK 0x400000
+#define CB_DEBUG_BUS_12__FC_QUAD_KILLED_BY_COLOR_INVALID__SHIFT 0x16
+#define CB_DEBUG_BUS_12__FC_QUAD_KILLED_BY_NULL_TARGET_SHADER_MASK_MASK 0x800000
+#define CB_DEBUG_BUS_12__FC_QUAD_KILLED_BY_NULL_TARGET_SHADER_MASK__SHIFT 0x17
+#define CB_DEBUG_BUS_13__FC_PF_FC_KEYID_RDLAT_FIFO_FULL_MASK 0x1
+#define CB_DEBUG_BUS_13__FC_PF_FC_KEYID_RDLAT_FIFO_FULL__SHIFT 0x0
+#define CB_DEBUG_BUS_13__FC_DOC_QTILE_CAM_MISS_MASK 0x2
+#define CB_DEBUG_BUS_13__FC_DOC_QTILE_CAM_MISS__SHIFT 0x1
+#define CB_DEBUG_BUS_13__FC_DOC_QTILE_CAM_HIT_MASK 0x4
+#define CB_DEBUG_BUS_13__FC_DOC_QTILE_CAM_HIT__SHIFT 0x2
+#define CB_DEBUG_BUS_13__FC_DOC_CLINE_CAM_MISS_MASK 0x8
+#define CB_DEBUG_BUS_13__FC_DOC_CLINE_CAM_MISS__SHIFT 0x3
+#define CB_DEBUG_BUS_13__FC_DOC_CLINE_CAM_HIT_MASK 0x10
+#define CB_DEBUG_BUS_13__FC_DOC_CLINE_CAM_HIT__SHIFT 0x4
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_1_SECTOR_MASK 0x20
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_1_SECTOR__SHIFT 0x5
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_2_SECTORS_MASK 0x40
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_2_SECTORS__SHIFT 0x6
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_3_SECTORS_MASK 0x80
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_3_SECTORS__SHIFT 0x7
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_4_SECTORS_MASK 0x100
+#define CB_DEBUG_BUS_13__FC_DOC_OVERWROTE_4_SECTORS__SHIFT 0x8
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_HIT_MASK 0x200
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_HIT__SHIFT 0x9
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_TAG_MISS_MASK 0x400
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_TAG_MISS__SHIFT 0xa
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_SECTOR_MISS_MASK 0x800
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_SECTOR_MISS__SHIFT 0xb
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_REEVICTION_STALL_MASK 0x1000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_REEVICTION_STALL__SHIFT 0xc
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_EVICT_NONZERO_INFLIGHT_STALL_MASK 0x2000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_EVICT_NONZERO_INFLIGHT_STALL__SHIFT 0xd
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_REPLACE_PENDING_EVICT_STALL_MASK 0x4000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_REPLACE_PENDING_EVICT_STALL__SHIFT 0xe
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL_MASK 0x8000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL__SHIFT 0xf
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_READ_OUTPUT_STALL_MASK 0x10000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_READ_OUTPUT_STALL__SHIFT 0x10
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_WRITE_OUTPUT_STALL_MASK 0x20000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_WRITE_OUTPUT_STALL__SHIFT 0x11
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_ACK_OUTPUT_STALL_MASK 0x40000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_ACK_OUTPUT_STALL__SHIFT 0x12
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_STALL_MASK 0x80000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_STALL__SHIFT 0x13
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_FLUSH_MASK 0x100000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_FLUSH__SHIFT 0x14
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_SECTORS_FLUSHED_MASK 0x200000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_SECTORS_FLUSHED__SHIFT 0x15
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_DIRTY_SECTORS_FLUSHED_MASK 0x400000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_DIRTY_SECTORS_FLUSHED__SHIFT 0x16
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_TAGS_FLUSHED_MASK 0x800000
+#define CB_DEBUG_BUS_13__FC_PF_DCC_CACHE_TAGS_FLUSHED__SHIFT 0x17
+#define CB_DEBUG_BUS_14__FC_MC_DCC_WRITE_REQUESTS_IN_FLIGHT_MASK 0x7ff
+#define CB_DEBUG_BUS_14__FC_MC_DCC_WRITE_REQUESTS_IN_FLIGHT__SHIFT 0x0
+#define CB_DEBUG_BUS_14__FC_MC_DCC_READ_REQUESTS_IN_FLIGHT_MASK 0x3ff800
+#define CB_DEBUG_BUS_14__FC_MC_DCC_READ_REQUESTS_IN_FLIGHT__SHIFT 0xb
+#define CB_DEBUG_BUS_14__CC_PF_DCC_BEYOND_TILE_SPLIT_MASK 0x400000
+#define CB_DEBUG_BUS_14__CC_PF_DCC_BEYOND_TILE_SPLIT__SHIFT 0x16
+#define CB_DEBUG_BUS_14__CC_PF_DCC_RDREQ_STALL_MASK 0x800000
+#define CB_DEBUG_BUS_14__CC_PF_DCC_RDREQ_STALL__SHIFT 0x17
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_2TO1_MASK 0x7
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_2TO1__SHIFT 0x0
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_4TO1_MASK 0x18
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_4TO1__SHIFT 0x3
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_4TO2_MASK 0x60
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_4TO2__SHIFT 0x5
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_4TO3_MASK 0x180
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_4TO3__SHIFT 0x7
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO1_MASK 0x600
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO1__SHIFT 0x9
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO2_MASK 0x1800
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO2__SHIFT 0xb
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO3_MASK 0x6000
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO3__SHIFT 0xd
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO4_MASK 0x18000
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO4__SHIFT 0xf
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO5_MASK 0x60000
+#define CB_DEBUG_BUS_15__CC_PF_DCC_COMPRESS_RATIO_6TO5__SHIFT 0x11
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO1_MASK 0x1
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO1__SHIFT 0x0
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO2_MASK 0x2
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO2__SHIFT 0x1
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO3_MASK 0x4
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO3__SHIFT 0x2
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO4_MASK 0x8
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO4__SHIFT 0x3
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO5_MASK 0x10
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO5__SHIFT 0x4
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO6_MASK 0x20
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO6__SHIFT 0x5
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO7_MASK 0x40
+#define CB_DEBUG_BUS_16__CC_PF_DCC_COMPRESS_RATIO_8TO7__SHIFT 0x6
+#define CB_DEBUG_BUS_17__TILE_INTFC_BUSY_MASK 0x1
+#define CB_DEBUG_BUS_17__TILE_INTFC_BUSY__SHIFT 0x0
+#define CB_DEBUG_BUS_17__MU_BUSY_MASK 0x2
+#define CB_DEBUG_BUS_17__MU_BUSY__SHIFT 0x1
+#define CB_DEBUG_BUS_17__TQ_BUSY_MASK 0x4
+#define CB_DEBUG_BUS_17__TQ_BUSY__SHIFT 0x2
+#define CB_DEBUG_BUS_17__AC_BUSY_MASK 0x8
+#define CB_DEBUG_BUS_17__AC_BUSY__SHIFT 0x3
+#define CB_DEBUG_BUS_17__CRW_BUSY_MASK 0x10
+#define CB_DEBUG_BUS_17__CRW_BUSY__SHIFT 0x4
+#define CB_DEBUG_BUS_17__CACHE_CTRL_BUSY_MASK 0x20
+#define CB_DEBUG_BUS_17__CACHE_CTRL_BUSY__SHIFT 0x5
+#define CB_DEBUG_BUS_17__MC_WR_PENDING_MASK 0x40
+#define CB_DEBUG_BUS_17__MC_WR_PENDING__SHIFT 0x6
+#define CB_DEBUG_BUS_17__FC_WR_PENDING_MASK 0x80
+#define CB_DEBUG_BUS_17__FC_WR_PENDING__SHIFT 0x7
+#define CB_DEBUG_BUS_17__FC_RD_PENDING_MASK 0x100
+#define CB_DEBUG_BUS_17__FC_RD_PENDING__SHIFT 0x8
+#define CB_DEBUG_BUS_17__EVICT_PENDING_MASK 0x200
+#define CB_DEBUG_BUS_17__EVICT_PENDING__SHIFT 0x9
+#define CB_DEBUG_BUS_17__LAST_RD_ARB_WINNER_MASK 0x400
+#define CB_DEBUG_BUS_17__LAST_RD_ARB_WINNER__SHIFT 0xa
+#define CB_DEBUG_BUS_17__MU_STATE_MASK 0x7f800
+#define CB_DEBUG_BUS_17__MU_STATE__SHIFT 0xb
+#define CB_DEBUG_BUS_18__TILE_RETIREMENT_BUSY_MASK 0x1
+#define CB_DEBUG_BUS_18__TILE_RETIREMENT_BUSY__SHIFT 0x0
+#define CB_DEBUG_BUS_18__FOP_BUSY_MASK 0x2
+#define CB_DEBUG_BUS_18__FOP_BUSY__SHIFT 0x1
+#define CB_DEBUG_BUS_18__CLEAR_BUSY_MASK 0x4
+#define CB_DEBUG_BUS_18__CLEAR_BUSY__SHIFT 0x2
+#define CB_DEBUG_BUS_18__LAT_BUSY_MASK 0x8
+#define CB_DEBUG_BUS_18__LAT_BUSY__SHIFT 0x3
+#define CB_DEBUG_BUS_18__CACHE_CTL_BUSY_MASK 0x10
+#define CB_DEBUG_BUS_18__CACHE_CTL_BUSY__SHIFT 0x4
+#define CB_DEBUG_BUS_18__ADDR_BUSY_MASK 0x20
+#define CB_DEBUG_BUS_18__ADDR_BUSY__SHIFT 0x5
+#define CB_DEBUG_BUS_18__MERGE_BUSY_MASK 0x40
+#define CB_DEBUG_BUS_18__MERGE_BUSY__SHIFT 0x6
+#define CB_DEBUG_BUS_18__QUAD_BUSY_MASK 0x80
+#define CB_DEBUG_BUS_18__QUAD_BUSY__SHIFT 0x7
+#define CB_DEBUG_BUS_18__TILE_BUSY_MASK 0x100
+#define CB_DEBUG_BUS_18__TILE_BUSY__SHIFT 0x8
+#define CB_DEBUG_BUS_18__DCC_BUSY_MASK 0x200
+#define CB_DEBUG_BUS_18__DCC_BUSY__SHIFT 0x9
+#define CB_DEBUG_BUS_18__DOC_BUSY_MASK 0x400
+#define CB_DEBUG_BUS_18__DOC_BUSY__SHIFT 0xa
+#define CB_DEBUG_BUS_18__DAG_BUSY_MASK 0x800
+#define CB_DEBUG_BUS_18__DAG_BUSY__SHIFT 0xb
+#define CB_DEBUG_BUS_18__DOC_STALL_MASK 0x1000
+#define CB_DEBUG_BUS_18__DOC_STALL__SHIFT 0xc
+#define CB_DEBUG_BUS_18__DOC_QT_CAM_FULL_MASK 0x2000
+#define CB_DEBUG_BUS_18__DOC_QT_CAM_FULL__SHIFT 0xd
+#define CB_DEBUG_BUS_18__DOC_CL_CAM_FULL_MASK 0x4000
+#define CB_DEBUG_BUS_18__DOC_CL_CAM_FULL__SHIFT 0xe
+#define CB_DEBUG_BUS_18__DOC_QUAD_PTR_FIFO_FULL_MASK 0x8000
+#define CB_DEBUG_BUS_18__DOC_QUAD_PTR_FIFO_FULL__SHIFT 0xf
+#define CB_DEBUG_BUS_18__DOC_SECTOR_MASK_FIFO_FULL_MASK 0x10000
+#define CB_DEBUG_BUS_18__DOC_SECTOR_MASK_FIFO_FULL__SHIFT 0x10
+#define CB_DEBUG_BUS_18__DCS_READ_WINNER_LAST_MASK 0x20000
+#define CB_DEBUG_BUS_18__DCS_READ_WINNER_LAST__SHIFT 0x11
+#define CB_DEBUG_BUS_18__DCS_READ_EV_PENDING_MASK 0x40000
+#define CB_DEBUG_BUS_18__DCS_READ_EV_PENDING__SHIFT 0x12
+#define CB_DEBUG_BUS_18__DCS_WRITE_CC_PENDING_MASK 0x80000
+#define CB_DEBUG_BUS_18__DCS_WRITE_CC_PENDING__SHIFT 0x13
+#define CB_DEBUG_BUS_18__DCS_READ_CC_PENDING_MASK 0x100000
+#define CB_DEBUG_BUS_18__DCS_READ_CC_PENDING__SHIFT 0x14
+#define CB_DEBUG_BUS_18__DCS_WRITE_MC_PENDING_MASK 0x200000
+#define CB_DEBUG_BUS_18__DCS_WRITE_MC_PENDING__SHIFT 0x15
+#define CB_DEBUG_BUS_19__SURF_SYNC_STATE_MASK 0x3
+#define CB_DEBUG_BUS_19__SURF_SYNC_STATE__SHIFT 0x0
+#define CB_DEBUG_BUS_19__SURF_SYNC_START_MASK 0x4
+#define CB_DEBUG_BUS_19__SURF_SYNC_START__SHIFT 0x2
+#define CB_DEBUG_BUS_19__SF_BUSY_MASK 0x8
+#define CB_DEBUG_BUS_19__SF_BUSY__SHIFT 0x3
+#define CB_DEBUG_BUS_19__CS_BUSY_MASK 0x10
+#define CB_DEBUG_BUS_19__CS_BUSY__SHIFT 0x4
+#define CB_DEBUG_BUS_19__RB_BUSY_MASK 0x20
+#define CB_DEBUG_BUS_19__RB_BUSY__SHIFT 0x5
+#define CB_DEBUG_BUS_19__DS_BUSY_MASK 0x40
+#define CB_DEBUG_BUS_19__DS_BUSY__SHIFT 0x6
+#define CB_DEBUG_BUS_19__TB_BUSY_MASK 0x80
+#define CB_DEBUG_BUS_19__TB_BUSY__SHIFT 0x7
+#define CB_DEBUG_BUS_19__IB_BUSY_MASK 0x100
+#define CB_DEBUG_BUS_19__IB_BUSY__SHIFT 0x8
+#define CB_DEBUG_BUS_19__DRR_BUSY_MASK 0x200
+#define CB_DEBUG_BUS_19__DRR_BUSY__SHIFT 0x9
+#define CB_DEBUG_BUS_19__DF_BUSY_MASK 0x400
+#define CB_DEBUG_BUS_19__DF_BUSY__SHIFT 0xa
+#define CB_DEBUG_BUS_19__DD_BUSY_MASK 0x800
+#define CB_DEBUG_BUS_19__DD_BUSY__SHIFT 0xb
+#define CB_DEBUG_BUS_19__DC_BUSY_MASK 0x1000
+#define CB_DEBUG_BUS_19__DC_BUSY__SHIFT 0xc
+#define CB_DEBUG_BUS_19__DK_BUSY_MASK 0x2000
+#define CB_DEBUG_BUS_19__DK_BUSY__SHIFT 0xd
+#define CB_DEBUG_BUS_19__DF_SKID_FIFO_EMPTY_MASK 0x4000
+#define CB_DEBUG_BUS_19__DF_SKID_FIFO_EMPTY__SHIFT 0xe
+#define CB_DEBUG_BUS_19__DF_CLEAR_FIFO_EMPTY_MASK 0x8000
+#define CB_DEBUG_BUS_19__DF_CLEAR_FIFO_EMPTY__SHIFT 0xf
+#define CB_DEBUG_BUS_19__DD_READY_MASK 0x10000
+#define CB_DEBUG_BUS_19__DD_READY__SHIFT 0x10
+#define CB_DEBUG_BUS_19__DC_FIFO_FULL_MASK 0x20000
+#define CB_DEBUG_BUS_19__DC_FIFO_FULL__SHIFT 0x11
+#define CB_DEBUG_BUS_19__DC_READY_MASK 0x40000
+#define CB_DEBUG_BUS_19__DC_READY__SHIFT 0x12
+#define CB_DEBUG_BUS_20__MC_RDREQ_CREDITS_MASK 0x3f
+#define CB_DEBUG_BUS_20__MC_RDREQ_CREDITS__SHIFT 0x0
+#define CB_DEBUG_BUS_20__MC_WRREQ_CREDITS_MASK 0xfc0
+#define CB_DEBUG_BUS_20__MC_WRREQ_CREDITS__SHIFT 0x6
+#define CB_DEBUG_BUS_20__CC_RDREQ_HAD_ITS_TURN_MASK 0x1000
+#define CB_DEBUG_BUS_20__CC_RDREQ_HAD_ITS_TURN__SHIFT 0xc
+#define CB_DEBUG_BUS_20__FC_RDREQ_HAD_ITS_TURN_MASK 0x2000
+#define CB_DEBUG_BUS_20__FC_RDREQ_HAD_ITS_TURN__SHIFT 0xd
+#define CB_DEBUG_BUS_20__CM_RDREQ_HAD_ITS_TURN_MASK 0x4000
+#define CB_DEBUG_BUS_20__CM_RDREQ_HAD_ITS_TURN__SHIFT 0xe
+#define CB_DEBUG_BUS_20__CC_WRREQ_HAD_ITS_TURN_MASK 0x10000
+#define CB_DEBUG_BUS_20__CC_WRREQ_HAD_ITS_TURN__SHIFT 0x10
+#define CB_DEBUG_BUS_20__FC_WRREQ_HAD_ITS_TURN_MASK 0x20000
+#define CB_DEBUG_BUS_20__FC_WRREQ_HAD_ITS_TURN__SHIFT 0x11
+#define CB_DEBUG_BUS_20__CM_WRREQ_HAD_ITS_TURN_MASK 0x40000
+#define CB_DEBUG_BUS_20__CM_WRREQ_HAD_ITS_TURN__SHIFT 0x12
+#define CB_DEBUG_BUS_20__CC_WRREQ_FIFO_EMPTY_MASK 0x100000
+#define CB_DEBUG_BUS_20__CC_WRREQ_FIFO_EMPTY__SHIFT 0x14
+#define CB_DEBUG_BUS_20__FC_WRREQ_FIFO_EMPTY_MASK 0x200000
+#define CB_DEBUG_BUS_20__FC_WRREQ_FIFO_EMPTY__SHIFT 0x15
+#define CB_DEBUG_BUS_20__CM_WRREQ_FIFO_EMPTY_MASK 0x400000
+#define CB_DEBUG_BUS_20__CM_WRREQ_FIFO_EMPTY__SHIFT 0x16
+#define CB_DEBUG_BUS_20__DCC_WRREQ_FIFO_EMPTY_MASK 0x800000
+#define CB_DEBUG_BUS_20__DCC_WRREQ_FIFO_EMPTY__SHIFT 0x17
+#define CB_DEBUG_BUS_21__CM_BUSY_MASK 0x1
+#define CB_DEBUG_BUS_21__CM_BUSY__SHIFT 0x0
+#define CB_DEBUG_BUS_21__FC_BUSY_MASK 0x2
+#define CB_DEBUG_BUS_21__FC_BUSY__SHIFT 0x1
+#define CB_DEBUG_BUS_21__CC_BUSY_MASK 0x4
+#define CB_DEBUG_BUS_21__CC_BUSY__SHIFT 0x2
+#define CB_DEBUG_BUS_21__BB_BUSY_MASK 0x8
+#define CB_DEBUG_BUS_21__BB_BUSY__SHIFT 0x3
+#define CB_DEBUG_BUS_21__MA_BUSY_MASK 0x10
+#define CB_DEBUG_BUS_21__MA_BUSY__SHIFT 0x4
+#define CB_DEBUG_BUS_21__CORE_SCLK_VLD_MASK 0x20
+#define CB_DEBUG_BUS_21__CORE_SCLK_VLD__SHIFT 0x5
+#define CB_DEBUG_BUS_21__REG_SCLK1_VLD_MASK 0x40
+#define CB_DEBUG_BUS_21__REG_SCLK1_VLD__SHIFT 0x6
+#define CB_DEBUG_BUS_21__REG_SCLK0_VLD_MASK 0x80
+#define CB_DEBUG_BUS_21__REG_SCLK0_VLD__SHIFT 0x7
+#define CB_DEBUG_BUS_22__OUTSTANDING_MC_READS_MASK 0xfff
+#define CB_DEBUG_BUS_22__OUTSTANDING_MC_READS__SHIFT 0x0
+#define CB_DEBUG_BUS_22__OUTSTANDING_MC_WRITES_MASK 0xfff000
+#define CB_DEBUG_BUS_22__OUTSTANDING_MC_WRITES__SHIFT 0xc
+#define CP_DFY_CNTL__POLICY_MASK 0x1
+#define CP_DFY_CNTL__POLICY__SHIFT 0x0
+#define CP_DFY_CNTL__MTYPE_MASK 0xc
+#define CP_DFY_CNTL__MTYPE__SHIFT 0x2
+#define CP_DFY_CNTL__LFSR_RESET_MASK 0x10000000
+#define CP_DFY_CNTL__LFSR_RESET__SHIFT 0x1c
+#define CP_DFY_CNTL__MODE_MASK 0x60000000
+#define CP_DFY_CNTL__MODE__SHIFT 0x1d
+#define CP_DFY_CNTL__ENABLE_MASK 0x80000000
+#define CP_DFY_CNTL__ENABLE__SHIFT 0x1f
+#define CP_DFY_STAT__BURST_COUNT_MASK 0xffff
+#define CP_DFY_STAT__BURST_COUNT__SHIFT 0x0
+#define CP_DFY_STAT__TAGS_PENDING_MASK 0x1ff0000
+#define CP_DFY_STAT__TAGS_PENDING__SHIFT 0x10
+#define CP_DFY_STAT__BUSY_MASK 0x80000000
+#define CP_DFY_STAT__BUSY__SHIFT 0x1f
+#define CP_DFY_ADDR_HI__ADDR_HI_MASK 0xffffffff
+#define CP_DFY_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_DFY_ADDR_LO__ADDR_LO_MASK 0xffffffe0
+#define CP_DFY_ADDR_LO__ADDR_LO__SHIFT 0x5
+#define CP_DFY_DATA_0__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_0__DATA__SHIFT 0x0
+#define CP_DFY_DATA_1__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_1__DATA__SHIFT 0x0
+#define CP_DFY_DATA_2__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_2__DATA__SHIFT 0x0
+#define CP_DFY_DATA_3__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_3__DATA__SHIFT 0x0
+#define CP_DFY_DATA_4__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_4__DATA__SHIFT 0x0
+#define CP_DFY_DATA_5__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_5__DATA__SHIFT 0x0
+#define CP_DFY_DATA_6__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_6__DATA__SHIFT 0x0
+#define CP_DFY_DATA_7__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_7__DATA__SHIFT 0x0
+#define CP_DFY_DATA_8__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_8__DATA__SHIFT 0x0
+#define CP_DFY_DATA_9__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_9__DATA__SHIFT 0x0
+#define CP_DFY_DATA_10__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_10__DATA__SHIFT 0x0
+#define CP_DFY_DATA_11__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_11__DATA__SHIFT 0x0
+#define CP_DFY_DATA_12__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_12__DATA__SHIFT 0x0
+#define CP_DFY_DATA_13__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_13__DATA__SHIFT 0x0
+#define CP_DFY_DATA_14__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_14__DATA__SHIFT 0x0
+#define CP_DFY_DATA_15__DATA_MASK 0xffffffff
+#define CP_DFY_DATA_15__DATA__SHIFT 0x0
+#define CP_DFY_CMD__OFFSET_MASK 0x1ff
+#define CP_DFY_CMD__OFFSET__SHIFT 0x0
+#define CP_DFY_CMD__SIZE_MASK 0xffff0000
+#define CP_DFY_CMD__SIZE__SHIFT 0x10
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD_MASK 0xff
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD__SHIFT 0x0
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD_MASK 0xff00
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD__SHIFT 0x8
+#define CP_ATCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK 0x3ff
+#define CP_ATCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT 0x0
+#define CP_RB0_BASE__RB_BASE_MASK 0xffffffff
+#define CP_RB0_BASE__RB_BASE__SHIFT 0x0
+#define CP_RB0_BASE_HI__RB_BASE_HI_MASK 0xff
+#define CP_RB0_BASE_HI__RB_BASE_HI__SHIFT 0x0
+#define CP_RB_BASE__RB_BASE_MASK 0xffffffff
+#define CP_RB_BASE__RB_BASE__SHIFT 0x0
+#define CP_RB1_BASE__RB_BASE_MASK 0xffffffff
+#define CP_RB1_BASE__RB_BASE__SHIFT 0x0
+#define CP_RB1_BASE_HI__RB_BASE_HI_MASK 0xff
+#define CP_RB1_BASE_HI__RB_BASE_HI__SHIFT 0x0
+#define CP_RB2_BASE__RB_BASE_MASK 0xffffffff
+#define CP_RB2_BASE__RB_BASE__SHIFT 0x0
+#define CP_RB0_CNTL__RB_BUFSZ_MASK 0x3f
+#define CP_RB0_CNTL__RB_BUFSZ__SHIFT 0x0
+#define CP_RB0_CNTL__RB_BLKSZ_MASK 0x3f00
+#define CP_RB0_CNTL__RB_BLKSZ__SHIFT 0x8
+#define CP_RB0_CNTL__MTYPE_MASK 0x18000
+#define CP_RB0_CNTL__MTYPE__SHIFT 0xf
+#define CP_RB0_CNTL__BUF_SWAP_MASK 0x60000
+#define CP_RB0_CNTL__BUF_SWAP__SHIFT 0x11
+#define CP_RB0_CNTL__MIN_AVAILSZ_MASK 0x300000
+#define CP_RB0_CNTL__MIN_AVAILSZ__SHIFT 0x14
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16
+#define CP_RB0_CNTL__CACHE_POLICY_MASK 0x1000000
+#define CP_RB0_CNTL__CACHE_POLICY__SHIFT 0x18
+#define CP_RB0_CNTL__RB_NO_UPDATE_MASK 0x8000000
+#define CP_RB0_CNTL__RB_NO_UPDATE__SHIFT 0x1b
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f
+#define CP_RB_CNTL__RB_BUFSZ_MASK 0x3f
+#define CP_RB_CNTL__RB_BUFSZ__SHIFT 0x0
+#define CP_RB_CNTL__RB_BLKSZ_MASK 0x3f00
+#define CP_RB_CNTL__RB_BLKSZ__SHIFT 0x8
+#define CP_RB_CNTL__MTYPE_MASK 0x18000
+#define CP_RB_CNTL__MTYPE__SHIFT 0xf
+#define CP_RB_CNTL__BUF_SWAP_MASK 0x60000
+#define CP_RB_CNTL__BUF_SWAP__SHIFT 0x11
+#define CP_RB_CNTL__MIN_AVAILSZ_MASK 0x300000
+#define CP_RB_CNTL__MIN_AVAILSZ__SHIFT 0x14
+#define CP_RB_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000
+#define CP_RB_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16
+#define CP_RB_CNTL__CACHE_POLICY_MASK 0x1000000
+#define CP_RB_CNTL__CACHE_POLICY__SHIFT 0x18
+#define CP_RB_CNTL__RB_NO_UPDATE_MASK 0x8000000
+#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT 0x1b
+#define CP_RB_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000
+#define CP_RB_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f
+#define CP_RB1_CNTL__RB_BUFSZ_MASK 0x3f
+#define CP_RB1_CNTL__RB_BUFSZ__SHIFT 0x0
+#define CP_RB1_CNTL__RB_BLKSZ_MASK 0x3f00
+#define CP_RB1_CNTL__RB_BLKSZ__SHIFT 0x8
+#define CP_RB1_CNTL__MTYPE_MASK 0x18000
+#define CP_RB1_CNTL__MTYPE__SHIFT 0xf
+#define CP_RB1_CNTL__MIN_AVAILSZ_MASK 0x300000
+#define CP_RB1_CNTL__MIN_AVAILSZ__SHIFT 0x14
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16
+#define CP_RB1_CNTL__CACHE_POLICY_MASK 0x1000000
+#define CP_RB1_CNTL__CACHE_POLICY__SHIFT 0x18
+#define CP_RB1_CNTL__RB_NO_UPDATE_MASK 0x8000000
+#define CP_RB1_CNTL__RB_NO_UPDATE__SHIFT 0x1b
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f
+#define CP_RB2_CNTL__RB_BUFSZ_MASK 0x3f
+#define CP_RB2_CNTL__RB_BUFSZ__SHIFT 0x0
+#define CP_RB2_CNTL__RB_BLKSZ_MASK 0x3f00
+#define CP_RB2_CNTL__RB_BLKSZ__SHIFT 0x8
+#define CP_RB2_CNTL__MTYPE_MASK 0x18000
+#define CP_RB2_CNTL__MTYPE__SHIFT 0xf
+#define CP_RB2_CNTL__MIN_AVAILSZ_MASK 0x300000
+#define CP_RB2_CNTL__MIN_AVAILSZ__SHIFT 0x14
+#define CP_RB2_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000
+#define CP_RB2_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16
+#define CP_RB2_CNTL__CACHE_POLICY_MASK 0x1000000
+#define CP_RB2_CNTL__CACHE_POLICY__SHIFT 0x18
+#define CP_RB2_CNTL__RB_NO_UPDATE_MASK 0x8000000
+#define CP_RB2_CNTL__RB_NO_UPDATE__SHIFT 0x1b
+#define CP_RB2_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000
+#define CP_RB2_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f
+#define CP_RB_RPTR_WR__RB_RPTR_WR_MASK 0xfffff
+#define CP_RB_RPTR_WR__RB_RPTR_WR__SHIFT 0x0
+#define CP_RB0_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3
+#define CP_RB0_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2
+#define CP_RB_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3
+#define CP_RB_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2
+#define CP_RB1_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3
+#define CP_RB1_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2
+#define CP_RB2_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3
+#define CP_RB2_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0
+#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc
+#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0
+#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff
+#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0
+#define CP_RB0_WPTR__RB_WPTR_MASK 0xfffff
+#define CP_RB0_WPTR__RB_WPTR__SHIFT 0x0
+#define CP_RB_WPTR__RB_WPTR_MASK 0xfffff
+#define CP_RB_WPTR__RB_WPTR__SHIFT 0x0
+#define CP_RB1_WPTR__RB_WPTR_MASK 0xfffff
+#define CP_RB1_WPTR__RB_WPTR__SHIFT 0x0
+#define CP_RB2_WPTR__RB_WPTR_MASK 0xfffff
+#define CP_RB2_WPTR__RB_WPTR__SHIFT 0x0
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO_MASK 0xfffffffc
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO__SHIFT 0x2
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI_MASK 0xff
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI__SHIFT 0x0
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE_MASK 0x800
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT 0xb
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE_MASK 0x40000
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE__SHIFT 0x12
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK 0x80000
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE__SHIFT 0x13
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK 0x100000
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE_MASK 0x200000
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE__SHIFT 0x15
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE_MASK 0x400000
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE__SHIFT 0x16
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE_MASK 0x800
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT 0xb
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE_MASK 0x40000
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE__SHIFT 0x12
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK 0x80000
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE__SHIFT 0x13
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK 0x100000
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE_MASK 0x200000
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE__SHIFT 0x15
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK 0x400000
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE__SHIFT 0x16
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_INT_CNTL_RING1__CP_VM_DOORBELL_WR_INT_ENABLE_MASK 0x800
+#define CP_INT_CNTL_RING1__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT 0xb
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_INT_CNTL_RING1__CMP_BUSY_INT_ENABLE_MASK 0x40000
+#define CP_INT_CNTL_RING1__CMP_BUSY_INT_ENABLE__SHIFT 0x12
+#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE_MASK 0x80000
+#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE__SHIFT 0x13
+#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE_MASK 0x100000
+#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14
+#define CP_INT_CNTL_RING1__GFX_IDLE_INT_ENABLE_MASK 0x200000
+#define CP_INT_CNTL_RING1__GFX_IDLE_INT_ENABLE__SHIFT 0x15
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE_MASK 0x400000
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE__SHIFT 0x16
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_INT_CNTL_RING2__CP_VM_DOORBELL_WR_INT_ENABLE_MASK 0x800
+#define CP_INT_CNTL_RING2__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT 0xb
+#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_INT_CNTL_RING2__CMP_BUSY_INT_ENABLE_MASK 0x40000
+#define CP_INT_CNTL_RING2__CMP_BUSY_INT_ENABLE__SHIFT 0x12
+#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE_MASK 0x80000
+#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE__SHIFT 0x13
+#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE_MASK 0x100000
+#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14
+#define CP_INT_CNTL_RING2__GFX_IDLE_INT_ENABLE_MASK 0x200000
+#define CP_INT_CNTL_RING2__GFX_IDLE_INT_ENABLE__SHIFT 0x15
+#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE_MASK 0x400000
+#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE__SHIFT 0x16
+#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT_MASK 0x800
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT__SHIFT 0xb
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT_MASK 0x4000
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT__SHIFT 0xe
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT_MASK 0x40000
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT__SHIFT 0x12
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT_MASK 0x80000
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT__SHIFT 0x13
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT_MASK 0x100000
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT__SHIFT 0x14
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT_MASK 0x200000
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT__SHIFT 0x15
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT_MASK 0x400000
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT__SHIFT 0x16
+#define CP_INT_STATUS__PRIV_REG_INT_STAT_MASK 0x800000
+#define CP_INT_STATUS__PRIV_REG_INT_STAT__SHIFT 0x17
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT_MASK 0x1000000
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT__SHIFT 0x18
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT_MASK 0x4000000
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT__SHIFT 0x1a
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b
+#define CP_INT_STATUS__GENERIC2_INT_STAT_MASK 0x20000000
+#define CP_INT_STATUS__GENERIC2_INT_STAT__SHIFT 0x1d
+#define CP_INT_STATUS__GENERIC1_INT_STAT_MASK 0x40000000
+#define CP_INT_STATUS__GENERIC1_INT_STAT__SHIFT 0x1e
+#define CP_INT_STATUS__GENERIC0_INT_STAT_MASK 0x80000000
+#define CP_INT_STATUS__GENERIC0_INT_STAT__SHIFT 0x1f
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT_MASK 0x800
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT__SHIFT 0xb
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT_MASK 0x4000
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT__SHIFT 0xe
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT_MASK 0x40000
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT__SHIFT 0x12
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT_MASK 0x80000
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT__SHIFT 0x13
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT_MASK 0x100000
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT__SHIFT 0x14
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT_MASK 0x200000
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT__SHIFT 0x15
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT_MASK 0x400000
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT__SHIFT 0x16
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT_MASK 0x800000
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT__SHIFT 0x17
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT_MASK 0x1000000
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT__SHIFT 0x18
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT_MASK 0x4000000
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT__SHIFT 0x1a
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT_MASK 0x20000000
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT__SHIFT 0x1d
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT_MASK 0x40000000
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT__SHIFT 0x1e
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT_MASK 0x80000000
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT__SHIFT 0x1f
+#define CP_INT_STATUS_RING1__CP_VM_DOORBELL_WR_INT_STAT_MASK 0x800
+#define CP_INT_STATUS_RING1__CP_VM_DOORBELL_WR_INT_STAT__SHIFT 0xb
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT_MASK 0x4000
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT__SHIFT 0xe
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11
+#define CP_INT_STATUS_RING1__CMP_BUSY_INT_STAT_MASK 0x40000
+#define CP_INT_STATUS_RING1__CMP_BUSY_INT_STAT__SHIFT 0x12
+#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT_MASK 0x80000
+#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT__SHIFT 0x13
+#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT_MASK 0x100000
+#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT__SHIFT 0x14
+#define CP_INT_STATUS_RING1__GFX_IDLE_INT_STAT_MASK 0x200000
+#define CP_INT_STATUS_RING1__GFX_IDLE_INT_STAT__SHIFT 0x15
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT_MASK 0x400000
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT__SHIFT 0x16
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT_MASK 0x800000
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT__SHIFT 0x17
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT_MASK 0x1000000
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT__SHIFT 0x18
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT_MASK 0x4000000
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT__SHIFT 0x1a
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT_MASK 0x20000000
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT__SHIFT 0x1d
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT_MASK 0x40000000
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT__SHIFT 0x1e
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT_MASK 0x80000000
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT__SHIFT 0x1f
+#define CP_INT_STATUS_RING2__CP_VM_DOORBELL_WR_INT_STAT_MASK 0x800
+#define CP_INT_STATUS_RING2__CP_VM_DOORBELL_WR_INT_STAT__SHIFT 0xb
+#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT_MASK 0x4000
+#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT__SHIFT 0xe
+#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000
+#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11
+#define CP_INT_STATUS_RING2__CMP_BUSY_INT_STAT_MASK 0x40000
+#define CP_INT_STATUS_RING2__CMP_BUSY_INT_STAT__SHIFT 0x12
+#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT_MASK 0x80000
+#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT__SHIFT 0x13
+#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT_MASK 0x100000
+#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT__SHIFT 0x14
+#define CP_INT_STATUS_RING2__GFX_IDLE_INT_STAT_MASK 0x200000
+#define CP_INT_STATUS_RING2__GFX_IDLE_INT_STAT__SHIFT 0x15
+#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT_MASK 0x400000
+#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT__SHIFT 0x16
+#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT_MASK 0x800000
+#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT__SHIFT 0x17
+#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT_MASK 0x1000000
+#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT__SHIFT 0x18
+#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT_MASK 0x4000000
+#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT__SHIFT 0x1a
+#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000
+#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b
+#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT_MASK 0x20000000
+#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT__SHIFT 0x1d
+#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT_MASK 0x40000000
+#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT__SHIFT 0x1e
+#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT_MASK 0x80000000
+#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT__SHIFT 0x1f
+#define CP_DEVICE_ID__DEVICE_ID_MASK 0xff
+#define CP_DEVICE_ID__DEVICE_ID__SHIFT 0x0
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18
+#define CP_RING0_PRIORITY__PRIORITY_MASK 0x3
+#define CP_RING0_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_RING1_PRIORITY__PRIORITY_MASK 0x3
+#define CP_RING1_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME0_PIPE1_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME0_PIPE1_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_RING2_PRIORITY__PRIORITY_MASK 0x3
+#define CP_RING2_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME0_PIPE2_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME0_PIPE2_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ENDIAN_SWAP__ENDIAN_SWAP_MASK 0x3
+#define CP_ENDIAN_SWAP__ENDIAN_SWAP__SHIFT 0x0
+#define CP_RB_VMID__RB0_VMID_MASK 0xf
+#define CP_RB_VMID__RB0_VMID__SHIFT 0x0
+#define CP_RB_VMID__RB1_VMID_MASK 0xf00
+#define CP_RB_VMID__RB1_VMID__SHIFT 0x8
+#define CP_RB_VMID__RB2_VMID_MASK 0xf0000
+#define CP_RB_VMID__RB2_VMID__SHIFT 0x10
+#define CP_ME0_PIPE0_VMID__VMID_MASK 0xf
+#define CP_ME0_PIPE0_VMID__VMID__SHIFT 0x0
+#define CP_ME0_PIPE1_VMID__VMID_MASK 0xf
+#define CP_ME0_PIPE1_VMID__VMID__SHIFT 0x0
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK 0x7ffffc
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT 0x2
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN_MASK 0x40000000
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN__SHIFT 0x1e
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT_MASK 0x80000000
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT 0x1f
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK 0x7ffffc
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT 0x2
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK 0x7ffffc
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT 0x2
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK 0x7ffffc
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT 0x2
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK 0x7ffffc
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT 0x2
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR_MASK 0x1fff
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0
+#define CP_PFP_UCODE_DATA__UCODE_DATA_MASK 0xffffffff
+#define CP_PFP_UCODE_DATA__UCODE_DATA__SHIFT 0x0
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR_MASK 0x1fff
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR__SHIFT 0x0
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR_MASK 0x1fff
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR__SHIFT 0x0
+#define CP_ME_RAM_DATA__ME_RAM_DATA_MASK 0xffffffff
+#define CP_ME_RAM_DATA__ME_RAM_DATA__SHIFT 0x0
+#define CGTT_CPC_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_CPC_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK 0x20000000
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT 0x1d
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f
+#define CGTT_CPF_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_CPF_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK 0x20000000
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT 0x1d
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f
+#define CGTT_CP_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_CP_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK 0x20000000
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT 0x1d
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f
+#define CP_CE_UCODE_ADDR__UCODE_ADDR_MASK 0xfff
+#define CP_CE_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0
+#define CP_CE_UCODE_DATA__UCODE_DATA_MASK 0xffffffff
+#define CP_CE_UCODE_DATA__UCODE_DATA__SHIFT 0x0
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR_MASK 0x1ffff
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA_MASK 0xffffffff
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA__SHIFT 0x0
+#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR_MASK 0x1ffff
+#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0
+#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA_MASK 0xffffffff
+#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA__SHIFT 0x0
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT_MASK 0x2
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT__SHIFT 0x1
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT_MASK 0x2
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT__SHIFT 0x1
+#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT_MASK 0x2
+#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT__SHIFT 0x1
+#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT_MASK 0x1
+#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT 0x0
+#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT_MASK 0x2
+#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT__SHIFT 0x1
+#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK 0x4
+#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT 0x2
+#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT_MASK 0x8
+#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT__SHIFT 0x3
+#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT_MASK 0x10
+#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT 0x4
+#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK 0x20
+#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT 0x5
+#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT_MASK 0x40
+#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT 0x6
+#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT_MASK 0x80
+#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT 0x7
+#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT_MASK 0x100
+#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT 0x8
+#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT_MASK 0x200
+#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT__SHIFT 0x9
+#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT_MASK 0x1
+#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT 0x0
+#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT_MASK 0x2
+#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT__SHIFT 0x1
+#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK 0x4
+#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT 0x2
+#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT_MASK 0x8
+#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT__SHIFT 0x3
+#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT_MASK 0x10
+#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT 0x4
+#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK 0x20
+#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT 0x5
+#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT_MASK 0x40
+#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT 0x6
+#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT_MASK 0x80
+#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT 0x7
+#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT_MASK 0x100
+#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT 0x8
+#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT_MASK 0x200
+#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT__SHIFT 0x9
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0_MASK 0x1
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0__SHIFT 0x0
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1_MASK 0x2
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1__SHIFT 0x1
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0_MASK 0x100
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0__SHIFT 0x8
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1_MASK 0x200
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1__SHIFT 0x9
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2_MASK 0x400
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2__SHIFT 0xa
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3_MASK 0x800
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3__SHIFT 0xb
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0_MASK 0x10000
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0__SHIFT 0x10
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1_MASK 0x20000
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1__SHIFT 0x11
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2_MASK 0x40000
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2__SHIFT 0x12
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3_MASK 0x80000
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3__SHIFT 0x13
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK 0x1
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN__SHIFT 0x0
+#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN_MASK 0x2
+#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN__SHIFT 0x1
+#define CP_MEM_SLP_CNTL__RESERVED_MASK 0x7c
+#define CP_MEM_SLP_CNTL__RESERVED__SHIFT 0x2
+#define CP_MEM_SLP_CNTL__CP_LS_DS_BUSY_OVERRIDE_MASK 0x80
+#define CP_MEM_SLP_CNTL__CP_LS_DS_BUSY_OVERRIDE__SHIFT 0x7
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY_MASK 0xff00
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY__SHIFT 0x8
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY_MASK 0xff0000
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY__SHIFT 0x10
+#define CP_MEM_SLP_CNTL__RESERVED1_MASK 0xff000000
+#define CP_MEM_SLP_CNTL__RESERVED1__SHIFT 0x18
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE_MASK 0x3
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE__SHIFT 0x0
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT_MASK 0xf0
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT__SHIFT 0x4
+#define CP_ECC_FIRSTOCCURRENCE__ME_MASK 0x300
+#define CP_ECC_FIRSTOCCURRENCE__ME__SHIFT 0x8
+#define CP_ECC_FIRSTOCCURRENCE__PIPE_MASK 0xc00
+#define CP_ECC_FIRSTOCCURRENCE__PIPE__SHIFT 0xa
+#define CP_ECC_FIRSTOCCURRENCE__QUEUE_MASK 0x7000
+#define CP_ECC_FIRSTOCCURRENCE__QUEUE__SHIFT 0xc
+#define CP_ECC_FIRSTOCCURRENCE__VMID_MASK 0xf0000
+#define CP_ECC_FIRSTOCCURRENCE__VMID__SHIFT 0x10
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE_MASK 0xffffffff
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE__SHIFT 0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE_MASK 0xffffffff
+#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE__SHIFT 0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING2__OBSOLETE_MASK 0xffffffff
+#define CP_ECC_FIRSTOCCURRENCE_RING2__OBSOLETE__SHIFT 0x0
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD_MASK 0xff
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD__SHIFT 0x0
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE_MASK 0x40000000
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE__SHIFT 0x1e
+#define CP_PQ_WPTR_POLL_CNTL__EN_MASK 0x80000000
+#define CP_PQ_WPTR_POLL_CNTL__EN__SHIFT 0x1f
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK_MASK 0xffffffff
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK__SHIFT 0x0
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK 0x1000
+#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT 0xc
+#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000
+#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd
+#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000
+#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe
+#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK 0x8000
+#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT 0xf
+#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000
+#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11
+#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17
+#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000
+#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18
+#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a
+#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000
+#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK 0x1000
+#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc
+#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000
+#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd
+#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000
+#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe
+#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000
+#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11
+#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000
+#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17
+#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000
+#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18
+#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000
+#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a
+#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000
+#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK 0x1000
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT 0xc
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK 0x2000
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT 0xd
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x4000
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x20000
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x800000
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x1000000
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x4000000
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x8000000
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK 0x1000
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT 0xc
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK 0x2000
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT 0xd
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x4000
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK 0x8000
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT 0xf
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x20000
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x800000
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x1000000
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x4000000
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x8000000
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18
+#define CP_ME2_PIPE0_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME2_PIPE0_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME2_PIPE1_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME2_PIPE1_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME2_PIPE2_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME2_PIPE2_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_ME2_PIPE3_PRIORITY__PRIORITY_MASK 0x3
+#define CP_ME2_PIPE3_PRIORITY__PRIORITY__SHIFT 0x0
+#define CP_CE_PRGRM_CNTR_START__IP_START_MASK 0x7ff
+#define CP_CE_PRGRM_CNTR_START__IP_START__SHIFT 0x0
+#define CP_PFP_PRGRM_CNTR_START__IP_START_MASK 0xfff
+#define CP_PFP_PRGRM_CNTR_START__IP_START__SHIFT 0x0
+#define CP_ME_PRGRM_CNTR_START__IP_START_MASK 0xfff
+#define CP_ME_PRGRM_CNTR_START__IP_START__SHIFT 0x0
+#define CP_MEC1_PRGRM_CNTR_START__IP_START_MASK 0xffff
+#define CP_MEC1_PRGRM_CNTR_START__IP_START__SHIFT 0x0
+#define CP_MEC2_PRGRM_CNTR_START__IP_START_MASK 0xffff
+#define CP_MEC2_PRGRM_CNTR_START__IP_START__SHIFT 0x0
+#define CP_CE_INTR_ROUTINE_START__IR_START_MASK 0x7ff
+#define CP_CE_INTR_ROUTINE_START__IR_START__SHIFT 0x0
+#define CP_PFP_INTR_ROUTINE_START__IR_START_MASK 0xfff
+#define CP_PFP_INTR_ROUTINE_START__IR_START__SHIFT 0x0
+#define CP_ME_INTR_ROUTINE_START__IR_START_MASK 0xfff
+#define CP_ME_INTR_ROUTINE_START__IR_START__SHIFT 0x0
+#define CP_MEC1_INTR_ROUTINE_START__IR_START_MASK 0xffff
+#define CP_MEC1_INTR_ROUTINE_START__IR_START__SHIFT 0x0
+#define CP_MEC2_INTR_ROUTINE_START__IR_START_MASK 0xffff
+#define CP_MEC2_INTR_ROUTINE_START__IR_START__SHIFT 0x0
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX_MASK 0x7
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX__SHIFT 0x0
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX_MASK 0x70
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX__SHIFT 0x4
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX_MASK 0x70000
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX__SHIFT 0x10
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX_MASK 0x700000
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX__SHIFT 0x14
+#define CP_MAX_CONTEXT__MAX_CONTEXT_MASK 0x7
+#define CP_MAX_CONTEXT__MAX_CONTEXT__SHIFT 0x0
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD_MASK 0xff
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD__SHIFT 0x0
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD_MASK 0xff00
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD__SHIFT 0x8
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD_MASK 0xff0000
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD__SHIFT 0x10
+#define CP_IQ_WAIT_TIME1__GWS_MASK 0xff000000
+#define CP_IQ_WAIT_TIME1__GWS__SHIFT 0x18
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP_MASK 0xff
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP__SHIFT 0x0
+#define CP_IQ_WAIT_TIME2__SCH_WAVE_MASK 0xff00
+#define CP_IQ_WAIT_TIME2__SCH_WAVE__SHIFT 0x8
+#define CP_IQ_WAIT_TIME2__SEM_REARM_MASK 0xff0000
+#define CP_IQ_WAIT_TIME2__SEM_REARM__SHIFT 0x10
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY_MASK 0xff000000
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY__SHIFT 0x18
+#define CP_VMID_RESET__RESET_REQUEST_MASK 0xffff
+#define CP_VMID_RESET__RESET_REQUEST__SHIFT 0x0
+#define CP_VMID_RESET__RESET_STATUS_MASK 0xffff0000
+#define CP_VMID_RESET__RESET_STATUS__SHIFT 0x10
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST_MASK 0xffff
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST__SHIFT 0x0
+#define CP_VMID_PREEMPT__VIRT_COMMAND_MASK 0xf0000
+#define CP_VMID_PREEMPT__VIRT_COMMAND__SHIFT 0x10
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS_MASK 0xffff
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS__SHIFT 0x0
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS_MASK 0xffff0000
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS__SHIFT 0x10
+#define CPC_INT_CNTX_ID__CNTX_ID_MASK 0xfffffff
+#define CPC_INT_CNTX_ID__CNTX_ID__SHIFT 0x0
+#define CPC_INT_CNTX_ID__QUEUE_ID_MASK 0x70000000
+#define CPC_INT_CNTX_ID__QUEUE_ID__SHIFT 0x1c
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MASK 0x1
+#define CP_PQ_STATUS__DOORBELL_UPDATED__SHIFT 0x0
+#define CP_PQ_STATUS__DOORBELL_ENABLE_MASK 0x2
+#define CP_PQ_STATUS__DOORBELL_ENABLE__SHIFT 0x1
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO_MASK 0xfffff000
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO__SHIFT 0xc
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI_MASK 0xffff
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI__SHIFT 0x0
+#define CP_CPC_IC_BASE_CNTL__VMID_MASK 0xf
+#define CP_CPC_IC_BASE_CNTL__VMID__SHIFT 0x0
+#define CP_CPC_IC_BASE_CNTL__ATC_MASK 0x800000
+#define CP_CPC_IC_BASE_CNTL__ATC__SHIFT 0x17
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY_MASK 0x1000000
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY__SHIFT 0x18
+#define CP_CPC_IC_BASE_CNTL__MTYPE_MASK 0x18000000
+#define CP_CPC_IC_BASE_CNTL__MTYPE__SHIFT 0x1b
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_MASK 0x1
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT 0x0
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE_MASK 0x10
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE__SHIFT 0x4
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED_MASK 0x20
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED__SHIFT 0x5
+#define CP_CPC_STATUS__MEC1_BUSY_MASK 0x1
+#define CP_CPC_STATUS__MEC1_BUSY__SHIFT 0x0
+#define CP_CPC_STATUS__MEC2_BUSY_MASK 0x2
+#define CP_CPC_STATUS__MEC2_BUSY__SHIFT 0x1
+#define CP_CPC_STATUS__DC0_BUSY_MASK 0x4
+#define CP_CPC_STATUS__DC0_BUSY__SHIFT 0x2
+#define CP_CPC_STATUS__DC1_BUSY_MASK 0x8
+#define CP_CPC_STATUS__DC1_BUSY__SHIFT 0x3
+#define CP_CPC_STATUS__RCIU1_BUSY_MASK 0x10
+#define CP_CPC_STATUS__RCIU1_BUSY__SHIFT 0x4
+#define CP_CPC_STATUS__RCIU2_BUSY_MASK 0x20
+#define CP_CPC_STATUS__RCIU2_BUSY__SHIFT 0x5
+#define CP_CPC_STATUS__ROQ1_BUSY_MASK 0x40
+#define CP_CPC_STATUS__ROQ1_BUSY__SHIFT 0x6
+#define CP_CPC_STATUS__ROQ2_BUSY_MASK 0x80
+#define CP_CPC_STATUS__ROQ2_BUSY__SHIFT 0x7
+#define CP_CPC_STATUS__TCIU_BUSY_MASK 0x400
+#define CP_CPC_STATUS__TCIU_BUSY__SHIFT 0xa
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY_MASK 0x800
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY__SHIFT 0xb
+#define CP_CPC_STATUS__QU_BUSY_MASK 0x1000
+#define CP_CPC_STATUS__QU_BUSY__SHIFT 0xc
+#define CP_CPC_STATUS__ATCL2IU_BUSY_MASK 0x2000
+#define CP_CPC_STATUS__ATCL2IU_BUSY__SHIFT 0xd
+#define CP_CPC_STATUS__CPG_CPC_BUSY_MASK 0x20000000
+#define CP_CPC_STATUS__CPG_CPC_BUSY__SHIFT 0x1d
+#define CP_CPC_STATUS__CPF_CPC_BUSY_MASK 0x40000000
+#define CP_CPC_STATUS__CPF_CPC_BUSY__SHIFT 0x1e
+#define CP_CPC_STATUS__CPC_BUSY_MASK 0x80000000
+#define CP_CPC_STATUS__CPC_BUSY__SHIFT 0x1f
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY_MASK 0x1
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY__SHIFT 0x0
+#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY_MASK 0x2
+#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY__SHIFT 0x1
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY_MASK 0x4
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY__SHIFT 0x2
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY_MASK 0x8
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY__SHIFT 0x3
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY_MASK 0x10
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY__SHIFT 0x4
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY_MASK 0x20
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY__SHIFT 0x5
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY_MASK 0x40
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY__SHIFT 0x6
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY_MASK 0x80
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY__SHIFT 0x7
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY_MASK 0x100
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY__SHIFT 0x8
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY_MASK 0x200
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY__SHIFT 0x9
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY_MASK 0x400
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY__SHIFT 0xa
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY_MASK 0x800
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY__SHIFT 0xb
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY_MASK 0x1000
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY__SHIFT 0xc
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY_MASK 0x2000
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY__SHIFT 0xd
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY_MASK 0x10000
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY__SHIFT 0x10
+#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY_MASK 0x20000
+#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY__SHIFT 0x11
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY_MASK 0x40000
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY__SHIFT 0x12
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY_MASK 0x80000
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY__SHIFT 0x13
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY_MASK 0x100000
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY__SHIFT 0x14
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY_MASK 0x200000
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY__SHIFT 0x15
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY_MASK 0x400000
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY__SHIFT 0x16
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY_MASK 0x800000
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY__SHIFT 0x17
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY_MASK 0x1000000
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY__SHIFT 0x18
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY_MASK 0x2000000
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY__SHIFT 0x19
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY_MASK 0x4000000
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY__SHIFT 0x1a
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY_MASK 0x8000000
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY__SHIFT 0x1b
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY_MASK 0x10000000
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY__SHIFT 0x1c
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY_MASK 0x20000000
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY__SHIFT 0x1d
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL_MASK 0x8
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL__SHIFT 0x3
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION_MASK 0x10
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION__SHIFT 0x4
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL_MASK 0x40
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL__SHIFT 0x6
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET_MASK 0x100
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET__SHIFT 0x8
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_MASK 0x200
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU__SHIFT 0x9
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ_MASK 0x400
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ__SHIFT 0xa
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA_MASK 0x2000
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA__SHIFT 0xd
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET_MASK 0x10000
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET__SHIFT 0x10
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_MASK 0x20000
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU__SHIFT 0x11
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ_MASK 0x40000
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ__SHIFT 0x12
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA_MASK 0x200000
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA__SHIFT 0x15
+#define CP_CPC_STALLED_STAT1__ATCL2IU_WAITING_ON_FREE_MASK 0x400000
+#define CP_CPC_STALLED_STAT1__ATCL2IU_WAITING_ON_FREE__SHIFT 0x16
+#define CP_CPC_STALLED_STAT1__ATCL2IU_WAITING_ON_TAGS_MASK 0x800000
+#define CP_CPC_STALLED_STAT1__ATCL2IU_WAITING_ON_TAGS__SHIFT 0x17
+#define CP_CPC_STALLED_STAT1__ATCL1_WAITING_ON_TRANS_MASK 0x1000000
+#define CP_CPC_STALLED_STAT1__ATCL1_WAITING_ON_TRANS__SHIFT 0x18
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY_MASK 0x1
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY__SHIFT 0x0
+#define CP_CPF_STATUS__CSF_BUSY_MASK 0x2
+#define CP_CPF_STATUS__CSF_BUSY__SHIFT 0x1
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY_MASK 0x10
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY__SHIFT 0x4
+#define CP_CPF_STATUS__ROQ_RING_BUSY_MASK 0x20
+#define CP_CPF_STATUS__ROQ_RING_BUSY__SHIFT 0x5
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY_MASK 0x40
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY__SHIFT 0x6
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY_MASK 0x80
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY__SHIFT 0x7
+#define CP_CPF_STATUS__ROQ_STATE_BUSY_MASK 0x100
+#define CP_CPF_STATUS__ROQ_STATE_BUSY__SHIFT 0x8
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY_MASK 0x200
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY__SHIFT 0x9
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY_MASK 0x400
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY__SHIFT 0xa
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY_MASK 0x800
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY__SHIFT 0xb
+#define CP_CPF_STATUS__SEMAPHORE_BUSY_MASK 0x1000
+#define CP_CPF_STATUS__SEMAPHORE_BUSY__SHIFT 0xc
+#define CP_CPF_STATUS__INTERRUPT_BUSY_MASK 0x2000
+#define CP_CPF_STATUS__INTERRUPT_BUSY__SHIFT 0xd
+#define CP_CPF_STATUS__TCIU_BUSY_MASK 0x4000
+#define CP_CPF_STATUS__TCIU_BUSY__SHIFT 0xe
+#define CP_CPF_STATUS__HQD_BUSY_MASK 0x8000
+#define CP_CPF_STATUS__HQD_BUSY__SHIFT 0xf
+#define CP_CPF_STATUS__PRT_BUSY_MASK 0x10000
+#define CP_CPF_STATUS__PRT_BUSY__SHIFT 0x10
+#define CP_CPF_STATUS__ATCL2IU_BUSY_MASK 0x20000
+#define CP_CPF_STATUS__ATCL2IU_BUSY__SHIFT 0x11
+#define CP_CPF_STATUS__CPF_GFX_BUSY_MASK 0x4000000
+#define CP_CPF_STATUS__CPF_GFX_BUSY__SHIFT 0x1a
+#define CP_CPF_STATUS__CPF_CMP_BUSY_MASK 0x8000000
+#define CP_CPF_STATUS__CPF_CMP_BUSY__SHIFT 0x1b
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY_MASK 0x30000000
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY__SHIFT 0x1c
+#define CP_CPF_STATUS__CPC_CPF_BUSY_MASK 0x40000000
+#define CP_CPF_STATUS__CPC_CPF_BUSY__SHIFT 0x1e
+#define CP_CPF_STATUS__CPF_BUSY_MASK 0x80000000
+#define CP_CPF_STATUS__CPF_BUSY__SHIFT 0x1f
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK 0x1
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT 0x0
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY_MASK 0x2
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY__SHIFT 0x1
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY_MASK 0x4
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY__SHIFT 0x2
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY_MASK 0x8
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY__SHIFT 0x3
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY_MASK 0x10
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY__SHIFT 0x4
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY_MASK 0x20
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY__SHIFT 0x5
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY_MASK 0x40
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY__SHIFT 0x6
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY_MASK 0x80
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY__SHIFT 0x7
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY_MASK 0x100
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY__SHIFT 0x8
+#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS_MASK 0x200
+#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS__SHIFT 0x9
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY_MASK 0x800
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY__SHIFT 0xb
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY_MASK 0x1000
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY__SHIFT 0xc
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY_MASK 0x2000
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY__SHIFT 0xd
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY_MASK 0x4000
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY__SHIFT 0xe
+#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY_MASK 0x8000
+#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY__SHIFT 0xf
+#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY_MASK 0x10000
+#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY__SHIFT 0x10
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY_MASK 0x20000
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY__SHIFT 0x11
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY_MASK 0x40000
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY__SHIFT 0x12
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY_MASK 0x80000
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY__SHIFT 0x13
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY_MASK 0x100000
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY__SHIFT 0x14
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY_MASK 0x200000
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY__SHIFT 0x15
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY_MASK 0x400000
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY__SHIFT 0x16
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY_MASK 0x800000
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY__SHIFT 0x17
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY_MASK 0x1000000
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY__SHIFT 0x18
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY_MASK 0x2000000
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY__SHIFT 0x19
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY_MASK 0x4000000
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY__SHIFT 0x1a
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY_MASK 0x8000000
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY__SHIFT 0x1b
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY_MASK 0x10000000
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY__SHIFT 0x1c
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY_MASK 0x20000000
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY__SHIFT 0x1d
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY_MASK 0x40000000
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY__SHIFT 0x1e
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY_MASK 0x80000000
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY__SHIFT 0x1f
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA_MASK 0x1
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA__SHIFT 0x0
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA_MASK 0x2
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA__SHIFT 0x1
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA_MASK 0x4
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA__SHIFT 0x2
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA_MASK 0x8
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA__SHIFT 0x3
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE_MASK 0x20
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE__SHIFT 0x5
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK 0x40
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT 0x6
+#define CP_CPF_STALLED_STAT1__ATCL2IU_WAITING_ON_FREE_MASK 0x80
+#define CP_CPF_STALLED_STAT1__ATCL2IU_WAITING_ON_FREE__SHIFT 0x7
+#define CP_CPF_STALLED_STAT1__ATCL2IU_WAITING_ON_TAGS_MASK 0x100
+#define CP_CPF_STALLED_STAT1__ATCL2IU_WAITING_ON_TAGS__SHIFT 0x8
+#define CP_CPF_STALLED_STAT1__ATCL1_WAITING_ON_TRANS_MASK 0x200
+#define CP_CPF_STALLED_STAT1__ATCL1_WAITING_ON_TRANS__SHIFT 0x9
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT_MASK 0x3f
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT__SHIFT 0x0
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK 0x10
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE__SHIFT 0x4
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK 0x10000
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET__SHIFT 0x10
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK 0x20000
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET__SHIFT 0x11
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK 0x40000
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET__SHIFT 0x12
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK 0x80000
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET__SHIFT 0x13
+#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK 0x100000
+#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET__SHIFT 0x14
+#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK 0x200000
+#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET__SHIFT 0x15
+#define CP_MEC_CNTL__MEC_ME2_HALT_MASK 0x10000000
+#define CP_MEC_CNTL__MEC_ME2_HALT__SHIFT 0x1c
+#define CP_MEC_CNTL__MEC_ME2_STEP_MASK 0x20000000
+#define CP_MEC_CNTL__MEC_ME2_STEP__SHIFT 0x1d
+#define CP_MEC_CNTL__MEC_ME1_HALT_MASK 0x40000000
+#define CP_MEC_CNTL__MEC_ME1_HALT__SHIFT 0x1e
+#define CP_MEC_CNTL__MEC_ME1_STEP_MASK 0x80000000
+#define CP_MEC_CNTL__MEC_ME1_STEP__SHIFT 0x1f
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP_MASK 0xffffffff
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP__SHIFT 0x0
+#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP_MASK 0xffffffff
+#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP__SHIFT 0x0
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_MASK 0x1ff
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT 0x0
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA_MASK 0xffffffff
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA__SHIFT 0x0
+#define CPG_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f
+#define CPG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3f
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xfc00
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xfc00
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CPC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f
+#define CPC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3f
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xfc00
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xfc00
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CPF_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f
+#define CPF_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3f
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xfc00
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xfc00
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CP_CPC_HALT_HYST_COUNT__COUNT_MASK 0xf
+#define CP_CPC_HALT_HYST_COUNT__COUNT__SHIFT 0x0
+#define CP_DRAW_OBJECT__OBJECT_MASK 0xffffffff
+#define CP_DRAW_OBJECT__OBJECT__SHIFT 0x0
+#define CP_DRAW_OBJECT_COUNTER__COUNT_MASK 0xffff
+#define CP_DRAW_OBJECT_COUNTER__COUNT__SHIFT 0x0
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI_MASK 0xffffffff
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI__SHIFT 0x0
+#define CP_DRAW_WINDOW_HI__WINDOW_HI_MASK 0xffffffff
+#define CP_DRAW_WINDOW_HI__WINDOW_HI__SHIFT 0x0
+#define CP_DRAW_WINDOW_LO__MIN_MASK 0xffff
+#define CP_DRAW_WINDOW_LO__MIN__SHIFT 0x0
+#define CP_DRAW_WINDOW_LO__MAX_MASK 0xffff0000
+#define CP_DRAW_WINDOW_LO__MAX__SHIFT 0x10
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX_MASK 0x1
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX__SHIFT 0x0
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN_MASK 0x2
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN__SHIFT 0x1
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI_MASK 0x4
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI__SHIFT 0x2
+#define CP_DRAW_WINDOW_CNTL__MODE_MASK 0x100
+#define CP_DRAW_WINDOW_CNTL__MODE__SHIFT 0x8
+#define CP_PRT_LOD_STATS_CNTL0__BU_SIZE_MASK 0xffffffff
+#define CP_PRT_LOD_STATS_CNTL0__BU_SIZE__SHIFT 0x0
+#define CP_PRT_LOD_STATS_CNTL1__BASE_LO_MASK 0xffffffff
+#define CP_PRT_LOD_STATS_CNTL1__BASE_LO__SHIFT 0x0
+#define CP_PRT_LOD_STATS_CNTL2__BASE_HI_MASK 0x3
+#define CP_PRT_LOD_STATS_CNTL2__BASE_HI__SHIFT 0x0
+#define CP_PRT_LOD_STATS_CNTL2__INTERVAL_MASK 0x3fc
+#define CP_PRT_LOD_STATS_CNTL2__INTERVAL__SHIFT 0x2
+#define CP_PRT_LOD_STATS_CNTL2__RESET_CNT_MASK 0x3fc00
+#define CP_PRT_LOD_STATS_CNTL2__RESET_CNT__SHIFT 0xa
+#define CP_PRT_LOD_STATS_CNTL2__RESET_FORCE_MASK 0x40000
+#define CP_PRT_LOD_STATS_CNTL2__RESET_FORCE__SHIFT 0x12
+#define CP_PRT_LOD_STATS_CNTL2__REPORT_AND_RESET_MASK 0x80000
+#define CP_PRT_LOD_STATS_CNTL2__REPORT_AND_RESET__SHIFT 0x13
+#define CP_PRT_LOD_STATS_CNTL2__MC_VMID_MASK 0x7800000
+#define CP_PRT_LOD_STATS_CNTL2__MC_VMID__SHIFT 0x17
+#define CP_PRT_LOD_STATS_CNTL2__CACHE_POLICY_MASK 0x10000000
+#define CP_PRT_LOD_STATS_CNTL2__CACHE_POLICY__SHIFT 0x1c
+#define CP_PRT_LOD_STATS_CNTL2__MTYPE_MASK 0xc0000000
+#define CP_PRT_LOD_STATS_CNTL2__MTYPE__SHIFT 0x1e
+#define CP_CE_COMPARE_COUNT__COMPARE_COUNT_MASK 0xffffffff
+#define CP_CE_COMPARE_COUNT__COMPARE_COUNT__SHIFT 0x0
+#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT_MASK 0xffffffff
+#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT 0x0
+#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT_MASK 0xffffffff
+#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT__SHIFT 0x0
+#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT_MASK 0xffffffff
+#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT__SHIFT 0x0
+#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT_MASK 0xffffffff
+#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT 0x0
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP_MASK 0x7f
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP__SHIFT 0x0
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA_MASK 0x3f000
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA__SHIFT 0xc
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_CONTROL_MASK 0x2000000
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_CONTROL__SHIFT 0x19
+#define CP_EOP_DONE_EVENT_CNTL__MTYPE_MASK 0x18000000
+#define CP_EOP_DONE_EVENT_CNTL__MTYPE__SHIFT 0x1b
+#define CP_EOP_DONE_DATA_CNTL__CNTX_ID_MASK 0xffff
+#define CP_EOP_DONE_DATA_CNTL__CNTX_ID__SHIFT 0x0
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL_MASK 0x30000
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL__SHIFT 0x10
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL_MASK 0x7000000
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT 0x18
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL_MASK 0xe0000000
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT 0x1d
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID_MASK 0xfffffff
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID__SHIFT 0x0
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO_MASK 0xfffffffc
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO__SHIFT 0x2
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_EOP_DONE_DATA_LO__DATA_LO_MASK 0xffffffff
+#define CP_EOP_DONE_DATA_LO__DATA_LO__SHIFT 0x0
+#define CP_EOP_DONE_DATA_HI__DATA_HI_MASK 0xffffffff
+#define CP_EOP_DONE_DATA_HI__DATA_HI__SHIFT 0x0
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO_MASK 0xffffffff
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO__SHIFT 0x0
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI_MASK 0xffffffff
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI__SHIFT 0x0
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO_MASK 0xfffffffc
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO__SHIFT 0x2
+#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI_MASK 0xffff
+#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO__SHIFT 0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO__SHIFT 0x0
+#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI_MASK 0xffffffff
+#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI__SHIFT 0x0
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO_MASK 0xfffffffc
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO__SHIFT 0x2
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI_MASK 0xffff
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI__SHIFT 0x0
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO__SHIFT 0x0
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI__SHIFT 0x0
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO__SHIFT 0x0
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI__SHIFT 0x0
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO__SHIFT 0x0
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI__SHIFT 0x0
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO__SHIFT 0x0
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI__SHIFT 0x0
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO__SHIFT 0x0
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI__SHIFT 0x0
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO__SHIFT 0x0
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI__SHIFT 0x0
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO__SHIFT 0x0
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI__SHIFT 0x0
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO_MASK 0xffffffff
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO__SHIFT 0x0
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI_MASK 0xffffffff
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI__SHIFT 0x0
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO_MASK 0xffffffff
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO__SHIFT 0x0
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI_MASK 0xffffffff
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI__SHIFT 0x0
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO_MASK 0xffffffff
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO__SHIFT 0x0
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI_MASK 0xffffffff
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI__SHIFT 0x0
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE_MASK 0xffffffff
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE__SHIFT 0x0
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE_MASK 0xffffffff
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE__SHIFT 0x0
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO_MASK 0xffffffff
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO__SHIFT 0x0
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI_MASK 0xffffffff
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI__SHIFT 0x0
+#define CP_PIPE_STATS_CONTROL__CACHE_CONTROL_MASK 0x2000000
+#define CP_PIPE_STATS_CONTROL__CACHE_CONTROL__SHIFT 0x19
+#define CP_PIPE_STATS_CONTROL__MTYPE_MASK 0x18000000
+#define CP_PIPE_STATS_CONTROL__MTYPE__SHIFT 0x1b
+#define CP_STREAM_OUT_CONTROL__CACHE_CONTROL_MASK 0x2000000
+#define CP_STREAM_OUT_CONTROL__CACHE_CONTROL__SHIFT 0x19
+#define CP_STREAM_OUT_CONTROL__MTYPE_MASK 0x18000000
+#define CP_STREAM_OUT_CONTROL__MTYPE__SHIFT 0x1b
+#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE_MASK 0x1
+#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE__SHIFT 0x0
+#define SCRATCH_REG0__SCRATCH_REG0_MASK 0xffffffff
+#define SCRATCH_REG0__SCRATCH_REG0__SHIFT 0x0
+#define SCRATCH_REG1__SCRATCH_REG1_MASK 0xffffffff
+#define SCRATCH_REG1__SCRATCH_REG1__SHIFT 0x0
+#define SCRATCH_REG2__SCRATCH_REG2_MASK 0xffffffff
+#define SCRATCH_REG2__SCRATCH_REG2__SHIFT 0x0
+#define SCRATCH_REG3__SCRATCH_REG3_MASK 0xffffffff
+#define SCRATCH_REG3__SCRATCH_REG3__SHIFT 0x0
+#define SCRATCH_REG4__SCRATCH_REG4_MASK 0xffffffff
+#define SCRATCH_REG4__SCRATCH_REG4__SHIFT 0x0
+#define SCRATCH_REG5__SCRATCH_REG5_MASK 0xffffffff
+#define SCRATCH_REG5__SCRATCH_REG5__SHIFT 0x0
+#define SCRATCH_REG6__SCRATCH_REG6_MASK 0xffffffff
+#define SCRATCH_REG6__SCRATCH_REG6__SHIFT 0x0
+#define SCRATCH_REG7__SCRATCH_REG7_MASK 0xffffffff
+#define SCRATCH_REG7__SCRATCH_REG7__SHIFT 0x0
+#define SCRATCH_UMSK__OBSOLETE_UMSK_MASK 0xff
+#define SCRATCH_UMSK__OBSOLETE_UMSK__SHIFT 0x0
+#define SCRATCH_UMSK__OBSOLETE_SWAP_MASK 0x30000
+#define SCRATCH_UMSK__OBSOLETE_SWAP__SHIFT 0x10
+#define SCRATCH_ADDR__OBSOLETE_ADDR_MASK 0xffffffff
+#define SCRATCH_ADDR__OBSOLETE_ADDR__SHIFT 0x0
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK 0xffffffff
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT 0x0
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK 0xffffffff
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT 0x0
+#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK 0xffffffff
+#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT 0x0
+#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK 0xffffffff
+#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT 0x0
+#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK 0xffffffff
+#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT 0x0
+#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK 0xffffffff
+#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT 0x0
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO_MASK 0xfffffffc
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO__SHIFT 0x2
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI_MASK 0xffff
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI__SHIFT 0x0
+#define CP_APPEND_ADDR_HI__CS_PS_SEL_MASK 0x10000
+#define CP_APPEND_ADDR_HI__CS_PS_SEL__SHIFT 0x10
+#define CP_APPEND_ADDR_HI__CACHE_POLICY_MASK 0x2000000
+#define CP_APPEND_ADDR_HI__CACHE_POLICY__SHIFT 0x19
+#define CP_APPEND_ADDR_HI__MTYPE_MASK 0x18000000
+#define CP_APPEND_ADDR_HI__MTYPE__SHIFT 0x1b
+#define CP_APPEND_ADDR_HI__COMMAND_MASK 0xe0000000
+#define CP_APPEND_ADDR_HI__COMMAND__SHIFT 0x1d
+#define CP_APPEND_DATA__DATA_MASK 0xffffffff
+#define CP_APPEND_DATA__DATA__SHIFT 0x0
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE_MASK 0xffffffff
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE__SHIFT 0x0
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE_MASK 0xffffffff
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE__SHIFT 0x0
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK 0xffffffff
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT 0x0
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK 0xffffffff
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT 0x0
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK 0xffffffff
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT 0x0
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK 0xffffffff
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT 0x0
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK 0xffffffff
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT 0x0
+#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK 0xffffffff
+#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT 0x0
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK 0xffffffff
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT 0x0
+#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK 0xffffffff
+#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT 0x0
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK 0xffffffff
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT 0x0
+#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK 0xffffffff
+#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT 0x0
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK 0xffffffff
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT 0x0
+#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK 0xffffffff
+#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT 0x0
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_SWAP_MASK 0x3
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_SWAP__SHIFT 0x0
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO_MASK 0xfffffffc
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO__SHIFT 0x2
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI_MASK 0xffff
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI__SHIFT 0x0
+#define CP_ME_MC_WADDR_HI__MTYPE_MASK 0x300000
+#define CP_ME_MC_WADDR_HI__MTYPE__SHIFT 0x14
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY_MASK 0x400000
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY__SHIFT 0x16
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO_MASK 0xffffffff
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO__SHIFT 0x0
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI_MASK 0xffffffff
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI__SHIFT 0x0
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_SWAP_MASK 0x3
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_SWAP__SHIFT 0x0
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO_MASK 0xfffffffc
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO__SHIFT 0x2
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI_MASK 0xffff
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI__SHIFT 0x0
+#define CP_ME_MC_RADDR_HI__MTYPE_MASK 0x300000
+#define CP_ME_MC_RADDR_HI__MTYPE__SHIFT 0x14
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY_MASK 0x400000
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY__SHIFT 0x16
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER_MASK 0xffffffff
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER__SHIFT 0x0
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK 0x3
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT 0x0
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO_MASK 0xfffffff8
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT 0x3
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI_MASK 0xffff
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT 0x0
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK 0x10000
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT 0x10
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK 0x100000
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT 0x14
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK 0x3000000
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT 0x18
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT_MASK 0xe0000000
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT__SHIFT 0x1d
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK 0x3
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT 0x0
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO_MASK 0xfffffff8
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT 0x3
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI_MASK 0xffff
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT 0x0
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK 0x10000
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT 0x10
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK 0x100000
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT 0x14
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK 0x3000000
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT 0x18
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT_MASK 0xe0000000
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT__SHIFT 0x1d
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT_MASK 0xffffffff
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT__SHIFT 0x0
+#define CP_COHER_START_DELAY__START_DELAY_COUNT_MASK 0x3f
+#define CP_COHER_START_DELAY__START_DELAY_COUNT__SHIFT 0x0
+#define CP_COHER_CNTL__DEST_BASE_0_ENA_MASK 0x1
+#define CP_COHER_CNTL__DEST_BASE_0_ENA__SHIFT 0x0
+#define CP_COHER_CNTL__DEST_BASE_1_ENA_MASK 0x2
+#define CP_COHER_CNTL__DEST_BASE_1_ENA__SHIFT 0x1
+#define CP_COHER_CNTL__TC_SD_ACTION_ENA_MASK 0x4
+#define CP_COHER_CNTL__TC_SD_ACTION_ENA__SHIFT 0x2
+#define CP_COHER_CNTL__TC_NC_ACTION_ENA_MASK 0x8
+#define CP_COHER_CNTL__TC_NC_ACTION_ENA__SHIFT 0x3
+#define CP_COHER_CNTL__CB0_DEST_BASE_ENA_MASK 0x40
+#define CP_COHER_CNTL__CB0_DEST_BASE_ENA__SHIFT 0x6
+#define CP_COHER_CNTL__CB1_DEST_BASE_ENA_MASK 0x80
+#define CP_COHER_CNTL__CB1_DEST_BASE_ENA__SHIFT 0x7
+#define CP_COHER_CNTL__CB2_DEST_BASE_ENA_MASK 0x100
+#define CP_COHER_CNTL__CB2_DEST_BASE_ENA__SHIFT 0x8
+#define CP_COHER_CNTL__CB3_DEST_BASE_ENA_MASK 0x200
+#define CP_COHER_CNTL__CB3_DEST_BASE_ENA__SHIFT 0x9
+#define CP_COHER_CNTL__CB4_DEST_BASE_ENA_MASK 0x400
+#define CP_COHER_CNTL__CB4_DEST_BASE_ENA__SHIFT 0xa
+#define CP_COHER_CNTL__CB5_DEST_BASE_ENA_MASK 0x800
+#define CP_COHER_CNTL__CB5_DEST_BASE_ENA__SHIFT 0xb
+#define CP_COHER_CNTL__CB6_DEST_BASE_ENA_MASK 0x1000
+#define CP_COHER_CNTL__CB6_DEST_BASE_ENA__SHIFT 0xc
+#define CP_COHER_CNTL__CB7_DEST_BASE_ENA_MASK 0x2000
+#define CP_COHER_CNTL__CB7_DEST_BASE_ENA__SHIFT 0xd
+#define CP_COHER_CNTL__DB_DEST_BASE_ENA_MASK 0x4000
+#define CP_COHER_CNTL__DB_DEST_BASE_ENA__SHIFT 0xe
+#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA_MASK 0x8000
+#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA__SHIFT 0xf
+#define CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK 0x40000
+#define CP_COHER_CNTL__TC_WB_ACTION_ENA__SHIFT 0x12
+#define CP_COHER_CNTL__DEST_BASE_2_ENA_MASK 0x80000
+#define CP_COHER_CNTL__DEST_BASE_2_ENA__SHIFT 0x13
+#define CP_COHER_CNTL__DEST_BASE_3_ENA_MASK 0x200000
+#define CP_COHER_CNTL__DEST_BASE_3_ENA__SHIFT 0x15
+#define CP_COHER_CNTL__TCL1_ACTION_ENA_MASK 0x400000
+#define CP_COHER_CNTL__TCL1_ACTION_ENA__SHIFT 0x16
+#define CP_COHER_CNTL__TC_ACTION_ENA_MASK 0x800000
+#define CP_COHER_CNTL__TC_ACTION_ENA__SHIFT 0x17
+#define CP_COHER_CNTL__CB_ACTION_ENA_MASK 0x2000000
+#define CP_COHER_CNTL__CB_ACTION_ENA__SHIFT 0x19
+#define CP_COHER_CNTL__DB_ACTION_ENA_MASK 0x4000000
+#define CP_COHER_CNTL__DB_ACTION_ENA__SHIFT 0x1a
+#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK 0x8000000
+#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA__SHIFT 0x1b
+#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA_MASK 0x10000000
+#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA__SHIFT 0x1c
+#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK 0x20000000
+#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA__SHIFT 0x1d
+#define CP_COHER_CNTL__SH_KCACHE_WB_ACTION_ENA_MASK 0x40000000
+#define CP_COHER_CNTL__SH_KCACHE_WB_ACTION_ENA__SHIFT 0x1e
+#define CP_COHER_CNTL__SH_SD_ACTION_ENA_MASK 0x80000000
+#define CP_COHER_CNTL__SH_SD_ACTION_ENA__SHIFT 0x1f
+#define CP_COHER_SIZE__COHER_SIZE_256B_MASK 0xffffffff
+#define CP_COHER_SIZE__COHER_SIZE_256B__SHIFT 0x0
+#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK 0xff
+#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT 0x0
+#define CP_COHER_BASE__COHER_BASE_256B_MASK 0xffffffff
+#define CP_COHER_BASE__COHER_BASE_256B__SHIFT 0x0
+#define CP_COHER_BASE_HI__COHER_BASE_HI_256B_MASK 0xff
+#define CP_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT 0x0
+#define CP_COHER_STATUS__MATCHING_GFX_CNTX_MASK 0xff
+#define CP_COHER_STATUS__MATCHING_GFX_CNTX__SHIFT 0x0
+#define CP_COHER_STATUS__MEID_MASK 0x3000000
+#define CP_COHER_STATUS__MEID__SHIFT 0x18
+#define CP_COHER_STATUS__PHASE1_STATUS_MASK 0x40000000
+#define CP_COHER_STATUS__PHASE1_STATUS__SHIFT 0x1e
+#define CP_COHER_STATUS__STATUS_MASK 0x80000000
+#define CP_COHER_STATUS__STATUS__SHIFT 0x1f
+#define COHER_DEST_BASE_0__DEST_BASE_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_0__DEST_BASE_256B__SHIFT 0x0
+#define COHER_DEST_BASE_1__DEST_BASE_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_1__DEST_BASE_256B__SHIFT 0x0
+#define COHER_DEST_BASE_2__DEST_BASE_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_2__DEST_BASE_256B__SHIFT 0x0
+#define COHER_DEST_BASE_3__DEST_BASE_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_3__DEST_BASE_256B__SHIFT 0x0
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B__SHIFT 0x0
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B__SHIFT 0x0
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B__SHIFT 0x0
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B_MASK 0xffffffff
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B__SHIFT 0x0
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR_MASK 0xffffffff
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR__SHIFT 0x0
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI_MASK 0xffff
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT 0x0
+#define CP_DMA_ME_DST_ADDR__DST_ADDR_MASK 0xffffffff
+#define CP_DMA_ME_DST_ADDR__DST_ADDR__SHIFT 0x0
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI_MASK 0xffff
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI__SHIFT 0x0
+#define CP_DMA_ME_CONTROL__SRC_MTYPE_MASK 0xc00
+#define CP_DMA_ME_CONTROL__SRC_MTYPE__SHIFT 0xa
+#define CP_DMA_ME_CONTROL__SRC_ATC_MASK 0x1000
+#define CP_DMA_ME_CONTROL__SRC_ATC__SHIFT 0xc
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY_MASK 0x2000
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY__SHIFT 0xd
+#define CP_DMA_ME_CONTROL__DST_SELECT_MASK 0x300000
+#define CP_DMA_ME_CONTROL__DST_SELECT__SHIFT 0x14
+#define CP_DMA_ME_CONTROL__DST_MTYPE_MASK 0xc00000
+#define CP_DMA_ME_CONTROL__DST_MTYPE__SHIFT 0x16
+#define CP_DMA_ME_CONTROL__DST_ATC_MASK 0x1000000
+#define CP_DMA_ME_CONTROL__DST_ATC__SHIFT 0x18
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY_MASK 0x2000000
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY__SHIFT 0x19
+#define CP_DMA_ME_CONTROL__SRC_SELECT_MASK 0x60000000
+#define CP_DMA_ME_CONTROL__SRC_SELECT__SHIFT 0x1d
+#define CP_DMA_ME_COMMAND__BYTE_COUNT_MASK 0x1fffff
+#define CP_DMA_ME_COMMAND__BYTE_COUNT__SHIFT 0x0
+#define CP_DMA_ME_COMMAND__DIS_WC_MASK 0x200000
+#define CP_DMA_ME_COMMAND__DIS_WC__SHIFT 0x15
+#define CP_DMA_ME_COMMAND__SRC_SWAP_MASK 0xc00000
+#define CP_DMA_ME_COMMAND__SRC_SWAP__SHIFT 0x16
+#define CP_DMA_ME_COMMAND__DST_SWAP_MASK 0x3000000
+#define CP_DMA_ME_COMMAND__DST_SWAP__SHIFT 0x18
+#define CP_DMA_ME_COMMAND__SAS_MASK 0x4000000
+#define CP_DMA_ME_COMMAND__SAS__SHIFT 0x1a
+#define CP_DMA_ME_COMMAND__DAS_MASK 0x8000000
+#define CP_DMA_ME_COMMAND__DAS__SHIFT 0x1b
+#define CP_DMA_ME_COMMAND__SAIC_MASK 0x10000000
+#define CP_DMA_ME_COMMAND__SAIC__SHIFT 0x1c
+#define CP_DMA_ME_COMMAND__DAIC_MASK 0x20000000
+#define CP_DMA_ME_COMMAND__DAIC__SHIFT 0x1d
+#define CP_DMA_ME_COMMAND__RAW_WAIT_MASK 0x40000000
+#define CP_DMA_ME_COMMAND__RAW_WAIT__SHIFT 0x1e
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR_MASK 0xffffffff
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR__SHIFT 0x0
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI_MASK 0xffff
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT 0x0
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR_MASK 0xffffffff
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR__SHIFT 0x0
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI_MASK 0xffff
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI__SHIFT 0x0
+#define CP_DMA_PFP_CONTROL__SRC_MTYPE_MASK 0xc00
+#define CP_DMA_PFP_CONTROL__SRC_MTYPE__SHIFT 0xa
+#define CP_DMA_PFP_CONTROL__SRC_ATC_MASK 0x1000
+#define CP_DMA_PFP_CONTROL__SRC_ATC__SHIFT 0xc
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY_MASK 0x2000
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY__SHIFT 0xd
+#define CP_DMA_PFP_CONTROL__DST_SELECT_MASK 0x300000
+#define CP_DMA_PFP_CONTROL__DST_SELECT__SHIFT 0x14
+#define CP_DMA_PFP_CONTROL__DST_MTYPE_MASK 0xc00000
+#define CP_DMA_PFP_CONTROL__DST_MTYPE__SHIFT 0x16
+#define CP_DMA_PFP_CONTROL__DST_ATC_MASK 0x1000000
+#define CP_DMA_PFP_CONTROL__DST_ATC__SHIFT 0x18
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY_MASK 0x2000000
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY__SHIFT 0x19
+#define CP_DMA_PFP_CONTROL__SRC_SELECT_MASK 0x60000000
+#define CP_DMA_PFP_CONTROL__SRC_SELECT__SHIFT 0x1d
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT_MASK 0x1fffff
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT__SHIFT 0x0
+#define CP_DMA_PFP_COMMAND__DIS_WC_MASK 0x200000
+#define CP_DMA_PFP_COMMAND__DIS_WC__SHIFT 0x15
+#define CP_DMA_PFP_COMMAND__SRC_SWAP_MASK 0xc00000
+#define CP_DMA_PFP_COMMAND__SRC_SWAP__SHIFT 0x16
+#define CP_DMA_PFP_COMMAND__DST_SWAP_MASK 0x3000000
+#define CP_DMA_PFP_COMMAND__DST_SWAP__SHIFT 0x18
+#define CP_DMA_PFP_COMMAND__SAS_MASK 0x4000000
+#define CP_DMA_PFP_COMMAND__SAS__SHIFT 0x1a
+#define CP_DMA_PFP_COMMAND__DAS_MASK 0x8000000
+#define CP_DMA_PFP_COMMAND__DAS__SHIFT 0x1b
+#define CP_DMA_PFP_COMMAND__SAIC_MASK 0x10000000
+#define CP_DMA_PFP_COMMAND__SAIC__SHIFT 0x1c
+#define CP_DMA_PFP_COMMAND__DAIC_MASK 0x20000000
+#define CP_DMA_PFP_COMMAND__DAIC__SHIFT 0x1d
+#define CP_DMA_PFP_COMMAND__RAW_WAIT_MASK 0x40000000
+#define CP_DMA_PFP_COMMAND__RAW_WAIT__SHIFT 0x1e
+#define CP_DMA_CNTL__MIN_AVAILSZ_MASK 0x30
+#define CP_DMA_CNTL__MIN_AVAILSZ__SHIFT 0x4
+#define CP_DMA_CNTL__BUFFER_DEPTH_MASK 0xf0000
+#define CP_DMA_CNTL__BUFFER_DEPTH__SHIFT 0x10
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY_MASK 0x10000000
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY__SHIFT 0x1c
+#define CP_DMA_CNTL__PIO_FIFO_FULL_MASK 0x20000000
+#define CP_DMA_CNTL__PIO_FIFO_FULL__SHIFT 0x1d
+#define CP_DMA_CNTL__PIO_COUNT_MASK 0xc0000000
+#define CP_DMA_CNTL__PIO_COUNT__SHIFT 0x1e
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_MASK 0x3ffffff
+#define CP_DMA_READ_TAGS__DMA_READ_TAG__SHIFT 0x0
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID_MASK 0x10000000
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID__SHIFT 0x1c
+#define CP_PFP_IB_CONTROL__IB_EN_MASK 0xff
+#define CP_PFP_IB_CONTROL__IB_EN__SHIFT 0x0
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN_MASK 0x1
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN__SHIFT 0x0
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN_MASK 0x2
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN__SHIFT 0x1
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN_MASK 0x10000
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN__SHIFT 0x10
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN_MASK 0x1000000
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN__SHIFT 0x18
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_MASK 0xff
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT 0x0
+#define CP_SCRATCH_DATA__SCRATCH_DATA_MASK 0xffffffff
+#define CP_SCRATCH_DATA__SCRATCH_DATA__SHIFT 0x0
+#define CP_RB_OFFSET__RB_OFFSET_MASK 0xfffff
+#define CP_RB_OFFSET__RB_OFFSET__SHIFT 0x0
+#define CP_IB1_OFFSET__IB1_OFFSET_MASK 0xfffff
+#define CP_IB1_OFFSET__IB1_OFFSET__SHIFT 0x0
+#define CP_IB2_OFFSET__IB2_OFFSET_MASK 0xfffff
+#define CP_IB2_OFFSET__IB2_OFFSET__SHIFT 0x0
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN_MASK 0xfffff
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN__SHIFT 0x0
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END_MASK 0xfffff
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END__SHIFT 0x0
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN_MASK 0xfffff
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN__SHIFT 0x0
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END_MASK 0xfffff
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END__SHIFT 0x0
+#define CP_CE_IB1_OFFSET__IB1_OFFSET_MASK 0xfffff
+#define CP_CE_IB1_OFFSET__IB1_OFFSET__SHIFT 0x0
+#define CP_CE_IB2_OFFSET__IB2_OFFSET_MASK 0xfffff
+#define CP_CE_IB2_OFFSET__IB2_OFFSET__SHIFT 0x0
+#define CP_CE_COUNTER__CONST_ENGINE_COUNT_MASK 0xffffffff
+#define CP_CE_COUNTER__CONST_ENGINE_COUNT__SHIFT 0x0
+#define CP_CE_RB_OFFSET__RB_OFFSET_MASK 0xfffff
+#define CP_CE_RB_OFFSET__RB_OFFSET__SHIFT 0x0
+#define CP_PFP_COMPLETION_STATUS__STATUS_MASK 0x3
+#define CP_PFP_COMPLETION_STATUS__STATUS__SHIFT 0x0
+#define CP_CE_COMPLETION_STATUS__STATUS_MASK 0x3
+#define CP_CE_COMPLETION_STATUS__STATUS__SHIFT 0x0
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE_MASK 0x1
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE__SHIFT 0x0
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO_MASK 0xffffffff
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO__SHIFT 0x0
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_CE_METADATA_BASE_ADDR__ADDR_LO_MASK 0xffffffff
+#define CP_CE_METADATA_BASE_ADDR__ADDR_LO__SHIFT 0x0
+#define CP_CE_METADATA_BASE_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_CE_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO_MASK 0xffffffff
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO__SHIFT 0x0
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO_MASK 0xffffffff
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO__SHIFT 0x0
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_INDEX_BASE_ADDR__ADDR_LO_MASK 0xffffffff
+#define CP_INDEX_BASE_ADDR__ADDR_LO__SHIFT 0x0
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_INDEX_TYPE__INDEX_TYPE_MASK 0x3
+#define CP_INDEX_TYPE__INDEX_TYPE__SHIFT 0x0
+#define CP_GDS_BKUP_ADDR__ADDR_LO_MASK 0xffffffff
+#define CP_GDS_BKUP_ADDR__ADDR_LO__SHIFT 0x0
+#define CP_GDS_BKUP_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_GDS_BKUP_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE_MASK 0x1
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE__SHIFT 0x0
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE_MASK 0x2
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE__SHIFT 0x1
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE_MASK 0x4
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE__SHIFT 0x2
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE_MASK 0x8
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE__SHIFT 0x3
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE_MASK 0x10
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE__SHIFT 0x4
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE_MASK 0x20
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE__SHIFT 0x5
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE_MASK 0x40
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE__SHIFT 0x6
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE_MASK 0x80
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE__SHIFT 0x7
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV_MASK 0x1
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV__SHIFT 0x0
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_MASK 0x4
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV__SHIFT 0x2
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_MASK 0x10
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV__SHIFT 0x4
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG_MASK 0x400
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG__SHIFT 0xa
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG_MASK 0x800
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG__SHIFT 0xb
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM_MASK 0x1000
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM__SHIFT 0xc
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA_MASK 0x2000
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA__SHIFT 0xd
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA_MASK 0x4000
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA__SHIFT 0xe
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA_MASK 0x8000
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA__SHIFT 0xf
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE_MASK 0x800000
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE__SHIFT 0x17
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE_MASK 0x1000000
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE__SHIFT 0x18
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE_MASK 0x2000000
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE__SHIFT 0x19
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ_MASK 0x4000000
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ__SHIFT 0x1a
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ_MASK 0x8000000
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ__SHIFT 0x1b
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ_MASK 0x10000000
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ__SHIFT 0x1c
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION_MASK 0x20000000
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION__SHIFT 0x1d
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV_MASK 0x1
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV__SHIFT 0x0
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV_MASK 0x2
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV__SHIFT 0x1
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV_MASK 0x4
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV__SHIFT 0x2
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING_MASK 0x10
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING__SHIFT 0x4
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING_MASK 0x20
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING__SHIFT 0x5
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA_MASK 0x100
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA__SHIFT 0x8
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER_MASK 0x200
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER__SHIFT 0x9
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER_MASK 0x400
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER__SHIFT 0xa
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME_MASK 0x800
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME__SHIFT 0xb
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV_MASK 0x1000
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV__SHIFT 0xc
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV_MASK 0x2000
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV__SHIFT 0xd
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP_MASK 0x4000
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP__SHIFT 0xe
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH_MASK 0x8000
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH__SHIFT 0xf
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV_MASK 0x10000
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV__SHIFT 0x10
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV_MASK 0x20000
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV__SHIFT 0x11
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ_MASK 0x40000
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ__SHIFT 0x12
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM_MASK 0x80000
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM__SHIFT 0x13
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA_MASK 0x100000
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA__SHIFT 0x14
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE_MASK 0x200000
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE__SHIFT 0x15
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM_MASK 0x400000
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM__SHIFT 0x16
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING_MASK 0x800000
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING__SHIFT 0x17
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING_MASK 0x1000000
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING__SHIFT 0x18
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE_MASK 0x2000000
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE__SHIFT 0x19
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE_MASK 0x4000000
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE__SHIFT 0x1a
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM_MASK 0x8000000
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM__SHIFT 0x1b
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION_MASK 0x10000000
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION__SHIFT 0x1c
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE_MASK 0x20000000
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE__SHIFT 0x1d
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS_MASK 0x40000000
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS__SHIFT 0x1e
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN_MASK 0x80000000
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN__SHIFT 0x1f
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV_MASK 0x1
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV__SHIFT 0x0
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV_MASK 0x2
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV__SHIFT 0x1
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER_MASK 0x4
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER__SHIFT 0x2
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY_MASK 0x8
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY__SHIFT 0x3
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY_MASK 0x10
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY__SHIFT 0x4
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY_MASK 0x20
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY__SHIFT 0x5
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV_MASK 0x40
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV__SHIFT 0x6
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV_MASK 0x80
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV__SHIFT 0x7
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA_MASK 0x400
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA__SHIFT 0xa
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG_MASK 0x800
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG__SHIFT 0xb
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_MASK 0x1000
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER__SHIFT 0xc
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW_MASK 0x2000
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW__SHIFT 0xd
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE_MASK 0x4000
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE__SHIFT 0xe
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS_MASK 0x8000
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS__SHIFT 0xf
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM_MASK 0x10000
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM__SHIFT 0x10
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA_MASK 0x20000
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA__SHIFT 0x11
+#define CP_STALLED_STAT3__ATCL2IU_WAITING_ON_FREE_MASK 0x40000
+#define CP_STALLED_STAT3__ATCL2IU_WAITING_ON_FREE__SHIFT 0x12
+#define CP_STALLED_STAT3__ATCL2IU_WAITING_ON_TAGS_MASK 0x80000
+#define CP_STALLED_STAT3__ATCL2IU_WAITING_ON_TAGS__SHIFT 0x13
+#define CP_STALLED_STAT3__ATCL1_WAITING_ON_TRANS_MASK 0x100000
+#define CP_STALLED_STAT3__ATCL1_WAITING_ON_TRANS__SHIFT 0x14
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK 0x1
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT 0x0
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO_MASK 0x40
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO__SHIFT 0x6
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS_MASK 0x80
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS__SHIFT 0x7
+#define CP_BUSY_STAT__ME_PARSING_PACKETS_MASK 0x100
+#define CP_BUSY_STAT__ME_PARSING_PACKETS__SHIFT 0x8
+#define CP_BUSY_STAT__RCIU_PFP_BUSY_MASK 0x200
+#define CP_BUSY_STAT__RCIU_PFP_BUSY__SHIFT 0x9
+#define CP_BUSY_STAT__RCIU_ME_BUSY_MASK 0x400
+#define CP_BUSY_STAT__RCIU_ME_BUSY__SHIFT 0xa
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY_MASK 0x1000
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY__SHIFT 0xc
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING_MASK 0x2000
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING__SHIFT 0xd
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS_MASK 0x4000
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS__SHIFT 0xe
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY_MASK 0x8000
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY__SHIFT 0xf
+#define CP_BUSY_STAT__ME_PARSER_BUSY_MASK 0x20000
+#define CP_BUSY_STAT__ME_PARSER_BUSY__SHIFT 0x11
+#define CP_BUSY_STAT__EOP_DONE_BUSY_MASK 0x40000
+#define CP_BUSY_STAT__EOP_DONE_BUSY__SHIFT 0x12
+#define CP_BUSY_STAT__STRM_OUT_BUSY_MASK 0x80000
+#define CP_BUSY_STAT__STRM_OUT_BUSY__SHIFT 0x13
+#define CP_BUSY_STAT__PIPE_STATS_BUSY_MASK 0x100000
+#define CP_BUSY_STAT__PIPE_STATS_BUSY__SHIFT 0x14
+#define CP_BUSY_STAT__RCIU_CE_BUSY_MASK 0x200000
+#define CP_BUSY_STAT__RCIU_CE_BUSY__SHIFT 0x15
+#define CP_BUSY_STAT__CE_PARSING_PACKETS_MASK 0x400000
+#define CP_BUSY_STAT__CE_PARSING_PACKETS__SHIFT 0x16
+#define CP_STAT__ROQ_RING_BUSY_MASK 0x200
+#define CP_STAT__ROQ_RING_BUSY__SHIFT 0x9
+#define CP_STAT__ROQ_INDIRECT1_BUSY_MASK 0x400
+#define CP_STAT__ROQ_INDIRECT1_BUSY__SHIFT 0xa
+#define CP_STAT__ROQ_INDIRECT2_BUSY_MASK 0x800
+#define CP_STAT__ROQ_INDIRECT2_BUSY__SHIFT 0xb
+#define CP_STAT__ROQ_STATE_BUSY_MASK 0x1000
+#define CP_STAT__ROQ_STATE_BUSY__SHIFT 0xc
+#define CP_STAT__DC_BUSY_MASK 0x2000
+#define CP_STAT__DC_BUSY__SHIFT 0xd
+#define CP_STAT__ATCL2IU_BUSY_MASK 0x4000
+#define CP_STAT__ATCL2IU_BUSY__SHIFT 0xe
+#define CP_STAT__PFP_BUSY_MASK 0x8000
+#define CP_STAT__PFP_BUSY__SHIFT 0xf
+#define CP_STAT__MEQ_BUSY_MASK 0x10000
+#define CP_STAT__MEQ_BUSY__SHIFT 0x10
+#define CP_STAT__ME_BUSY_MASK 0x20000
+#define CP_STAT__ME_BUSY__SHIFT 0x11
+#define CP_STAT__QUERY_BUSY_MASK 0x40000
+#define CP_STAT__QUERY_BUSY__SHIFT 0x12
+#define CP_STAT__SEMAPHORE_BUSY_MASK 0x80000
+#define CP_STAT__SEMAPHORE_BUSY__SHIFT 0x13
+#define CP_STAT__INTERRUPT_BUSY_MASK 0x100000
+#define CP_STAT__INTERRUPT_BUSY__SHIFT 0x14
+#define CP_STAT__SURFACE_SYNC_BUSY_MASK 0x200000
+#define CP_STAT__SURFACE_SYNC_BUSY__SHIFT 0x15
+#define CP_STAT__DMA_BUSY_MASK 0x400000
+#define CP_STAT__DMA_BUSY__SHIFT 0x16
+#define CP_STAT__RCIU_BUSY_MASK 0x800000
+#define CP_STAT__RCIU_BUSY__SHIFT 0x17
+#define CP_STAT__SCRATCH_RAM_BUSY_MASK 0x1000000
+#define CP_STAT__SCRATCH_RAM_BUSY__SHIFT 0x18
+#define CP_STAT__CPC_CPG_BUSY_MASK 0x2000000
+#define CP_STAT__CPC_CPG_BUSY__SHIFT 0x19
+#define CP_STAT__CE_BUSY_MASK 0x4000000
+#define CP_STAT__CE_BUSY__SHIFT 0x1a
+#define CP_STAT__TCIU_BUSY_MASK 0x8000000
+#define CP_STAT__TCIU_BUSY__SHIFT 0x1b
+#define CP_STAT__ROQ_CE_RING_BUSY_MASK 0x10000000
+#define CP_STAT__ROQ_CE_RING_BUSY__SHIFT 0x1c
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY_MASK 0x20000000
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY__SHIFT 0x1d
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY_MASK 0x40000000
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY__SHIFT 0x1e
+#define CP_STAT__CP_BUSY_MASK 0x80000000
+#define CP_STAT__CP_BUSY__SHIFT 0x1f
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP_MASK 0xffffffff
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP__SHIFT 0x0
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP_MASK 0xffffffff
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP__SHIFT 0x0
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_MASK 0x3f
+#define CP_GRBM_FREE_COUNT__FREE_COUNT__SHIFT 0x0
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS_MASK 0x3f00
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS__SHIFT 0x8
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP_MASK 0x3f0000
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP__SHIFT 0x10
+#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP_MASK 0xffffffff
+#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP__SHIFT 0x0
+#define CP_CSF_STAT__BUFFER_SLOTS_ALLOCATED_MASK 0xf
+#define CP_CSF_STAT__BUFFER_SLOTS_ALLOCATED__SHIFT 0x0
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT_MASK 0x1ff00
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT__SHIFT 0x8
+#define CP_CSF_CNTL__FETCH_BUFFER_DEPTH_MASK 0xf
+#define CP_CSF_CNTL__FETCH_BUFFER_DEPTH__SHIFT 0x0
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE_MASK 0x10
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE__SHIFT 0x4
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE_MASK 0x40
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE__SHIFT 0x6
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE_MASK 0x100
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE__SHIFT 0x8
+#define CP_ME_CNTL__CE_PIPE0_RESET_MASK 0x10000
+#define CP_ME_CNTL__CE_PIPE0_RESET__SHIFT 0x10
+#define CP_ME_CNTL__PFP_PIPE0_RESET_MASK 0x40000
+#define CP_ME_CNTL__PFP_PIPE0_RESET__SHIFT 0x12
+#define CP_ME_CNTL__ME_PIPE0_RESET_MASK 0x100000
+#define CP_ME_CNTL__ME_PIPE0_RESET__SHIFT 0x14
+#define CP_ME_CNTL__CE_HALT_MASK 0x1000000
+#define CP_ME_CNTL__CE_HALT__SHIFT 0x18
+#define CP_ME_CNTL__CE_STEP_MASK 0x2000000
+#define CP_ME_CNTL__CE_STEP__SHIFT 0x19
+#define CP_ME_CNTL__PFP_HALT_MASK 0x4000000
+#define CP_ME_CNTL__PFP_HALT__SHIFT 0x1a
+#define CP_ME_CNTL__PFP_STEP_MASK 0x8000000
+#define CP_ME_CNTL__PFP_STEP__SHIFT 0x1b
+#define CP_ME_CNTL__ME_HALT_MASK 0x10000000
+#define CP_ME_CNTL__ME_HALT__SHIFT 0x1c
+#define CP_ME_CNTL__ME_STEP_MASK 0x20000000
+#define CP_ME_CNTL__ME_STEP__SHIFT 0x1d
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS_MASK 0xff
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS__SHIFT 0x0
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT_MASK 0x700
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT__SHIFT 0x8
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS_MASK 0xff00000
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS__SHIFT 0x14
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT_MASK 0x70000000
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT__SHIFT 0x1c
+#define CP_ME_PREEMPTION__OBSOLETE_MASK 0x1
+#define CP_ME_PREEMPTION__OBSOLETE__SHIFT 0x0
+#define CP_RB0_RPTR__RB_RPTR_MASK 0xfffff
+#define CP_RB0_RPTR__RB_RPTR__SHIFT 0x0
+#define CP_RB_RPTR__RB_RPTR_MASK 0xfffff
+#define CP_RB_RPTR__RB_RPTR__SHIFT 0x0
+#define CP_RB1_RPTR__RB_RPTR_MASK 0xfffff
+#define CP_RB1_RPTR__RB_RPTR__SHIFT 0x0
+#define CP_RB2_RPTR__RB_RPTR_MASK 0xfffff
+#define CP_RB2_RPTR__RB_RPTR__SHIFT 0x0
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER_MASK 0xfffffff
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER__SHIFT 0x0
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT_MASK 0xf0000000
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT__SHIFT 0x1c
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK 0xffff
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT 0x0
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK 0xffff0000
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT 0x10
+#define CP_CE_INIT_BASE_LO__INIT_BASE_LO_MASK 0xffffffe0
+#define CP_CE_INIT_BASE_LO__INIT_BASE_LO__SHIFT 0x5
+#define CP_CE_INIT_BASE_HI__INIT_BASE_HI_MASK 0xffff
+#define CP_CE_INIT_BASE_HI__INIT_BASE_HI__SHIFT 0x0
+#define CP_CE_INIT_BUFSZ__INIT_BUFSZ_MASK 0xfff
+#define CP_CE_INIT_BUFSZ__INIT_BUFSZ__SHIFT 0x0
+#define CP_CE_IB1_BASE_LO__IB1_BASE_LO_MASK 0xfffffffc
+#define CP_CE_IB1_BASE_LO__IB1_BASE_LO__SHIFT 0x2
+#define CP_CE_IB1_BASE_HI__IB1_BASE_HI_MASK 0xffff
+#define CP_CE_IB1_BASE_HI__IB1_BASE_HI__SHIFT 0x0
+#define CP_CE_IB1_BUFSZ__IB1_BUFSZ_MASK 0xfffff
+#define CP_CE_IB1_BUFSZ__IB1_BUFSZ__SHIFT 0x0
+#define CP_CE_IB2_BASE_LO__IB2_BASE_LO_MASK 0xfffffffc
+#define CP_CE_IB2_BASE_LO__IB2_BASE_LO__SHIFT 0x2
+#define CP_CE_IB2_BASE_HI__IB2_BASE_HI_MASK 0xffff
+#define CP_CE_IB2_BASE_HI__IB2_BASE_HI__SHIFT 0x0
+#define CP_CE_IB2_BUFSZ__IB2_BUFSZ_MASK 0xfffff
+#define CP_CE_IB2_BUFSZ__IB2_BUFSZ__SHIFT 0x0
+#define CP_IB1_BASE_LO__IB1_BASE_LO_MASK 0xfffffffc
+#define CP_IB1_BASE_LO__IB1_BASE_LO__SHIFT 0x2
+#define CP_IB1_BASE_HI__IB1_BASE_HI_MASK 0xffff
+#define CP_IB1_BASE_HI__IB1_BASE_HI__SHIFT 0x0
+#define CP_IB1_BUFSZ__IB1_BUFSZ_MASK 0xfffff
+#define CP_IB1_BUFSZ__IB1_BUFSZ__SHIFT 0x0
+#define CP_IB2_BASE_LO__IB2_BASE_LO_MASK 0xfffffffc
+#define CP_IB2_BASE_LO__IB2_BASE_LO__SHIFT 0x2
+#define CP_IB2_BASE_HI__IB2_BASE_HI_MASK 0xffff
+#define CP_IB2_BASE_HI__IB2_BASE_HI__SHIFT 0x0
+#define CP_IB2_BUFSZ__IB2_BUFSZ_MASK 0xfffff
+#define CP_IB2_BUFSZ__IB2_BUFSZ__SHIFT 0x0
+#define CP_ST_BASE_LO__ST_BASE_LO_MASK 0xfffffffc
+#define CP_ST_BASE_LO__ST_BASE_LO__SHIFT 0x2
+#define CP_ST_BASE_HI__ST_BASE_HI_MASK 0xffff
+#define CP_ST_BASE_HI__ST_BASE_HI__SHIFT 0x0
+#define CP_ST_BUFSZ__ST_BUFSZ_MASK 0xfffff
+#define CP_ST_BUFSZ__ST_BUFSZ__SHIFT 0x0
+#define CP_ROQ_THRESHOLDS__IB1_START_MASK 0xff
+#define CP_ROQ_THRESHOLDS__IB1_START__SHIFT 0x0
+#define CP_ROQ_THRESHOLDS__IB2_START_MASK 0xff00
+#define CP_ROQ_THRESHOLDS__IB2_START__SHIFT 0x8
+#define CP_MEQ_STQ_THRESHOLD__STQ_START_MASK 0xff
+#define CP_MEQ_STQ_THRESHOLD__STQ_START__SHIFT 0x0
+#define CP_ROQ1_THRESHOLDS__RB1_START_MASK 0xff
+#define CP_ROQ1_THRESHOLDS__RB1_START__SHIFT 0x0
+#define CP_ROQ1_THRESHOLDS__RB2_START_MASK 0xff00
+#define CP_ROQ1_THRESHOLDS__RB2_START__SHIFT 0x8
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START_MASK 0xff0000
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START__SHIFT 0x10
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START_MASK 0xff000000
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START__SHIFT 0x18
+#define CP_ROQ2_THRESHOLDS__R2_IB1_START_MASK 0xff
+#define CP_ROQ2_THRESHOLDS__R2_IB1_START__SHIFT 0x0
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START_MASK 0xff00
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START__SHIFT 0x8
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START_MASK 0xff0000
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START__SHIFT 0x10
+#define CP_ROQ2_THRESHOLDS__R2_IB2_START_MASK 0xff000000
+#define CP_ROQ2_THRESHOLDS__R2_IB2_START__SHIFT 0x18
+#define CP_STQ_THRESHOLDS__STQ0_START_MASK 0xff
+#define CP_STQ_THRESHOLDS__STQ0_START__SHIFT 0x0
+#define CP_STQ_THRESHOLDS__STQ1_START_MASK 0xff00
+#define CP_STQ_THRESHOLDS__STQ1_START__SHIFT 0x8
+#define CP_STQ_THRESHOLDS__STQ2_START_MASK 0xff0000
+#define CP_STQ_THRESHOLDS__STQ2_START__SHIFT 0x10
+#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START_MASK 0x3f
+#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START__SHIFT 0x0
+#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START_MASK 0x3f00
+#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START__SHIFT 0x8
+#define CP_MEQ_THRESHOLDS__MEQ1_START_MASK 0xff
+#define CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT 0x0
+#define CP_MEQ_THRESHOLDS__MEQ2_START_MASK 0xff00
+#define CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT 0x8
+#define CP_ROQ_AVAIL__ROQ_CNT_RING_MASK 0x7ff
+#define CP_ROQ_AVAIL__ROQ_CNT_RING__SHIFT 0x0
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1_MASK 0x7ff0000
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1__SHIFT 0x10
+#define CP_STQ_AVAIL__STQ_CNT_MASK 0x1ff
+#define CP_STQ_AVAIL__STQ_CNT__SHIFT 0x0
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2_MASK 0x7ff
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2__SHIFT 0x0
+#define CP_MEQ_AVAIL__MEQ_CNT_MASK 0x3ff
+#define CP_MEQ_AVAIL__MEQ_CNT__SHIFT 0x0
+#define CP_CMD_INDEX__CMD_INDEX_MASK 0x7ff
+#define CP_CMD_INDEX__CMD_INDEX__SHIFT 0x0
+#define CP_CMD_INDEX__CMD_ME_SEL_MASK 0x3000
+#define CP_CMD_INDEX__CMD_ME_SEL__SHIFT 0xc
+#define CP_CMD_INDEX__CMD_QUEUE_SEL_MASK 0x70000
+#define CP_CMD_INDEX__CMD_QUEUE_SEL__SHIFT 0x10
+#define CP_CMD_DATA__CMD_DATA_MASK 0xffffffff
+#define CP_CMD_DATA__CMD_DATA__SHIFT 0x0
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY_MASK 0x3ff
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY__SHIFT 0x0
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY_MASK 0x3ff0000
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY__SHIFT 0x10
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1_MASK 0x3ff
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1__SHIFT 0x0
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1_MASK 0x3ff0000
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1__SHIFT 0x10
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2_MASK 0x3ff
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2__SHIFT 0x0
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2_MASK 0x3ff0000
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2__SHIFT 0x10
+#define CP_STQ_STAT__STQ_RPTR_MASK 0x3ff
+#define CP_STQ_STAT__STQ_RPTR__SHIFT 0x0
+#define CP_STQ_WR_STAT__STQ_WPTR_MASK 0x3ff
+#define CP_STQ_WR_STAT__STQ_WPTR__SHIFT 0x0
+#define CP_MEQ_STAT__MEQ_RPTR_MASK 0x3ff
+#define CP_MEQ_STAT__MEQ_RPTR__SHIFT 0x0
+#define CP_MEQ_STAT__MEQ_WPTR_MASK 0x3ff0000
+#define CP_MEQ_STAT__MEQ_WPTR__SHIFT 0x10
+#define CP_CEQ1_AVAIL__CEQ_CNT_RING_MASK 0x7ff
+#define CP_CEQ1_AVAIL__CEQ_CNT_RING__SHIFT 0x0
+#define CP_CEQ1_AVAIL__CEQ_CNT_IB1_MASK 0x7ff0000
+#define CP_CEQ1_AVAIL__CEQ_CNT_IB1__SHIFT 0x10
+#define CP_CEQ2_AVAIL__CEQ_CNT_IB2_MASK 0x7ff
+#define CP_CEQ2_AVAIL__CEQ_CNT_IB2__SHIFT 0x0
+#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY_MASK 0x3ff
+#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY__SHIFT 0x0
+#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY_MASK 0x3ff0000
+#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY__SHIFT 0x10
+#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1_MASK 0x3ff
+#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1__SHIFT 0x0
+#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1_MASK 0x3ff0000
+#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1__SHIFT 0x10
+#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2_MASK 0x3ff
+#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2__SHIFT 0x0
+#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2_MASK 0x3ff0000
+#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2__SHIFT 0x10
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED_MASK 0x800
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED__SHIFT 0xb
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x4000
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x20000
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED_MASK 0x40000
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED__SHIFT 0x12
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED_MASK 0x80000
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED__SHIFT 0x13
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED_MASK 0x100000
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED__SHIFT 0x14
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED_MASK 0x200000
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED__SHIFT 0x15
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED_MASK 0x400000
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED__SHIFT 0x16
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x800000
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x1000000
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x4000000
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x8000000
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f
+#define CP_PERFMON_CNTL__PERFMON_STATE_MASK 0xf
+#define CP_PERFMON_CNTL__PERFMON_STATE__SHIFT 0x0
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE_MASK 0xf0
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT 0x4
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK 0x300
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT 0x8
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK 0x400
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT 0xa
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK 0x80000000
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT 0x1f
+#define CP_RINGID__RINGID_MASK 0x3
+#define CP_RINGID__RINGID__SHIFT 0x0
+#define CP_PIPEID__PIPE_ID_MASK 0x3
+#define CP_PIPEID__PIPE_ID__SHIFT 0x0
+#define CP_VMID__VMID_MASK 0xf
+#define CP_VMID__VMID__SHIFT 0x0
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET_MASK 0x7
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET__SHIFT 0x0
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET_MASK 0x3f00
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET__SHIFT 0x8
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET_MASK 0x3f0000
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET__SHIFT 0x10
+#define CP_HPD_STATUS0__QUEUE_STATE_MASK 0x1f
+#define CP_HPD_STATUS0__QUEUE_STATE__SHIFT 0x0
+#define CP_HPD_STATUS0__MAPPED_QUEUE_MASK 0xe0
+#define CP_HPD_STATUS0__MAPPED_QUEUE__SHIFT 0x5
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE_MASK 0xff00
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE__SHIFT 0x8
+#define CP_MQD_BASE_ADDR__BASE_ADDR_MASK 0xfffffffc
+#define CP_MQD_BASE_ADDR__BASE_ADDR__SHIFT 0x2
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK 0xffff
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT 0x0
+#define CP_HQD_ACTIVE__ACTIVE_MASK 0x1
+#define CP_HQD_ACTIVE__ACTIVE__SHIFT 0x0
+#define CP_HQD_ACTIVE__BUSY_GATE_MASK 0x2
+#define CP_HQD_ACTIVE__BUSY_GATE__SHIFT 0x1
+#define CP_HQD_VMID__VMID_MASK 0xf
+#define CP_HQD_VMID__VMID__SHIFT 0x0
+#define CP_HQD_VMID__IB_VMID_MASK 0xf00
+#define CP_HQD_VMID__IB_VMID__SHIFT 0x8
+#define CP_HQD_VMID__VQID_MASK 0x3ff0000
+#define CP_HQD_VMID__VQID__SHIFT 0x10
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK 0x1
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ__SHIFT 0x0
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE_MASK 0x3ff00
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT 0x8
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE_MASK 0x10000000
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE__SHIFT 0x1c
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES_MASK 0x20000000
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES__SHIFT 0x1d
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE_MASK 0x40000000
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT 0x1e
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE_MASK 0x80000000
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE__SHIFT 0x1f
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY_MASK 0x3
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY__SHIFT 0x0
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK 0xf
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT 0x0
+#define CP_HQD_QUANTUM__QUANTUM_EN_MASK 0x1
+#define CP_HQD_QUANTUM__QUANTUM_EN__SHIFT 0x0
+#define CP_HQD_QUANTUM__QUANTUM_SCALE_MASK 0x10
+#define CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT 0x4
+#define CP_HQD_QUANTUM__QUANTUM_DURATION_MASK 0x3f00
+#define CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT 0x8
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE_MASK 0x80000000
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE__SHIFT 0x1f
+#define CP_HQD_PQ_BASE__ADDR_MASK 0xffffffff
+#define CP_HQD_PQ_BASE__ADDR__SHIFT 0x0
+#define CP_HQD_PQ_BASE_HI__ADDR_HI_MASK 0xff
+#define CP_HQD_PQ_BASE_HI__ADDR_HI__SHIFT 0x0
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET_MASK 0xffffffff
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET__SHIFT 0x0
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR_MASK 0xfffffffc
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR__SHIFT 0x2
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI_MASK 0xffff
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI__SHIFT 0x0
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR_MASK 0xfffffffc
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR__SHIFT 0x2
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI_MASK 0xffff
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI__SHIFT 0x0
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE_MASK 0x1
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE__SHIFT 0x0
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK 0x2
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT 0x1
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK 0x7ffffc
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT 0x2
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_CARRY_BITS_MASK 0x3800000
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_CARRY_BITS__SHIFT 0x17
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK 0x10000000
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE__SHIFT 0x1c
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT_MASK 0x20000000
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT__SHIFT 0x1d
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK 0x40000000
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT 0x1e
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK 0x80000000
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT 0x1f
+#define CP_HQD_PQ_WPTR__OFFSET_MASK 0xffffffff
+#define CP_HQD_PQ_WPTR__OFFSET__SHIFT 0x0
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK 0x3f
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE__SHIFT 0x0
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK 0x3f00
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT 0x8
+#define CP_HQD_PQ_CONTROL__MTYPE_MASK 0x18000
+#define CP_HQD_PQ_CONTROL__MTYPE__SHIFT 0xf
+#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP_MASK 0x60000
+#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP__SHIFT 0x11
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE_MASK 0x300000
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE__SHIFT 0x14
+#define CP_HQD_PQ_CONTROL__PQ_ATC_MASK 0x800000
+#define CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT 0x17
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY_MASK 0x1000000
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY__SHIFT 0x18
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR_MASK 0x6000000
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT 0x19
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK 0x8000000
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR__SHIFT 0x1b
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK 0x10000000
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH__SHIFT 0x1c
+#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP_MASK 0x20000000
+#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP__SHIFT 0x1d
+#define CP_HQD_PQ_CONTROL__PRIV_STATE_MASK 0x40000000
+#define CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT 0x1e
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK 0x80000000
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT 0x1f
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR_MASK 0xfffffffc
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR__SHIFT 0x2
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI_MASK 0xffff
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI__SHIFT 0x0
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET_MASK 0xfffff
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET__SHIFT 0x0
+#define CP_HQD_IB_CONTROL__IB_SIZE_MASK 0xfffff
+#define CP_HQD_IB_CONTROL__IB_SIZE__SHIFT 0x0
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE_MASK 0x300000
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT 0x14
+#define CP_HQD_IB_CONTROL__IB_ATC_MASK 0x800000
+#define CP_HQD_IB_CONTROL__IB_ATC__SHIFT 0x17
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY_MASK 0x1000000
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY__SHIFT 0x18
+#define CP_HQD_IB_CONTROL__MTYPE_MASK 0x18000000
+#define CP_HQD_IB_CONTROL__MTYPE__SHIFT 0x1b
+#define CP_HQD_IB_CONTROL__PROCESSING_IB_MASK 0x80000000
+#define CP_HQD_IB_CONTROL__PROCESSING_IB__SHIFT 0x1f
+#define CP_HQD_IQ_TIMER__WAIT_TIME_MASK 0xff
+#define CP_HQD_IQ_TIMER__WAIT_TIME__SHIFT 0x0
+#define CP_HQD_IQ_TIMER__RETRY_TYPE_MASK 0x700
+#define CP_HQD_IQ_TIMER__RETRY_TYPE__SHIFT 0x8
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE_MASK 0x800
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT 0xb
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK 0x3000
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT 0xc
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT_MASK 0xc000
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT__SHIFT 0xe
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE_MASK 0x3f0000
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE__SHIFT 0x10
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER_MASK 0x400000
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER__SHIFT 0x16
+#define CP_HQD_IQ_TIMER__IQ_ATC_MASK 0x800000
+#define CP_HQD_IQ_TIMER__IQ_ATC__SHIFT 0x17
+#define CP_HQD_IQ_TIMER__CACHE_POLICY_MASK 0x1000000
+#define CP_HQD_IQ_TIMER__CACHE_POLICY__SHIFT 0x18
+#define CP_HQD_IQ_TIMER__MTYPE_MASK 0x18000000
+#define CP_HQD_IQ_TIMER__MTYPE__SHIFT 0x1b
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN_MASK 0x20000000
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN__SHIFT 0x1d
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ_MASK 0x40000000
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ__SHIFT 0x1e
+#define CP_HQD_IQ_TIMER__ACTIVE_MASK 0x80000000
+#define CP_HQD_IQ_TIMER__ACTIVE__SHIFT 0x1f
+#define CP_HQD_IQ_RPTR__OFFSET_MASK 0x3f
+#define CP_HQD_IQ_RPTR__OFFSET__SHIFT 0x0
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK 0x7
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT 0x0
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK 0x10
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT 0x4
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT_MASK 0x100
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT__SHIFT 0x8
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN_MASK 0x200
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN__SHIFT 0x9
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN_MASK 0x400
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN__SHIFT 0xa
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_MASK 0x1
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD__SHIFT 0x0
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_MASK 0x1
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD__SHIFT 0x0
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN_MASK 0x2
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN__SHIFT 0x1
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_MASK 0x10
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD__SHIFT 0x4
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN_MASK 0x20
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN__SHIFT 0x5
+#define CP_HQD_SEMA_CMD__RETRY_MASK 0x1
+#define CP_HQD_SEMA_CMD__RETRY__SHIFT 0x0
+#define CP_HQD_SEMA_CMD__RESULT_MASK 0x6
+#define CP_HQD_SEMA_CMD__RESULT__SHIFT 0x1
+#define CP_HQD_MSG_TYPE__ACTION_MASK 0x7
+#define CP_HQD_MSG_TYPE__ACTION__SHIFT 0x0
+#define CP_HQD_MSG_TYPE__SAVE_STATE_MASK 0x70
+#define CP_HQD_MSG_TYPE__SAVE_STATE__SHIFT 0x4
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO_MASK 0xffffffff
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO__SHIFT 0x0
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI_MASK 0xffffffff
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI__SHIFT 0x0
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO_MASK 0xffffffff
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO__SHIFT 0x0
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI_MASK 0xffffffff
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI__SHIFT 0x0
+#define CP_HQD_HQ_SCHEDULER0__SCHEDULER_MASK 0xffffffff
+#define CP_HQD_HQ_SCHEDULER0__SCHEDULER__SHIFT 0x0
+#define CP_HQD_HQ_STATUS0__DEQUEUE_STATUS_MASK 0x3
+#define CP_HQD_HQ_STATUS0__DEQUEUE_STATUS__SHIFT 0x0
+#define CP_HQD_HQ_STATUS0__DEQUEUE_RETRY_CNT_MASK 0xc
+#define CP_HQD_HQ_STATUS0__DEQUEUE_RETRY_CNT__SHIFT 0x2
+#define CP_HQD_HQ_STATUS0__RSV_6_4_MASK 0x70
+#define CP_HQD_HQ_STATUS0__RSV_6_4__SHIFT 0x4
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT_MASK 0x80
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT__SHIFT 0x7
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY_MASK 0x100
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY__SHIFT 0x8
+#define CP_HQD_HQ_STATUS0__PG_ACTIVATED_MASK 0x200
+#define CP_HQD_HQ_STATUS0__PG_ACTIVATED__SHIFT 0x9
+#define CP_HQD_HQ_STATUS0__RSVR_31_10_MASK 0xfffffc00
+#define CP_HQD_HQ_STATUS0__RSVR_31_10__SHIFT 0xa
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER_MASK 0xffffffff
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER__SHIFT 0x0
+#define CP_HQD_HQ_CONTROL0__CONTROL_MASK 0xffffffff
+#define CP_HQD_HQ_CONTROL0__CONTROL__SHIFT 0x0
+#define CP_MQD_CONTROL__VMID_MASK 0xf
+#define CP_MQD_CONTROL__VMID__SHIFT 0x0
+#define CP_MQD_CONTROL__PROCESSING_MQD_MASK 0x1000
+#define CP_MQD_CONTROL__PROCESSING_MQD__SHIFT 0xc
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN_MASK 0x2000
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN__SHIFT 0xd
+#define CP_MQD_CONTROL__MQD_ATC_MASK 0x800000
+#define CP_MQD_CONTROL__MQD_ATC__SHIFT 0x17
+#define CP_MQD_CONTROL__CACHE_POLICY_MASK 0x1000000
+#define CP_MQD_CONTROL__CACHE_POLICY__SHIFT 0x18
+#define CP_MQD_CONTROL__MTYPE_MASK 0x18000000
+#define CP_MQD_CONTROL__MTYPE__SHIFT 0x1b
+#define CP_HQD_HQ_STATUS1__STATUS_MASK 0xffffffff
+#define CP_HQD_HQ_STATUS1__STATUS__SHIFT 0x0
+#define CP_HQD_HQ_CONTROL1__CONTROL_MASK 0xffffffff
+#define CP_HQD_HQ_CONTROL1__CONTROL__SHIFT 0x0
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR_MASK 0xffffffff
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR__SHIFT 0x0
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI_MASK 0xff
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT 0x0
+#define CP_HQD_EOP_CONTROL__EOP_SIZE_MASK 0x3f
+#define CP_HQD_EOP_CONTROL__EOP_SIZE__SHIFT 0x0
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP_MASK 0x100
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP__SHIFT 0x8
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN_MASK 0x1000
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN__SHIFT 0xc
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB_MASK 0x2000
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB__SHIFT 0xd
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN_MASK 0x4000
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN__SHIFT 0xe
+#define CP_HQD_EOP_CONTROL__MTYPE_MASK 0x18000
+#define CP_HQD_EOP_CONTROL__MTYPE__SHIFT 0xf
+#define CP_HQD_EOP_CONTROL__EOP_ATC_MASK 0x800000
+#define CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT 0x17
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY_MASK 0x1000000
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY__SHIFT 0x18
+#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT_MASK 0x60000000
+#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT__SHIFT 0x1d
+#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM_MASK 0x80000000
+#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM__SHIFT 0x1f
+#define CP_HQD_EOP_RPTR__RPTR_MASK 0x1fff
+#define CP_HQD_EOP_RPTR__RPTR__SHIFT 0x0
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR_MASK 0x40000000
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR__SHIFT 0x1e
+#define CP_HQD_EOP_RPTR__INIT_FETCHER_MASK 0x80000000
+#define CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT 0x1f
+#define CP_HQD_EOP_WPTR__WPTR_MASK 0x1fff
+#define CP_HQD_EOP_WPTR__WPTR__SHIFT 0x0
+#define CP_HQD_EOP_WPTR__EOP_AVAIL_MASK 0x1fff0000
+#define CP_HQD_EOP_WPTR__EOP_AVAIL__SHIFT 0x10
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT_MASK 0xfff
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT__SHIFT 0x0
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND_MASK 0x10000
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND__SHIFT 0x10
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK 0xfffff000
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT 0xc
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK 0xffff
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT 0x0
+#define CP_HQD_CTX_SAVE_CONTROL__ATC_MASK 0x1
+#define CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT 0x0
+#define CP_HQD_CTX_SAVE_CONTROL__MTYPE_MASK 0x6
+#define CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT 0x1
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY_MASK 0x8
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY__SHIFT 0x3
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET_MASK 0x7ffc
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET__SHIFT 0x2
+#define CP_HQD_CNTL_STACK_SIZE__SIZE_MASK 0x7000
+#define CP_HQD_CNTL_STACK_SIZE__SIZE__SHIFT 0xc
+#define CP_HQD_WG_STATE_OFFSET__OFFSET_MASK 0x1fffffc
+#define CP_HQD_WG_STATE_OFFSET__OFFSET__SHIFT 0x2
+#define CP_HQD_CTX_SAVE_SIZE__SIZE_MASK 0x1fff000
+#define CP_HQD_CTX_SAVE_SIZE__SIZE__SHIFT 0xc
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED_MASK 0x1
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED__SHIFT 0x0
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED_MASK 0x2
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED__SHIFT 0x1
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE_MASK 0x3f0
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE__SHIFT 0x4
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR_MASK 0x3f000
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR__SHIFT 0xc
+#define CP_HQD_ERROR__EDC_ERROR_ID_MASK 0xf
+#define CP_HQD_ERROR__EDC_ERROR_ID__SHIFT 0x0
+#define CP_HQD_ERROR__SUA_ERROR_MASK 0x10
+#define CP_HQD_ERROR__SUA_ERROR__SHIFT 0x4
+#define CP_HQD_EOP_WPTR_MEM__WPTR_MASK 0x1fff
+#define CP_HQD_EOP_WPTR_MEM__WPTR__SHIFT 0x0
+#define CP_HQD_EOP_DONES__DONE_COUNT_MASK 0xffffffff
+#define CP_HQD_EOP_DONES__DONE_COUNT__SHIFT 0x0
+#define DB_Z_READ_BASE__BASE_256B_MASK 0xffffffff
+#define DB_Z_READ_BASE__BASE_256B__SHIFT 0x0
+#define DB_STENCIL_READ_BASE__BASE_256B_MASK 0xffffffff
+#define DB_STENCIL_READ_BASE__BASE_256B__SHIFT 0x0
+#define DB_Z_WRITE_BASE__BASE_256B_MASK 0xffffffff
+#define DB_Z_WRITE_BASE__BASE_256B__SHIFT 0x0
+#define DB_STENCIL_WRITE_BASE__BASE_256B_MASK 0xffffffff
+#define DB_STENCIL_WRITE_BASE__BASE_256B__SHIFT 0x0
+#define DB_DEPTH_INFO__ADDR5_SWIZZLE_MASK_MASK 0xf
+#define DB_DEPTH_INFO__ADDR5_SWIZZLE_MASK__SHIFT 0x0
+#define DB_DEPTH_INFO__ARRAY_MODE_MASK 0xf0
+#define DB_DEPTH_INFO__ARRAY_MODE__SHIFT 0x4
+#define DB_DEPTH_INFO__PIPE_CONFIG_MASK 0x1f00
+#define DB_DEPTH_INFO__PIPE_CONFIG__SHIFT 0x8
+#define DB_DEPTH_INFO__BANK_WIDTH_MASK 0x6000
+#define DB_DEPTH_INFO__BANK_WIDTH__SHIFT 0xd
+#define DB_DEPTH_INFO__BANK_HEIGHT_MASK 0x18000
+#define DB_DEPTH_INFO__BANK_HEIGHT__SHIFT 0xf
+#define DB_DEPTH_INFO__MACRO_TILE_ASPECT_MASK 0x60000
+#define DB_DEPTH_INFO__MACRO_TILE_ASPECT__SHIFT 0x11
+#define DB_DEPTH_INFO__NUM_BANKS_MASK 0x180000
+#define DB_DEPTH_INFO__NUM_BANKS__SHIFT 0x13
+#define DB_Z_INFO__FORMAT_MASK 0x3
+#define DB_Z_INFO__FORMAT__SHIFT 0x0
+#define DB_Z_INFO__NUM_SAMPLES_MASK 0xc
+#define DB_Z_INFO__NUM_SAMPLES__SHIFT 0x2
+#define DB_Z_INFO__TILE_SPLIT_MASK 0xe000
+#define DB_Z_INFO__TILE_SPLIT__SHIFT 0xd
+#define DB_Z_INFO__TILE_MODE_INDEX_MASK 0x700000
+#define DB_Z_INFO__TILE_MODE_INDEX__SHIFT 0x14
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES_MASK 0x7800000
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES__SHIFT 0x17
+#define DB_Z_INFO__ALLOW_EXPCLEAR_MASK 0x8000000
+#define DB_Z_INFO__ALLOW_EXPCLEAR__SHIFT 0x1b
+#define DB_Z_INFO__READ_SIZE_MASK 0x10000000
+#define DB_Z_INFO__READ_SIZE__SHIFT 0x1c
+#define DB_Z_INFO__TILE_SURFACE_ENABLE_MASK 0x20000000
+#define DB_Z_INFO__TILE_SURFACE_ENABLE__SHIFT 0x1d
+#define DB_Z_INFO__CLEAR_DISALLOWED_MASK 0x40000000
+#define DB_Z_INFO__CLEAR_DISALLOWED__SHIFT 0x1e
+#define DB_Z_INFO__ZRANGE_PRECISION_MASK 0x80000000
+#define DB_Z_INFO__ZRANGE_PRECISION__SHIFT 0x1f
+#define DB_STENCIL_INFO__FORMAT_MASK 0x1
+#define DB_STENCIL_INFO__FORMAT__SHIFT 0x0
+#define DB_STENCIL_INFO__TILE_SPLIT_MASK 0xe000
+#define DB_STENCIL_INFO__TILE_SPLIT__SHIFT 0xd
+#define DB_STENCIL_INFO__TILE_MODE_INDEX_MASK 0x700000
+#define DB_STENCIL_INFO__TILE_MODE_INDEX__SHIFT 0x14
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR_MASK 0x8000000
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR__SHIFT 0x1b
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE_MASK 0x20000000
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE__SHIFT 0x1d
+#define DB_STENCIL_INFO__CLEAR_DISALLOWED_MASK 0x40000000
+#define DB_STENCIL_INFO__CLEAR_DISALLOWED__SHIFT 0x1e
+#define DB_DEPTH_SIZE__PITCH_TILE_MAX_MASK 0x7ff
+#define DB_DEPTH_SIZE__PITCH_TILE_MAX__SHIFT 0x0
+#define DB_DEPTH_SIZE__HEIGHT_TILE_MAX_MASK 0x3ff800
+#define DB_DEPTH_SIZE__HEIGHT_TILE_MAX__SHIFT 0xb
+#define DB_DEPTH_SLICE__SLICE_TILE_MAX_MASK 0x3fffff
+#define DB_DEPTH_SLICE__SLICE_TILE_MAX__SHIFT 0x0
+#define DB_DEPTH_VIEW__SLICE_START_MASK 0x7ff
+#define DB_DEPTH_VIEW__SLICE_START__SHIFT 0x0
+#define DB_DEPTH_VIEW__SLICE_MAX_MASK 0xffe000
+#define DB_DEPTH_VIEW__SLICE_MAX__SHIFT 0xd
+#define DB_DEPTH_VIEW__Z_READ_ONLY_MASK 0x1000000
+#define DB_DEPTH_VIEW__Z_READ_ONLY__SHIFT 0x18
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY_MASK 0x2000000
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY__SHIFT 0x19
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE_MASK 0x1
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE__SHIFT 0x0
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE_MASK 0x2
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE__SHIFT 0x1
+#define DB_RENDER_CONTROL__DEPTH_COPY_MASK 0x4
+#define DB_RENDER_CONTROL__DEPTH_COPY__SHIFT 0x2
+#define DB_RENDER_CONTROL__STENCIL_COPY_MASK 0x8
+#define DB_RENDER_CONTROL__STENCIL_COPY__SHIFT 0x3
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE_MASK 0x10
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE__SHIFT 0x4
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE_MASK 0x20
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE__SHIFT 0x5
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE_MASK 0x40
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE__SHIFT 0x6
+#define DB_RENDER_CONTROL__COPY_CENTROID_MASK 0x80
+#define DB_RENDER_CONTROL__COPY_CENTROID__SHIFT 0x7
+#define DB_RENDER_CONTROL__COPY_SAMPLE_MASK 0xf00
+#define DB_RENDER_CONTROL__COPY_SAMPLE__SHIFT 0x8
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE_MASK 0x1000
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE__SHIFT 0xc
+#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE_MASK 0x1
+#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE__SHIFT 0x0
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS_MASK 0x2
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS__SHIFT 0x1
+#define DB_COUNT_CONTROL__SAMPLE_RATE_MASK 0x70
+#define DB_COUNT_CONTROL__SAMPLE_RATE__SHIFT 0x4
+#define DB_COUNT_CONTROL__ZPASS_ENABLE_MASK 0xf00
+#define DB_COUNT_CONTROL__ZPASS_ENABLE__SHIFT 0x8
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE_MASK 0xf000
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE__SHIFT 0xc
+#define DB_COUNT_CONTROL__SFAIL_ENABLE_MASK 0xf0000
+#define DB_COUNT_CONTROL__SFAIL_ENABLE__SHIFT 0x10
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE_MASK 0xf00000
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE__SHIFT 0x14
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE_MASK 0xf000000
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE__SHIFT 0x18
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE_MASK 0xf0000000
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE__SHIFT 0x1c
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK 0x3
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT 0x0
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0_MASK 0xc
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0__SHIFT 0x2
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1_MASK 0x30
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1__SHIFT 0x4
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER_MASK 0x40
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER__SHIFT 0x6
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK 0x80
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT 0x7
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK 0x100
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT 0x8
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE_MASK 0x200
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE__SHIFT 0x9
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL_MASK 0x400
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL__SHIFT 0xa
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ_MASK 0x800
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ__SHIFT 0xb
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ_MASK 0x1000
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ__SHIFT 0xc
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK 0x6000
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT 0xd
+#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT_MASK 0x8000
+#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT__SHIFT 0xf
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP_MASK 0x10000
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP__SHIFT 0x10
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE_MASK 0x20000
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE__SHIFT 0x11
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED_MASK 0x40000
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED__SHIFT 0x12
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM_MASK 0x180000
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM__SHIFT 0x13
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT_MASK 0x3e00000
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT__SHIFT 0x15
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK 0x4000000
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT 0x1a
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY_MASK 0x8000000
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY__SHIFT 0x1b
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY_MASK 0x10000000
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY__SHIFT 0x1c
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID_MASK 0x20000000
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID__SHIFT 0x1d
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID_MASK 0x40000000
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID__SHIFT 0x1e
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION_MASK 0x80000000
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION__SHIFT 0x1f
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK 0x3
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL__SHIFT 0x0
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK 0x1c
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN__SHIFT 0x2
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION_MASK 0x20
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION__SHIFT 0x5
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION_MASK 0x40
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION__SHIFT 0x6
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION_MASK 0x80
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION__SHIFT 0x7
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH_MASK 0x100
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH__SHIFT 0x8
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP_MASK 0x200
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP__SHIFT 0x9
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK 0x400
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT 0xa
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE_MASK 0x800
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE__SHIFT 0xb
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC_MASK 0x7000
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC__SHIFT 0xc
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF_MASK 0x38000
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF__SHIFT 0xf
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF_MASK 0x1c0000
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF__SHIFT 0x12
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE_MASK 0x200000
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE__SHIFT 0x15
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS_MASK 0x400000
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS__SHIFT 0x16
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK 0x800000
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT 0x17
+#define DB_EQAA__MAX_ANCHOR_SAMPLES_MASK 0x7
+#define DB_EQAA__MAX_ANCHOR_SAMPLES__SHIFT 0x0
+#define DB_EQAA__PS_ITER_SAMPLES_MASK 0x70
+#define DB_EQAA__PS_ITER_SAMPLES__SHIFT 0x4
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES_MASK 0x700
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES__SHIFT 0x8
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES_MASK 0x7000
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES__SHIFT 0xc
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS_MASK 0x10000
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS__SHIFT 0x10
+#define DB_EQAA__INCOHERENT_EQAA_READS_MASK 0x20000
+#define DB_EQAA__INCOHERENT_EQAA_READS__SHIFT 0x11
+#define DB_EQAA__INTERPOLATE_COMP_Z_MASK 0x40000
+#define DB_EQAA__INTERPOLATE_COMP_Z__SHIFT 0x12
+#define DB_EQAA__INTERPOLATE_SRC_Z_MASK 0x80000
+#define DB_EQAA__INTERPOLATE_SRC_Z__SHIFT 0x13
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS_MASK 0x100000
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS__SHIFT 0x14
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE_MASK 0x200000
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE__SHIFT 0x15
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT_MASK 0x7000000
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT__SHIFT 0x18
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION_MASK 0x8000000
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION__SHIFT 0x1b
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE_MASK 0x1
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE__SHIFT 0x0
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE_MASK 0x2
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE__SHIFT 0x1
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE_MASK 0x4
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE__SHIFT 0x2
+#define DB_SHADER_CONTROL__Z_ORDER_MASK 0x30
+#define DB_SHADER_CONTROL__Z_ORDER__SHIFT 0x4
+#define DB_SHADER_CONTROL__KILL_ENABLE_MASK 0x40
+#define DB_SHADER_CONTROL__KILL_ENABLE__SHIFT 0x6
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE_MASK 0x80
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE__SHIFT 0x7
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE_MASK 0x100
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE__SHIFT 0x8
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL_MASK 0x200
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL__SHIFT 0x9
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP_MASK 0x400
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP__SHIFT 0xa
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE_MASK 0x800
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE__SHIFT 0xb
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER_MASK 0x1000
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER__SHIFT 0xc
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT_MASK 0x6000
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT 0xd
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE_MASK 0x8000
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE__SHIFT 0xf
+#define DB_DEPTH_BOUNDS_MIN__MIN_MASK 0xffffffff
+#define DB_DEPTH_BOUNDS_MIN__MIN__SHIFT 0x0
+#define DB_DEPTH_BOUNDS_MAX__MAX_MASK 0xffffffff
+#define DB_DEPTH_BOUNDS_MAX__MAX__SHIFT 0x0
+#define DB_STENCIL_CLEAR__CLEAR_MASK 0xff
+#define DB_STENCIL_CLEAR__CLEAR__SHIFT 0x0
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR_MASK 0xffffffff
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR__SHIFT 0x0
+#define DB_HTILE_DATA_BASE__BASE_256B_MASK 0xffffffff
+#define DB_HTILE_DATA_BASE__BASE_256B__SHIFT 0x0
+#define DB_HTILE_SURFACE__LINEAR_MASK 0x1
+#define DB_HTILE_SURFACE__LINEAR__SHIFT 0x0
+#define DB_HTILE_SURFACE__FULL_CACHE_MASK 0x2
+#define DB_HTILE_SURFACE__FULL_CACHE__SHIFT 0x1
+#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN_MASK 0x4
+#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN__SHIFT 0x2
+#define DB_HTILE_SURFACE__PRELOAD_MASK 0x8
+#define DB_HTILE_SURFACE__PRELOAD__SHIFT 0x3
+#define DB_HTILE_SURFACE__PREFETCH_WIDTH_MASK 0x3f0
+#define DB_HTILE_SURFACE__PREFETCH_WIDTH__SHIFT 0x4
+#define DB_HTILE_SURFACE__PREFETCH_HEIGHT_MASK 0xfc00
+#define DB_HTILE_SURFACE__PREFETCH_HEIGHT__SHIFT 0xa
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK 0x10000
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT 0x10
+#define DB_HTILE_SURFACE__TC_COMPATIBLE_MASK 0x20000
+#define DB_HTILE_SURFACE__TC_COMPATIBLE__SHIFT 0x11
+#define DB_PRELOAD_CONTROL__START_X_MASK 0xff
+#define DB_PRELOAD_CONTROL__START_X__SHIFT 0x0
+#define DB_PRELOAD_CONTROL__START_Y_MASK 0xff00
+#define DB_PRELOAD_CONTROL__START_Y__SHIFT 0x8
+#define DB_PRELOAD_CONTROL__MAX_X_MASK 0xff0000
+#define DB_PRELOAD_CONTROL__MAX_X__SHIFT 0x10
+#define DB_PRELOAD_CONTROL__MAX_Y_MASK 0xff000000
+#define DB_PRELOAD_CONTROL__MAX_Y__SHIFT 0x18
+#define DB_STENCILREFMASK__STENCILTESTVAL_MASK 0xff
+#define DB_STENCILREFMASK__STENCILTESTVAL__SHIFT 0x0
+#define DB_STENCILREFMASK__STENCILMASK_MASK 0xff00
+#define DB_STENCILREFMASK__STENCILMASK__SHIFT 0x8
+#define DB_STENCILREFMASK__STENCILWRITEMASK_MASK 0xff0000
+#define DB_STENCILREFMASK__STENCILWRITEMASK__SHIFT 0x10
+#define DB_STENCILREFMASK__STENCILOPVAL_MASK 0xff000000
+#define DB_STENCILREFMASK__STENCILOPVAL__SHIFT 0x18
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF_MASK 0xff
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF__SHIFT 0x0
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF_MASK 0xff00
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF__SHIFT 0x8
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF_MASK 0xff0000
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF__SHIFT 0x10
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF_MASK 0xff000000
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF__SHIFT 0x18
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0_MASK 0x7
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT 0x0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0_MASK 0xff0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT 0x4
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0_MASK 0xff000
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0__SHIFT 0xc
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0_MASK 0x1000000
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0__SHIFT 0x18
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1_MASK 0x7
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1__SHIFT 0x0
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1_MASK 0xff0
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1__SHIFT 0x4
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1_MASK 0xff000
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1__SHIFT 0xc
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1_MASK 0x1000000
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1__SHIFT 0x18
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE_MASK 0x1
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE__SHIFT 0x0
+#define DB_DEPTH_CONTROL__Z_ENABLE_MASK 0x2
+#define DB_DEPTH_CONTROL__Z_ENABLE__SHIFT 0x1
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE_MASK 0x4
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE__SHIFT 0x2
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE_MASK 0x8
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE__SHIFT 0x3
+#define DB_DEPTH_CONTROL__ZFUNC_MASK 0x70
+#define DB_DEPTH_CONTROL__ZFUNC__SHIFT 0x4
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE_MASK 0x80
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE__SHIFT 0x7
+#define DB_DEPTH_CONTROL__STENCILFUNC_MASK 0x700
+#define DB_DEPTH_CONTROL__STENCILFUNC__SHIFT 0x8
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF_MASK 0x700000
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF__SHIFT 0x14
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_MASK 0x40000000
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL__SHIFT 0x1e
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS_MASK 0x80000000
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS__SHIFT 0x1f
+#define DB_STENCIL_CONTROL__STENCILFAIL_MASK 0xf
+#define DB_STENCIL_CONTROL__STENCILFAIL__SHIFT 0x0
+#define DB_STENCIL_CONTROL__STENCILZPASS_MASK 0xf0
+#define DB_STENCIL_CONTROL__STENCILZPASS__SHIFT 0x4
+#define DB_STENCIL_CONTROL__STENCILZFAIL_MASK 0xf00
+#define DB_STENCIL_CONTROL__STENCILZFAIL__SHIFT 0x8
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF_MASK 0xf000
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF__SHIFT 0xc
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF_MASK 0xf0000
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF__SHIFT 0x10
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF_MASK 0xf00000
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF__SHIFT 0x14
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE_MASK 0x1
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE__SHIFT 0x0
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0_MASK 0x300
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0__SHIFT 0x8
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1_MASK 0xc00
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1__SHIFT 0xa
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2_MASK 0x3000
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2__SHIFT 0xc
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3_MASK 0xc000
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3__SHIFT 0xe
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND_MASK 0x10000
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND__SHIFT 0x10
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1_MASK 0xffc00
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT 0xa
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1_MASK 0xf000000
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT 0x18
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1_MASK 0xffc00
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT 0xa
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1_MASK 0xf000000
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT 0x18
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf000000
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x18
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE_MASK 0x1
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE__SHIFT 0x0
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE_MASK 0x2
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE__SHIFT 0x1
+#define DB_DEBUG__FETCH_FULL_Z_TILE_MASK 0x4
+#define DB_DEBUG__FETCH_FULL_Z_TILE__SHIFT 0x2
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE_MASK 0x8
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE__SHIFT 0x3
+#define DB_DEBUG__FORCE_Z_MODE_MASK 0x30
+#define DB_DEBUG__FORCE_Z_MODE__SHIFT 0x4
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ_MASK 0x40
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ__SHIFT 0x6
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ_MASK 0x80
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ__SHIFT 0x7
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE_MASK 0x300
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE__SHIFT 0x8
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0_MASK 0xc00
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0__SHIFT 0xa
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1_MASK 0x3000
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1__SHIFT 0xc
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE_MASK 0x4000
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE__SHIFT 0xe
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE_MASK 0x8000
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE__SHIFT 0xf
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE_MASK 0x10000
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE__SHIFT 0x10
+#define DB_DEBUG__DISABLE_SUMM_SQUADS_MASK 0x20000
+#define DB_DEBUG__DISABLE_SUMM_SQUADS__SHIFT 0x11
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS_MASK 0x40000
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS__SHIFT 0x12
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE_MASK 0x180000
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE__SHIFT 0x13
+#define DB_DEBUG__NEVER_FREE_Z_ONLY_MASK 0x200000
+#define DB_DEBUG__NEVER_FREE_Z_ONLY__SHIFT 0x15
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS_MASK 0x400000
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS__SHIFT 0x16
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK 0x800000
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT 0x17
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES_MASK 0xf000000
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES__SHIFT 0x18
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT_MASK 0x10000000
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT__SHIFT 0x1c
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT_MASK 0x20000000
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT__SHIFT 0x1d
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC_MASK 0x40000000
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC__SHIFT 0x1e
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC_MASK 0x80000000
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC__SHIFT 0x1f
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING_MASK 0x1
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING__SHIFT 0x0
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE_MASK 0x2
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE__SHIFT 0x1
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE_MASK 0x4
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE__SHIFT 0x2
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB_MASK 0x8
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB__SHIFT 0x3
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM_MASK 0x10
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM__SHIFT 0x4
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_MASK 0x20
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL__SHIFT 0x5
+#define DB_DEBUG2__ENABLE_PREZL_CB_STALL_MASK 0x40
+#define DB_DEBUG2__ENABLE_PREZL_CB_STALL__SHIFT 0x6
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_REZ_MASK 0x80
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_REZ__SHIFT 0x7
+#define DB_DEBUG2__DISABLE_PREZL_CB_STALL_REZ_MASK 0x100
+#define DB_DEBUG2__DISABLE_PREZL_CB_STALL_REZ__SHIFT 0x8
+#define DB_DEBUG2__CLK_OFF_DELAY_MASK 0x3e00
+#define DB_DEBUG2__CLK_OFF_DELAY__SHIFT 0x9
+#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER_MASK 0x4000
+#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER__SHIFT 0xe
+#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING_MASK 0x8000
+#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING__SHIFT 0xf
+#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES_MASK 0x10000
+#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES__SHIFT 0x10
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING_MASK 0x20000
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING__SHIFT 0x11
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING_MASK 0x40000
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING__SHIFT 0x12
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL_MASK 0x80000
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL__SHIFT 0x13
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM_MASK 0x10000000
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM__SHIFT 0x1c
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL_MASK 0x20000000
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL__SHIFT 0x1d
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM_MASK 0x40000000
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM__SHIFT 0x1e
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT_MASK 0x80000000
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT__SHIFT 0x1f
+#define DB_DEBUG3__FORCE_DB_IS_GOOD_MASK 0x4
+#define DB_DEBUG3__FORCE_DB_IS_GOOD__SHIFT 0x2
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION_MASK 0x8
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION__SHIFT 0x3
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP_MASK 0x10
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT 0x4
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z_MASK 0x20
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z__SHIFT 0x5
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z_MASK 0x40
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z__SHIFT 0x6
+#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS_MASK 0x80
+#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS__SHIFT 0x7
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION_MASK 0x100
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION__SHIFT 0x8
+#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT_MASK 0x200
+#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT__SHIFT 0x9
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP_MASK 0x400
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP__SHIFT 0xa
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS_MASK 0x800
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS__SHIFT 0xb
+#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING_MASK 0x1000
+#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING__SHIFT 0xc
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS_MASK 0x2000
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS__SHIFT 0xd
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE_MASK 0x4000
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE__SHIFT 0xe
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK_MASK 0x8000
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK__SHIFT 0xf
+#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION_MASK 0x10000
+#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION__SHIFT 0x10
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE_MASK 0x20000
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE__SHIFT 0x11
+#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING_MASK 0x40000
+#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING__SHIFT 0x12
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE_MASK 0x80000
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE__SHIFT 0x13
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE_MASK 0x100000
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE__SHIFT 0x14
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT_MASK 0x200000
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT__SHIFT 0x15
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB_MASK 0x400000
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB__SHIFT 0x16
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD_MASK 0x800000
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD__SHIFT 0x17
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT_MASK 0x1000000
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT__SHIFT 0x18
+#define DB_DEBUG3__DISABLE_DI_DT_STALL_MASK 0x2000000
+#define DB_DEBUG3__DISABLE_DI_DT_STALL__SHIFT 0x19
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET_MASK 0x4000000
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET__SHIFT 0x1a
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX_MASK 0x8000000
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX__SHIFT 0x1b
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND_MASK 0x10000000
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND__SHIFT 0x1c
+#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND_MASK 0x20000000
+#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND__SHIFT 0x1d
+#define DB_DEBUG3__DISABLE_4XAA_2P_DELAYED_WRITE_MASK 0x40000000
+#define DB_DEBUG3__DISABLE_4XAA_2P_DELAYED_WRITE__SHIFT 0x1e
+#define DB_DEBUG3__DISABLE_4XAA_2P_INTERLEAVED_PMASK_MASK 0x80000000
+#define DB_DEBUG3__DISABLE_4XAA_2P_INTERLEAVED_PMASK__SHIFT 0x1f
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION_MASK 0x1
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION__SHIFT 0x0
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION_MASK 0x2
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION__SHIFT 0x1
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL_MASK 0x4
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL__SHIFT 0x2
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL_MASK 0x8
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL__SHIFT 0x3
+#define DB_DEBUG4__DISABLE_4XAA_2P_ZD_HOLDOFF_MASK 0x10
+#define DB_DEBUG4__DISABLE_4XAA_2P_ZD_HOLDOFF__SHIFT 0x4
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION_MASK 0x20
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION__SHIFT 0x5
+#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE_MASK 0x40
+#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE__SHIFT 0x6
+#define DB_DEBUG4__DB_EXTRA_DEBUG4_MASK 0xffffff80
+#define DB_DEBUG4__DB_EXTRA_DEBUG4__SHIFT 0x7
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS_MASK 0x1f
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS__SHIFT 0x0
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS_MASK 0x3e0
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS__SHIFT 0x5
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS_MASK 0x1c00
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS__SHIFT 0xa
+#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS_MASK 0x7f000000
+#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS__SHIFT 0x18
+#define DB_WATERMARKS__DEPTH_FREE_MASK 0x1f
+#define DB_WATERMARKS__DEPTH_FREE__SHIFT 0x0
+#define DB_WATERMARKS__DEPTH_FLUSH_MASK 0x7e0
+#define DB_WATERMARKS__DEPTH_FLUSH__SHIFT 0x5
+#define DB_WATERMARKS__FORCE_SUMMARIZE_MASK 0x7800
+#define DB_WATERMARKS__FORCE_SUMMARIZE__SHIFT 0xb
+#define DB_WATERMARKS__DEPTH_PENDING_FREE_MASK 0xf8000
+#define DB_WATERMARKS__DEPTH_PENDING_FREE__SHIFT 0xf
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE_MASK 0x7f00000
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE__SHIFT 0x14
+#define DB_WATERMARKS__EARLY_Z_PANIC_DISABLE_MASK 0x8000000
+#define DB_WATERMARKS__EARLY_Z_PANIC_DISABLE__SHIFT 0x1b
+#define DB_WATERMARKS__LATE_Z_PANIC_DISABLE_MASK 0x10000000
+#define DB_WATERMARKS__LATE_Z_PANIC_DISABLE__SHIFT 0x1c
+#define DB_WATERMARKS__RE_Z_PANIC_DISABLE_MASK 0x20000000
+#define DB_WATERMARKS__RE_Z_PANIC_DISABLE__SHIFT 0x1d
+#define DB_WATERMARKS__AUTO_FLUSH_HTILE_MASK 0x40000000
+#define DB_WATERMARKS__AUTO_FLUSH_HTILE__SHIFT 0x1e
+#define DB_WATERMARKS__AUTO_FLUSH_QUAD_MASK 0x80000000
+#define DB_WATERMARKS__AUTO_FLUSH_QUAD__SHIFT 0x1f
+#define DB_SUBTILE_CONTROL__MSAA1_X_MASK 0x3
+#define DB_SUBTILE_CONTROL__MSAA1_X__SHIFT 0x0
+#define DB_SUBTILE_CONTROL__MSAA1_Y_MASK 0xc
+#define DB_SUBTILE_CONTROL__MSAA1_Y__SHIFT 0x2
+#define DB_SUBTILE_CONTROL__MSAA2_X_MASK 0x30
+#define DB_SUBTILE_CONTROL__MSAA2_X__SHIFT 0x4
+#define DB_SUBTILE_CONTROL__MSAA2_Y_MASK 0xc0
+#define DB_SUBTILE_CONTROL__MSAA2_Y__SHIFT 0x6
+#define DB_SUBTILE_CONTROL__MSAA4_X_MASK 0x300
+#define DB_SUBTILE_CONTROL__MSAA4_X__SHIFT 0x8
+#define DB_SUBTILE_CONTROL__MSAA4_Y_MASK 0xc00
+#define DB_SUBTILE_CONTROL__MSAA4_Y__SHIFT 0xa
+#define DB_SUBTILE_CONTROL__MSAA8_X_MASK 0x3000
+#define DB_SUBTILE_CONTROL__MSAA8_X__SHIFT 0xc
+#define DB_SUBTILE_CONTROL__MSAA8_Y_MASK 0xc000
+#define DB_SUBTILE_CONTROL__MSAA8_Y__SHIFT 0xe
+#define DB_SUBTILE_CONTROL__MSAA16_X_MASK 0x30000
+#define DB_SUBTILE_CONTROL__MSAA16_X__SHIFT 0x10
+#define DB_SUBTILE_CONTROL__MSAA16_Y_MASK 0xc0000
+#define DB_SUBTILE_CONTROL__MSAA16_Y__SHIFT 0x12
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH_MASK 0x7f
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH__SHIFT 0x0
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH_MASK 0x3f80
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH__SHIFT 0x7
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH_MASK 0x1fc000
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH__SHIFT 0xe
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH_MASK 0x1e00000
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH__SHIFT 0x15
+#define DB_FREE_CACHELINES__QUAD_READ_REQS_MASK 0xfe000000
+#define DB_FREE_CACHELINES__QUAD_READ_REQS__SHIFT 0x19
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH_MASK 0x1f
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH__SHIFT 0x0
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH_MASK 0x3e0
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH__SHIFT 0x5
+#define DB_FIFO_DEPTH1__MCC_DEPTH_MASK 0xfc00
+#define DB_FIFO_DEPTH1__MCC_DEPTH__SHIFT 0xa
+#define DB_FIFO_DEPTH1__QC_DEPTH_MASK 0x1f0000
+#define DB_FIFO_DEPTH1__QC_DEPTH__SHIFT 0x10
+#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH_MASK 0x1fe00000
+#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH__SHIFT 0x15
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH_MASK 0xff
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH__SHIFT 0x0
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH_MASK 0x7f00
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH__SHIFT 0x8
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH_MASK 0x1ff8000
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH__SHIFT 0xf
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH_MASK 0xfe000000
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH__SHIFT 0x19
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY_MASK 0xf
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY__SHIFT 0x0
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS_MASK 0xff0
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS__SHIFT 0x4
+#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK 0xfff000
+#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT 0xc
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK 0x1000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT 0x18
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK 0x2000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT 0x19
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK 0x4000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT 0x1a
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK 0x8000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT 0x1b
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK 0x10000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT 0x1c
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK 0x20000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT 0x1d
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK 0x40000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT 0x1e
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK 0x80000000
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT 0x1f
+#define DB_ZPASS_COUNT_LOW__COUNT_LOW_MASK 0xffffffff
+#define DB_ZPASS_COUNT_LOW__COUNT_LOW__SHIFT 0x0
+#define DB_ZPASS_COUNT_HI__COUNT_HI_MASK 0x7fffffff
+#define DB_ZPASS_COUNT_HI__COUNT_HI__SHIFT 0x0
+#define DB_RING_CONTROL__COUNTER_CONTROL_MASK 0x3
+#define DB_RING_CONTROL__COUNTER_CONTROL__SHIFT 0x0
+#define DB_READ_DEBUG_0__BUSY_DATA0_MASK 0xffffffff
+#define DB_READ_DEBUG_0__BUSY_DATA0__SHIFT 0x0
+#define DB_READ_DEBUG_1__BUSY_DATA1_MASK 0xffffffff
+#define DB_READ_DEBUG_1__BUSY_DATA1__SHIFT 0x0
+#define DB_READ_DEBUG_2__BUSY_DATA2_MASK 0xffffffff
+#define DB_READ_DEBUG_2__BUSY_DATA2__SHIFT 0x0
+#define DB_READ_DEBUG_3__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_3__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_4__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_4__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_5__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_5__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_6__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_6__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_7__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_7__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_8__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_8__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_9__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_9__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_A__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_A__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_B__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_B__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_C__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_C__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_D__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_D__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_E__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_E__DEBUG_DATA__SHIFT 0x0
+#define DB_READ_DEBUG_F__DEBUG_DATA_MASK 0xffffffff
+#define DB_READ_DEBUG_F__DEBUG_DATA__SHIFT 0x0
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW_MASK 0xffffffff
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW__SHIFT 0x0
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI_MASK 0x7fffffff
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI__SHIFT 0x0
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW_MASK 0xffffffff
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW__SHIFT 0x0
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI_MASK 0x7fffffff
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI__SHIFT 0x0
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW_MASK 0xffffffff
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW__SHIFT 0x0
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI_MASK 0x7fffffff
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI__SHIFT 0x0
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW_MASK 0xffffffff
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW__SHIFT 0x0
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI_MASK 0x7fffffff
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI__SHIFT 0x0
+#define CC_RB_REDUNDANCY__FAILED_RB0_MASK 0xf00
+#define CC_RB_REDUNDANCY__FAILED_RB0__SHIFT 0x8
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0_MASK 0x1000
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT 0xc
+#define CC_RB_REDUNDANCY__FAILED_RB1_MASK 0xf0000
+#define CC_RB_REDUNDANCY__FAILED_RB1__SHIFT 0x10
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1_MASK 0x100000
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT 0x14
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0xff0000
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x10
+#define GC_USER_RB_REDUNDANCY__FAILED_RB0_MASK 0xf00
+#define GC_USER_RB_REDUNDANCY__FAILED_RB0__SHIFT 0x8
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0_MASK 0x1000
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT 0xc
+#define GC_USER_RB_REDUNDANCY__FAILED_RB1_MASK 0xf0000
+#define GC_USER_RB_REDUNDANCY__FAILED_RB1__SHIFT 0x10
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1_MASK 0x100000
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT 0x14
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0xff0000
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x10
+#define GB_ADDR_CONFIG__NUM_PIPES_MASK 0x7
+#define GB_ADDR_CONFIG__NUM_PIPES__SHIFT 0x0
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x70
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x4
+#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x700
+#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x8
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x3000
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0xc
+#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x70000
+#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x10
+#define GB_ADDR_CONFIG__NUM_GPUS_MASK 0x700000
+#define GB_ADDR_CONFIG__NUM_GPUS__SHIFT 0x14
+#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x3000000
+#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x18
+#define GB_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000
+#define GB_ADDR_CONFIG__ROW_SIZE__SHIFT 0x1c
+#define GB_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000
+#define GB_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x1e
+#define GB_BACKEND_MAP__BACKEND_MAP_MASK 0xffffffff
+#define GB_BACKEND_MAP__BACKEND_MAP__SHIFT 0x0
+#define GB_GPU_ID__GPU_ID_MASK 0xf
+#define GB_GPU_ID__GPU_ID__SHIFT 0x0
+#define CC_RB_DAISY_CHAIN__RB_0_MASK 0xf
+#define CC_RB_DAISY_CHAIN__RB_0__SHIFT 0x0
+#define CC_RB_DAISY_CHAIN__RB_1_MASK 0xf0
+#define CC_RB_DAISY_CHAIN__RB_1__SHIFT 0x4
+#define CC_RB_DAISY_CHAIN__RB_2_MASK 0xf00
+#define CC_RB_DAISY_CHAIN__RB_2__SHIFT 0x8
+#define CC_RB_DAISY_CHAIN__RB_3_MASK 0xf000
+#define CC_RB_DAISY_CHAIN__RB_3__SHIFT 0xc
+#define CC_RB_DAISY_CHAIN__RB_4_MASK 0xf0000
+#define CC_RB_DAISY_CHAIN__RB_4__SHIFT 0x10
+#define CC_RB_DAISY_CHAIN__RB_5_MASK 0xf00000
+#define CC_RB_DAISY_CHAIN__RB_5__SHIFT 0x14
+#define CC_RB_DAISY_CHAIN__RB_6_MASK 0xf000000
+#define CC_RB_DAISY_CHAIN__RB_6__SHIFT 0x18
+#define CC_RB_DAISY_CHAIN__RB_7_MASK 0xf0000000
+#define CC_RB_DAISY_CHAIN__RB_7__SHIFT 0x1c
+#define GB_TILE_MODE0__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE0__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE0__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE0__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE0__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE0__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE0__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE1__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE1__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE1__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE1__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE1__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE1__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE1__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE1__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE2__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE2__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE2__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE2__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE2__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE2__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE2__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE2__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE3__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE3__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE3__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE3__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE3__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE3__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE3__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE3__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE4__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE4__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE4__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE4__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE4__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE4__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE4__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE4__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE5__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE5__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE5__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE5__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE5__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE5__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE5__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE5__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE6__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE6__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE6__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE6__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE6__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE6__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE6__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE6__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE7__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE7__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE7__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE7__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE7__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE7__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE7__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE7__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE8__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE8__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE8__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE8__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE8__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE8__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE8__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE8__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE9__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE9__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE9__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE9__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE9__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE9__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE9__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE9__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE10__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE10__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE10__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE10__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE10__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE10__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE10__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE10__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE11__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE11__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE11__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE11__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE11__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE11__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE11__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE11__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE12__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE12__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE12__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE12__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE12__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE12__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE12__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE12__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE13__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE13__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE13__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE13__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE13__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE13__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE13__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE13__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE14__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE14__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE14__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE14__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE14__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE14__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE14__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE14__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE15__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE15__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE15__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE15__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE15__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE15__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE15__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE15__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE16__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE16__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE16__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE16__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE16__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE16__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE16__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE16__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE17__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE17__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE17__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE17__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE17__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE17__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE17__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE17__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE18__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE18__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE18__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE18__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE18__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE18__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE18__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE18__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE19__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE19__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE19__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE19__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE19__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE19__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE19__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE19__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE20__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE20__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE20__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE20__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE20__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE20__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE20__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE20__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE21__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE21__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE21__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE21__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE21__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE21__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE21__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE21__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE22__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE22__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE22__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE22__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE22__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE22__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE22__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE22__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE23__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE23__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE23__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE23__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE23__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE23__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE23__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE23__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE24__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE24__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE24__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE24__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE24__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE24__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE24__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE24__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE25__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE25__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE25__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE25__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE25__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE25__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE25__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE25__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE26__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE26__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE26__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE26__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE26__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE26__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE26__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE26__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE27__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE27__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE27__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE27__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE27__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE27__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE27__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE27__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE28__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE28__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE28__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE28__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE28__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE28__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE28__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE28__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE29__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE29__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE29__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE29__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE29__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE29__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE29__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE29__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE30__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE30__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE30__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE30__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE30__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE30__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE30__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE30__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_TILE_MODE31__ARRAY_MODE_MASK 0x3c
+#define GB_TILE_MODE31__ARRAY_MODE__SHIFT 0x2
+#define GB_TILE_MODE31__PIPE_CONFIG_MASK 0x7c0
+#define GB_TILE_MODE31__PIPE_CONFIG__SHIFT 0x6
+#define GB_TILE_MODE31__TILE_SPLIT_MASK 0x3800
+#define GB_TILE_MODE31__TILE_SPLIT__SHIFT 0xb
+#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW_MASK 0x1c00000
+#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW__SHIFT 0x16
+#define GB_TILE_MODE31__SAMPLE_SPLIT_MASK 0x6000000
+#define GB_TILE_MODE31__SAMPLE_SPLIT__SHIFT 0x19
+#define GB_MACROTILE_MODE0__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE0__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE0__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE0__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE1__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE1__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE1__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE1__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE1__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE1__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE2__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE2__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE2__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE2__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE2__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE2__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE3__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE3__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE3__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE3__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE3__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE3__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE4__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE4__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE4__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE4__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE4__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE4__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE5__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE5__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE5__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE5__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE5__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE5__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE6__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE6__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE6__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE6__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE6__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE6__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE7__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE7__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE7__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE7__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE7__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE7__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE8__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE8__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE8__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE8__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE8__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE8__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE9__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE9__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE9__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE9__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE9__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE9__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE10__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE10__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE10__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE10__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE10__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE10__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE11__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE11__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE11__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE11__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE11__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE11__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE12__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE12__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE12__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE12__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE12__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE12__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE13__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE13__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE13__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE13__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE13__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE13__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE14__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE14__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE14__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE14__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE14__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE14__NUM_BANKS__SHIFT 0x6
+#define GB_MACROTILE_MODE15__BANK_WIDTH_MASK 0x3
+#define GB_MACROTILE_MODE15__BANK_WIDTH__SHIFT 0x0
+#define GB_MACROTILE_MODE15__BANK_HEIGHT_MASK 0xc
+#define GB_MACROTILE_MODE15__BANK_HEIGHT__SHIFT 0x2
+#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT_MASK 0x30
+#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT__SHIFT 0x4
+#define GB_MACROTILE_MODE15__NUM_BANKS_MASK 0xc0
+#define GB_MACROTILE_MODE15__NUM_BANKS__SHIFT 0x6
+#define GB_EDC_MODE__FORCE_SEC_ON_DED_MASK 0x10000
+#define GB_EDC_MODE__FORCE_SEC_ON_DED__SHIFT 0x10
+#define GB_EDC_MODE__DED_MODE_MASK 0x300000
+#define GB_EDC_MODE__DED_MODE__SHIFT 0x14
+#define GB_EDC_MODE__PROP_FED_MASK 0x20000000
+#define GB_EDC_MODE__PROP_FED__SHIFT 0x1d
+#define GB_EDC_MODE__BYPASS_MASK 0x80000000
+#define GB_EDC_MODE__BYPASS__SHIFT 0x1f
+#define CC_GC_EDC_CONFIG__DIS_EDC_MASK 0x2
+#define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT 0x1
+#define RAS_SIGNATURE_CONTROL__ENABLE_MASK 0x1
+#define RAS_SIGNATURE_CONTROL__ENABLE__SHIFT 0x0
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK_MASK 0xffffffff
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK__SHIFT 0x0
+#define RAS_SX_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_SX_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_SX_SIGNATURE1__SIGNATURE_MASK 0xffffffff
+#define RAS_SX_SIGNATURE1__SIGNATURE__SHIFT 0x0
+#define RAS_SX_SIGNATURE2__SIGNATURE_MASK 0xffffffff
+#define RAS_SX_SIGNATURE2__SIGNATURE__SHIFT 0x0
+#define RAS_SX_SIGNATURE3__SIGNATURE_MASK 0xffffffff
+#define RAS_SX_SIGNATURE3__SIGNATURE__SHIFT 0x0
+#define RAS_DB_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_DB_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_PA_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_PA_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_VGT_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_VGT_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE1__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE1__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE2__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE2__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE3__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE3__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE4__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE4__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE5__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE5__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE6__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE6__SIGNATURE__SHIFT 0x0
+#define RAS_SC_SIGNATURE7__SIGNATURE_MASK 0xffffffff
+#define RAS_SC_SIGNATURE7__SIGNATURE__SHIFT 0x0
+#define RAS_IA_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_IA_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_IA_SIGNATURE1__SIGNATURE_MASK 0xffffffff
+#define RAS_IA_SIGNATURE1__SIGNATURE__SHIFT 0x0
+#define RAS_SPI_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_SPI_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_SPI_SIGNATURE1__SIGNATURE_MASK 0xffffffff
+#define RAS_SPI_SIGNATURE1__SIGNATURE__SHIFT 0x0
+#define RAS_TA_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_TA_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_TD_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_TD_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_CB_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_CB_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_BCI_SIGNATURE0__SIGNATURE_MASK 0xffffffff
+#define RAS_BCI_SIGNATURE0__SIGNATURE__SHIFT 0x0
+#define RAS_BCI_SIGNATURE1__SIGNATURE_MASK 0xffffffff
+#define RAS_BCI_SIGNATURE1__SIGNATURE__SHIFT 0x0
+#define RAS_TA_SIGNATURE1__SIGNATURE_MASK 0xffffffff
+#define RAS_TA_SIGNATURE1__SIGNATURE__SHIFT 0x0
+#define GRBM_HYP_CAM_INDEX__CAM_INDEX_MASK 0x7
+#define GRBM_HYP_CAM_INDEX__CAM_INDEX__SHIFT 0x0
+#define GRBM_CAM_INDEX__CAM_INDEX_MASK 0x7
+#define GRBM_CAM_INDEX__CAM_INDEX__SHIFT 0x0
+#define GRBM_HYP_CAM_DATA__CAM_ADDR_MASK 0xffff
+#define GRBM_HYP_CAM_DATA__CAM_ADDR__SHIFT 0x0
+#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR_MASK 0xffff0000
+#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR__SHIFT 0x10
+#define GRBM_CAM_DATA__CAM_ADDR_MASK 0xffff
+#define GRBM_CAM_DATA__CAM_ADDR__SHIFT 0x0
+#define GRBM_CAM_DATA__CAM_REMAPADDR_MASK 0xffff0000
+#define GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT 0x10
+#define GRBM_CNTL__READ_TIMEOUT_MASK 0xff
+#define GRBM_CNTL__READ_TIMEOUT__SHIFT 0x0
+#define GRBM_CNTL__REPORT_LAST_RDERR_MASK 0x80000000
+#define GRBM_CNTL__REPORT_LAST_RDERR__SHIFT 0x1f
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD_MASK 0x3f
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD__SHIFT 0x0
+#define GRBM_SKEW_CNTL__SKEW_COUNT_MASK 0xfc0
+#define GRBM_SKEW_CNTL__SKEW_COUNT__SHIFT 0x6
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE_MASK 0x3
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE__SHIFT 0x0
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE_MASK 0xc
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE__SHIFT 0x2
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE_MASK 0x30
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE__SHIFT 0x4
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE_MASK 0xc0
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE__SHIFT 0x6
+#define GRBM_PWR_CNTL__GFX_REQ_EN_MASK 0x4000
+#define GRBM_PWR_CNTL__GFX_REQ_EN__SHIFT 0xe
+#define GRBM_PWR_CNTL__ALL_REQ_EN_MASK 0x8000
+#define GRBM_PWR_CNTL__ALL_REQ_EN__SHIFT 0xf
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL_MASK 0xf
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL__SHIFT 0x0
+#define GRBM_STATUS__SRBM_RQ_PENDING_MASK 0x20
+#define GRBM_STATUS__SRBM_RQ_PENDING__SHIFT 0x5
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING_MASK 0x80
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING__SHIFT 0x7
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING_MASK 0x100
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING__SHIFT 0x8
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING_MASK 0x200
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING__SHIFT 0x9
+#define GRBM_STATUS__DB_CLEAN_MASK 0x1000
+#define GRBM_STATUS__DB_CLEAN__SHIFT 0xc
+#define GRBM_STATUS__CB_CLEAN_MASK 0x2000
+#define GRBM_STATUS__CB_CLEAN__SHIFT 0xd
+#define GRBM_STATUS__TA_BUSY_MASK 0x4000
+#define GRBM_STATUS__TA_BUSY__SHIFT 0xe
+#define GRBM_STATUS__GDS_BUSY_MASK 0x8000
+#define GRBM_STATUS__GDS_BUSY__SHIFT 0xf
+#define GRBM_STATUS__WD_BUSY_NO_DMA_MASK 0x10000
+#define GRBM_STATUS__WD_BUSY_NO_DMA__SHIFT 0x10
+#define GRBM_STATUS__VGT_BUSY_MASK 0x20000
+#define GRBM_STATUS__VGT_BUSY__SHIFT 0x11
+#define GRBM_STATUS__IA_BUSY_NO_DMA_MASK 0x40000
+#define GRBM_STATUS__IA_BUSY_NO_DMA__SHIFT 0x12
+#define GRBM_STATUS__IA_BUSY_MASK 0x80000
+#define GRBM_STATUS__IA_BUSY__SHIFT 0x13
+#define GRBM_STATUS__SX_BUSY_MASK 0x100000
+#define GRBM_STATUS__SX_BUSY__SHIFT 0x14
+#define GRBM_STATUS__WD_BUSY_MASK 0x200000
+#define GRBM_STATUS__WD_BUSY__SHIFT 0x15
+#define GRBM_STATUS__SPI_BUSY_MASK 0x400000
+#define GRBM_STATUS__SPI_BUSY__SHIFT 0x16
+#define GRBM_STATUS__BCI_BUSY_MASK 0x800000
+#define GRBM_STATUS__BCI_BUSY__SHIFT 0x17
+#define GRBM_STATUS__SC_BUSY_MASK 0x1000000
+#define GRBM_STATUS__SC_BUSY__SHIFT 0x18
+#define GRBM_STATUS__PA_BUSY_MASK 0x2000000
+#define GRBM_STATUS__PA_BUSY__SHIFT 0x19
+#define GRBM_STATUS__DB_BUSY_MASK 0x4000000
+#define GRBM_STATUS__DB_BUSY__SHIFT 0x1a
+#define GRBM_STATUS__CP_COHERENCY_BUSY_MASK 0x10000000
+#define GRBM_STATUS__CP_COHERENCY_BUSY__SHIFT 0x1c
+#define GRBM_STATUS__CP_BUSY_MASK 0x20000000
+#define GRBM_STATUS__CP_BUSY__SHIFT 0x1d
+#define GRBM_STATUS__CB_BUSY_MASK 0x40000000
+#define GRBM_STATUS__CB_BUSY__SHIFT 0x1e
+#define GRBM_STATUS__GUI_ACTIVE_MASK 0x80000000
+#define GRBM_STATUS__GUI_ACTIVE__SHIFT 0x1f
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL_MASK 0xf
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL__SHIFT 0x0
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING_MASK 0x10
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING__SHIFT 0x4
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING_MASK 0x20
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING__SHIFT 0x5
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING_MASK 0x40
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING__SHIFT 0x6
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING_MASK 0x80
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING__SHIFT 0x7
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING_MASK 0x100
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING__SHIFT 0x8
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING_MASK 0x200
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING__SHIFT 0x9
+#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING_MASK 0x400
+#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING__SHIFT 0xa
+#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING_MASK 0x800
+#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING__SHIFT 0xb
+#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING_MASK 0x1000
+#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING__SHIFT 0xc
+#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING_MASK 0x2000
+#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING__SHIFT 0xd
+#define GRBM_STATUS2__RLC_RQ_PENDING_MASK 0x4000
+#define GRBM_STATUS2__RLC_RQ_PENDING__SHIFT 0xe
+#define GRBM_STATUS2__RLC_BUSY_MASK 0x1000000
+#define GRBM_STATUS2__RLC_BUSY__SHIFT 0x18
+#define GRBM_STATUS2__TC_BUSY_MASK 0x2000000
+#define GRBM_STATUS2__TC_BUSY__SHIFT 0x19
+#define GRBM_STATUS2__TCC_CC_RESIDENT_MASK 0x4000000
+#define GRBM_STATUS2__TCC_CC_RESIDENT__SHIFT 0x1a
+#define GRBM_STATUS2__CPF_BUSY_MASK 0x10000000
+#define GRBM_STATUS2__CPF_BUSY__SHIFT 0x1c
+#define GRBM_STATUS2__CPC_BUSY_MASK 0x20000000
+#define GRBM_STATUS2__CPC_BUSY__SHIFT 0x1d
+#define GRBM_STATUS2__CPG_BUSY_MASK 0x40000000
+#define GRBM_STATUS2__CPG_BUSY__SHIFT 0x1e
+#define GRBM_STATUS_SE0__DB_CLEAN_MASK 0x2
+#define GRBM_STATUS_SE0__DB_CLEAN__SHIFT 0x1
+#define GRBM_STATUS_SE0__CB_CLEAN_MASK 0x4
+#define GRBM_STATUS_SE0__CB_CLEAN__SHIFT 0x2
+#define GRBM_STATUS_SE0__BCI_BUSY_MASK 0x400000
+#define GRBM_STATUS_SE0__BCI_BUSY__SHIFT 0x16
+#define GRBM_STATUS_SE0__VGT_BUSY_MASK 0x800000
+#define GRBM_STATUS_SE0__VGT_BUSY__SHIFT 0x17
+#define GRBM_STATUS_SE0__PA_BUSY_MASK 0x1000000
+#define GRBM_STATUS_SE0__PA_BUSY__SHIFT 0x18
+#define GRBM_STATUS_SE0__TA_BUSY_MASK 0x2000000
+#define GRBM_STATUS_SE0__TA_BUSY__SHIFT 0x19
+#define GRBM_STATUS_SE0__SX_BUSY_MASK 0x4000000
+#define GRBM_STATUS_SE0__SX_BUSY__SHIFT 0x1a
+#define GRBM_STATUS_SE0__SPI_BUSY_MASK 0x8000000
+#define GRBM_STATUS_SE0__SPI_BUSY__SHIFT 0x1b
+#define GRBM_STATUS_SE0__SC_BUSY_MASK 0x20000000
+#define GRBM_STATUS_SE0__SC_BUSY__SHIFT 0x1d
+#define GRBM_STATUS_SE0__DB_BUSY_MASK 0x40000000
+#define GRBM_STATUS_SE0__DB_BUSY__SHIFT 0x1e
+#define GRBM_STATUS_SE0__CB_BUSY_MASK 0x80000000
+#define GRBM_STATUS_SE0__CB_BUSY__SHIFT 0x1f
+#define GRBM_STATUS_SE1__DB_CLEAN_MASK 0x2
+#define GRBM_STATUS_SE1__DB_CLEAN__SHIFT 0x1
+#define GRBM_STATUS_SE1__CB_CLEAN_MASK 0x4
+#define GRBM_STATUS_SE1__CB_CLEAN__SHIFT 0x2
+#define GRBM_STATUS_SE1__BCI_BUSY_MASK 0x400000
+#define GRBM_STATUS_SE1__BCI_BUSY__SHIFT 0x16
+#define GRBM_STATUS_SE1__VGT_BUSY_MASK 0x800000
+#define GRBM_STATUS_SE1__VGT_BUSY__SHIFT 0x17
+#define GRBM_STATUS_SE1__PA_BUSY_MASK 0x1000000
+#define GRBM_STATUS_SE1__PA_BUSY__SHIFT 0x18
+#define GRBM_STATUS_SE1__TA_BUSY_MASK 0x2000000
+#define GRBM_STATUS_SE1__TA_BUSY__SHIFT 0x19
+#define GRBM_STATUS_SE1__SX_BUSY_MASK 0x4000000
+#define GRBM_STATUS_SE1__SX_BUSY__SHIFT 0x1a
+#define GRBM_STATUS_SE1__SPI_BUSY_MASK 0x8000000
+#define GRBM_STATUS_SE1__SPI_BUSY__SHIFT 0x1b
+#define GRBM_STATUS_SE1__SC_BUSY_MASK 0x20000000
+#define GRBM_STATUS_SE1__SC_BUSY__SHIFT 0x1d
+#define GRBM_STATUS_SE1__DB_BUSY_MASK 0x40000000
+#define GRBM_STATUS_SE1__DB_BUSY__SHIFT 0x1e
+#define GRBM_STATUS_SE1__CB_BUSY_MASK 0x80000000
+#define GRBM_STATUS_SE1__CB_BUSY__SHIFT 0x1f
+#define GRBM_STATUS_SE2__DB_CLEAN_MASK 0x2
+#define GRBM_STATUS_SE2__DB_CLEAN__SHIFT 0x1
+#define GRBM_STATUS_SE2__CB_CLEAN_MASK 0x4
+#define GRBM_STATUS_SE2__CB_CLEAN__SHIFT 0x2
+#define GRBM_STATUS_SE2__BCI_BUSY_MASK 0x400000
+#define GRBM_STATUS_SE2__BCI_BUSY__SHIFT 0x16
+#define GRBM_STATUS_SE2__VGT_BUSY_MASK 0x800000
+#define GRBM_STATUS_SE2__VGT_BUSY__SHIFT 0x17
+#define GRBM_STATUS_SE2__PA_BUSY_MASK 0x1000000
+#define GRBM_STATUS_SE2__PA_BUSY__SHIFT 0x18
+#define GRBM_STATUS_SE2__TA_BUSY_MASK 0x2000000
+#define GRBM_STATUS_SE2__TA_BUSY__SHIFT 0x19
+#define GRBM_STATUS_SE2__SX_BUSY_MASK 0x4000000
+#define GRBM_STATUS_SE2__SX_BUSY__SHIFT 0x1a
+#define GRBM_STATUS_SE2__SPI_BUSY_MASK 0x8000000
+#define GRBM_STATUS_SE2__SPI_BUSY__SHIFT 0x1b
+#define GRBM_STATUS_SE2__SC_BUSY_MASK 0x20000000
+#define GRBM_STATUS_SE2__SC_BUSY__SHIFT 0x1d
+#define GRBM_STATUS_SE2__DB_BUSY_MASK 0x40000000
+#define GRBM_STATUS_SE2__DB_BUSY__SHIFT 0x1e
+#define GRBM_STATUS_SE2__CB_BUSY_MASK 0x80000000
+#define GRBM_STATUS_SE2__CB_BUSY__SHIFT 0x1f
+#define GRBM_STATUS_SE3__DB_CLEAN_MASK 0x2
+#define GRBM_STATUS_SE3__DB_CLEAN__SHIFT 0x1
+#define GRBM_STATUS_SE3__CB_CLEAN_MASK 0x4
+#define GRBM_STATUS_SE3__CB_CLEAN__SHIFT 0x2
+#define GRBM_STATUS_SE3__BCI_BUSY_MASK 0x400000
+#define GRBM_STATUS_SE3__BCI_BUSY__SHIFT 0x16
+#define GRBM_STATUS_SE3__VGT_BUSY_MASK 0x800000
+#define GRBM_STATUS_SE3__VGT_BUSY__SHIFT 0x17
+#define GRBM_STATUS_SE3__PA_BUSY_MASK 0x1000000
+#define GRBM_STATUS_SE3__PA_BUSY__SHIFT 0x18
+#define GRBM_STATUS_SE3__TA_BUSY_MASK 0x2000000
+#define GRBM_STATUS_SE3__TA_BUSY__SHIFT 0x19
+#define GRBM_STATUS_SE3__SX_BUSY_MASK 0x4000000
+#define GRBM_STATUS_SE3__SX_BUSY__SHIFT 0x1a
+#define GRBM_STATUS_SE3__SPI_BUSY_MASK 0x8000000
+#define GRBM_STATUS_SE3__SPI_BUSY__SHIFT 0x1b
+#define GRBM_STATUS_SE3__SC_BUSY_MASK 0x20000000
+#define GRBM_STATUS_SE3__SC_BUSY__SHIFT 0x1d
+#define GRBM_STATUS_SE3__DB_BUSY_MASK 0x40000000
+#define GRBM_STATUS_SE3__DB_BUSY__SHIFT 0x1e
+#define GRBM_STATUS_SE3__CB_BUSY_MASK 0x80000000
+#define GRBM_STATUS_SE3__CB_BUSY__SHIFT 0x1f
+#define GRBM_SOFT_RESET__SOFT_RESET_CP_MASK 0x1
+#define GRBM_SOFT_RESET__SOFT_RESET_CP__SHIFT 0x0
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK 0x4
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC__SHIFT 0x2
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK 0x10000
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX__SHIFT 0x10
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF_MASK 0x20000
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF__SHIFT 0x11
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC_MASK 0x40000
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC__SHIFT 0x12
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG_MASK 0x80000
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG__SHIFT 0x13
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC_MASK 0x100000
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC__SHIFT 0x14
+#define GRBM_DEBUG_CNTL__GRBM_DEBUG_INDEX_MASK 0x3f
+#define GRBM_DEBUG_CNTL__GRBM_DEBUG_INDEX__SHIFT 0x0
+#define GRBM_DEBUG_DATA__DATA_MASK 0xffffffff
+#define GRBM_DEBUG_DATA__DATA__SHIFT 0x0
+#define GRBM_CGTT_CLK_CNTL__ON_DELAY_MASK 0xf
+#define GRBM_CGTT_CLK_CNTL__ON_DELAY__SHIFT 0x0
+#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS_MASK 0xff0
+#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS__SHIFT 0x4
+#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN_MASK 0x40000000
+#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN__SHIFT 0x1e
+#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK 0xff
+#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT 0x0
+#define GRBM_GFX_INDEX__SH_INDEX_MASK 0xff00
+#define GRBM_GFX_INDEX__SH_INDEX__SHIFT 0x8
+#define GRBM_GFX_INDEX__SE_INDEX_MASK 0xff0000
+#define GRBM_GFX_INDEX__SE_INDEX__SHIFT 0x10
+#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK 0x20000000
+#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES__SHIFT 0x1d
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK 0x40000000
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT 0x1e
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK 0x80000000
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT 0x1f
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK 0xf
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT 0x0
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK 0x1f00
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT 0x8
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS_MASK 0xff
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS__SHIFT 0x0
+#define GRBM_DEBUG__IGNORE_RDY_MASK 0x2
+#define GRBM_DEBUG__IGNORE_RDY__SHIFT 0x1
+#define GRBM_DEBUG__IGNORE_FAO_MASK 0x20
+#define GRBM_DEBUG__IGNORE_FAO__SHIFT 0x5
+#define GRBM_DEBUG__DISABLE_READ_TIMEOUT_MASK 0x40
+#define GRBM_DEBUG__DISABLE_READ_TIMEOUT__SHIFT 0x6
+#define GRBM_DEBUG__SNAPSHOT_FREE_CNTRS_MASK 0x80
+#define GRBM_DEBUG__SNAPSHOT_FREE_CNTRS__SHIFT 0x7
+#define GRBM_DEBUG__HYSTERESIS_GUI_ACTIVE_MASK 0xf00
+#define GRBM_DEBUG__HYSTERESIS_GUI_ACTIVE__SHIFT 0x8
+#define GRBM_DEBUG__GFX_CLOCK_DOMAIN_OVERRIDE_MASK 0x1000
+#define GRBM_DEBUG__GFX_CLOCK_DOMAIN_OVERRIDE__SHIFT 0xc
+#define GRBM_DEBUG__GRBM_TRAP_ENABLE_MASK 0x2000
+#define GRBM_DEBUG__GRBM_TRAP_ENABLE__SHIFT 0xd
+#define GRBM_DEBUG__DEBUG_BUS_FGCG_EN_MASK 0x80000000
+#define GRBM_DEBUG__DEBUG_BUS_FGCG_EN__SHIFT 0x1f
+#define GRBM_DEBUG_SNAPSHOT__CPF_RDY_MASK 0x1
+#define GRBM_DEBUG_SNAPSHOT__CPF_RDY__SHIFT 0x0
+#define GRBM_DEBUG_SNAPSHOT__CPG_RDY_MASK 0x2
+#define GRBM_DEBUG_SNAPSHOT__CPG_RDY__SHIFT 0x1
+#define GRBM_DEBUG_SNAPSHOT__SRBM_RDY_MASK 0x4
+#define GRBM_DEBUG_SNAPSHOT__SRBM_RDY__SHIFT 0x2
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE0_RDY_MASK 0x8
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE0_RDY__SHIFT 0x3
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE1_RDY_MASK 0x10
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE1_RDY__SHIFT 0x4
+#define GRBM_DEBUG_SNAPSHOT__GDS_RDY_MASK 0x20
+#define GRBM_DEBUG_SNAPSHOT__GDS_RDY__SHIFT 0x5
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY0_MASK 0x40
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY0__SHIFT 0x6
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY0_MASK 0x80
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY0__SHIFT 0x7
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY0_MASK 0x100
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY0__SHIFT 0x8
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY0_MASK 0x200
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY0__SHIFT 0x9
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY0_MASK 0x400
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY0__SHIFT 0xa
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY0_MASK 0x800
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY0__SHIFT 0xb
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY0_MASK 0x1000
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY0__SHIFT 0xc
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY0_MASK 0x2000
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY0__SHIFT 0xd
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY1_MASK 0x4000
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY1__SHIFT 0xe
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY1_MASK 0x8000
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY1__SHIFT 0xf
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY1_MASK 0x10000
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY1__SHIFT 0x10
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY1_MASK 0x20000
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY1__SHIFT 0x11
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY1_MASK 0x40000
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY1__SHIFT 0x12
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY1_MASK 0x80000
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY1__SHIFT 0x13
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY1_MASK 0x100000
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY1__SHIFT 0x14
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY1_MASK 0x200000
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY1__SHIFT 0x15
+#define GRBM_READ_ERROR__READ_ADDRESS_MASK 0x3fffc
+#define GRBM_READ_ERROR__READ_ADDRESS__SHIFT 0x2
+#define GRBM_READ_ERROR__READ_PIPEID_MASK 0x300000
+#define GRBM_READ_ERROR__READ_PIPEID__SHIFT 0x14
+#define GRBM_READ_ERROR__READ_MEID_MASK 0xc00000
+#define GRBM_READ_ERROR__READ_MEID__SHIFT 0x16
+#define GRBM_READ_ERROR__READ_ERROR_MASK 0x80000000
+#define GRBM_READ_ERROR__READ_ERROR__SHIFT 0x1f
+#define GRBM_READ_ERROR2__READ_REQUESTER_SRBM_MASK 0x20000
+#define GRBM_READ_ERROR2__READ_REQUESTER_SRBM__SHIFT 0x11
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC_MASK 0x40000
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC__SHIFT 0x12
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA_MASK 0x80000
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA__SHIFT 0x13
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF_MASK 0x100000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF__SHIFT 0x14
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF_MASK 0x200000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF__SHIFT 0x15
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF_MASK 0x400000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF__SHIFT 0x16
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF_MASK 0x800000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF__SHIFT 0x17
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0_MASK 0x1000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0__SHIFT 0x18
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1_MASK 0x2000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1__SHIFT 0x19
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2_MASK 0x4000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2__SHIFT 0x1a
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3_MASK 0x8000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3__SHIFT 0x1b
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0_MASK 0x10000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0__SHIFT 0x1c
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1_MASK 0x20000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1__SHIFT 0x1d
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2_MASK 0x40000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2__SHIFT 0x1e
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3_MASK 0x80000000
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3__SHIFT 0x1f
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE_MASK 0x1
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE__SHIFT 0x0
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE_MASK 0x80000
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE__SHIFT 0x13
+#define GRBM_TRAP_OP__RW_MASK 0x1
+#define GRBM_TRAP_OP__RW__SHIFT 0x0
+#define GRBM_TRAP_ADDR__DATA_MASK 0xffff
+#define GRBM_TRAP_ADDR__DATA__SHIFT 0x0
+#define GRBM_TRAP_ADDR_MSK__DATA_MASK 0xffff
+#define GRBM_TRAP_ADDR_MSK__DATA__SHIFT 0x0
+#define GRBM_TRAP_WD__DATA_MASK 0xffffffff
+#define GRBM_TRAP_WD__DATA__SHIFT 0x0
+#define GRBM_TRAP_WD_MSK__DATA_MASK 0xffffffff
+#define GRBM_TRAP_WD_MSK__DATA__SHIFT 0x0
+#define GRBM_DSM_BYPASS__BYPASS_BITS_MASK 0x3
+#define GRBM_DSM_BYPASS__BYPASS_BITS__SHIFT 0x0
+#define GRBM_DSM_BYPASS__BYPASS_EN_MASK 0x4
+#define GRBM_DSM_BYPASS__BYPASS_EN__SHIFT 0x2
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC_MASK 0x1
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC__SHIFT 0x0
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_SRBM_MASK 0x2
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_SRBM__SHIFT 0x1
+#define GRBM_WRITE_ERROR__WRITE_SSRCID_MASK 0x1c
+#define GRBM_WRITE_ERROR__WRITE_SSRCID__SHIFT 0x2
+#define GRBM_WRITE_ERROR__WRITE_VFID_MASK 0x1e0
+#define GRBM_WRITE_ERROR__WRITE_VFID__SHIFT 0x5
+#define GRBM_WRITE_ERROR__WRITE_VF_MASK 0x1000
+#define GRBM_WRITE_ERROR__WRITE_VF__SHIFT 0xc
+#define GRBM_WRITE_ERROR__WRITE_VMID_MASK 0x1e000
+#define GRBM_WRITE_ERROR__WRITE_VMID__SHIFT 0xd
+#define GRBM_WRITE_ERROR__WRITE_PIPEID_MASK 0x300000
+#define GRBM_WRITE_ERROR__WRITE_PIPEID__SHIFT 0x14
+#define GRBM_WRITE_ERROR__WRITE_MEID_MASK 0xc00000
+#define GRBM_WRITE_ERROR__WRITE_MEID__SHIFT 0x16
+#define GRBM_WRITE_ERROR__WRITE_ERROR_MASK 0x80000000
+#define GRBM_WRITE_ERROR__WRITE_ERROR__SHIFT 0x1f
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb
+#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x1000
+#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0xc
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x2000
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xd
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x4000
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xe
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x10000
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x10
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x20000
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x11
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x40000
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x12
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK 0x80000
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT 0x13
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x100000
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x14
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x200000
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x15
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK 0x400000
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT 0x16
+#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK 0x800000
+#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT 0x17
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK 0x1000000
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT 0x18
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x2000000
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x19
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK 0x4000000
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT 0x1a
+#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK 0x8000000
+#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT 0x1b
+#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK 0x10000000
+#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT 0x1c
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb
+#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x1000
+#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0xc
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x2000
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xd
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x4000
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xe
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x10000
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x10
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x20000
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x11
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x40000
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x12
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK 0x80000
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT 0x13
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x100000
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x14
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x200000
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x15
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK 0x400000
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT 0x16
+#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK 0x800000
+#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT 0x17
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK 0x1000000
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT 0x18
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x2000000
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x19
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK 0x4000000
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT 0x1a
+#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK 0x8000000
+#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT 0x1b
+#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK 0x10000000
+#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT 0x1c
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12
+#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000
+#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12
+#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000
+#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15
+#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f
+#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb
+#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000
+#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc
+#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000
+#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd
+#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000
+#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf
+#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000
+#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12
+#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000
+#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13
+#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000
+#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14
+#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000
+#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15
+#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f
+#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb
+#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000
+#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc
+#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000
+#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd
+#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000
+#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf
+#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000
+#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12
+#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000
+#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13
+#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000
+#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14
+#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000
+#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0_MASK 0xffffffff
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0__SHIFT 0x0
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1_MASK 0xffffffff
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1__SHIFT 0x0
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2_MASK 0xffffffff
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2__SHIFT 0x0
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3_MASK 0xffffffff
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3__SHIFT 0x0
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4_MASK 0xffffffff
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4__SHIFT 0x0
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5_MASK 0xffffffff
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5__SHIFT 0x0
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6_MASK 0xffffffff
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6__SHIFT 0x0
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7_MASK 0xffffffff
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7__SHIFT 0x0
+#define DEBUG_INDEX__DEBUG_INDEX_MASK 0x3ffff
+#define DEBUG_INDEX__DEBUG_INDEX__SHIFT 0x0
+#define DEBUG_DATA__DEBUG_DATA_MASK 0xffffffff
+#define DEBUG_DATA__DEBUG_DATA__SHIFT 0x0
+#define GRBM_NOWHERE__DATA_MASK 0xffffffff
+#define GRBM_NOWHERE__DATA__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE_MASK 0xffffffff
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET_MASK 0xffffffff
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET__SHIFT 0x0
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA_MASK 0x1
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA__SHIFT 0x0
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA_MASK 0x2
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA__SHIFT 0x1
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA_MASK 0x4
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA__SHIFT 0x2
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA_MASK 0x8
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA__SHIFT 0x3
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA_MASK 0x10
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA__SHIFT 0x4
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA_MASK 0x20
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA__SHIFT 0x5
+#define PA_CL_VTE_CNTL__VTX_XY_FMT_MASK 0x100
+#define PA_CL_VTE_CNTL__VTX_XY_FMT__SHIFT 0x8
+#define PA_CL_VTE_CNTL__VTX_Z_FMT_MASK 0x200
+#define PA_CL_VTE_CNTL__VTX_Z_FMT__SHIFT 0x9
+#define PA_CL_VTE_CNTL__VTX_W0_FMT_MASK 0x400
+#define PA_CL_VTE_CNTL__VTX_W0_FMT__SHIFT 0xa
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF_MASK 0x800
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF__SHIFT 0xb
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK 0x1
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0__SHIFT 0x0
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1_MASK 0x2
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1__SHIFT 0x1
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2_MASK 0x4
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2__SHIFT 0x2
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3_MASK 0x8
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3__SHIFT 0x3
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4_MASK 0x10
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4__SHIFT 0x4
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5_MASK 0x20
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5__SHIFT 0x5
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6_MASK 0x40
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6__SHIFT 0x6
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7_MASK 0x80
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7__SHIFT 0x7
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0_MASK 0x100
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0__SHIFT 0x8
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1_MASK 0x200
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1__SHIFT 0x9
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2_MASK 0x400
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2__SHIFT 0xa
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3_MASK 0x800
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3__SHIFT 0xb
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4_MASK 0x1000
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4__SHIFT 0xc
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5_MASK 0x2000
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5__SHIFT 0xd
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6_MASK 0x4000
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6__SHIFT 0xe
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7_MASK 0x8000
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7__SHIFT 0xf
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE_MASK 0x10000
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE__SHIFT 0x10
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG_MASK 0x20000
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG__SHIFT 0x11
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX_MASK 0x40000
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX__SHIFT 0x12
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX_MASK 0x80000
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX__SHIFT 0x13
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG_MASK 0x100000
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG__SHIFT 0x14
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA_MASK 0x200000
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA__SHIFT 0x15
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA_MASK 0x400000
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA__SHIFT 0x16
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA_MASK 0x800000
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA__SHIFT 0x17
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK 0x1000000
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT 0x18
+#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG_MASK 0x2000000
+#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG__SHIFT 0x19
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH_MASK 0x4000000
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH__SHIFT 0x1a
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD_MASK 0x1
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD__SHIFT 0x0
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD_MASK 0x2
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD__SHIFT 0x1
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD_MASK 0x4
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD__SHIFT 0x2
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0_MASK 0x8
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0__SHIFT 0x3
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN_MASK 0x10
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN__SHIFT 0x4
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN_MASK 0x20
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN__SHIFT 0x5
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN_MASK 0x40
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN__SHIFT 0x6
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0_MASK 0x80
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0__SHIFT 0x7
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF_MASK 0x100
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF__SHIFT 0x8
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN_MASK 0x200
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN__SHIFT 0x9
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF_MASK 0x400
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF__SHIFT 0xa
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN_MASK 0x800
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN__SHIFT 0xb
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF_MASK 0x1000
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF__SHIFT 0xc
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN_MASK 0x2000
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN__SHIFT 0xd
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD_MASK 0x4000
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD__SHIFT 0xe
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0_MASK 0x100000
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0__SHIFT 0x14
+#define PA_CL_CLIP_CNTL__UCP_ENA_0_MASK 0x1
+#define PA_CL_CLIP_CNTL__UCP_ENA_0__SHIFT 0x0
+#define PA_CL_CLIP_CNTL__UCP_ENA_1_MASK 0x2
+#define PA_CL_CLIP_CNTL__UCP_ENA_1__SHIFT 0x1
+#define PA_CL_CLIP_CNTL__UCP_ENA_2_MASK 0x4
+#define PA_CL_CLIP_CNTL__UCP_ENA_2__SHIFT 0x2
+#define PA_CL_CLIP_CNTL__UCP_ENA_3_MASK 0x8
+#define PA_CL_CLIP_CNTL__UCP_ENA_3__SHIFT 0x3
+#define PA_CL_CLIP_CNTL__UCP_ENA_4_MASK 0x10
+#define PA_CL_CLIP_CNTL__UCP_ENA_4__SHIFT 0x4
+#define PA_CL_CLIP_CNTL__UCP_ENA_5_MASK 0x20
+#define PA_CL_CLIP_CNTL__UCP_ENA_5__SHIFT 0x5
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG_MASK 0x2000
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG__SHIFT 0xd
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE_MASK 0xc000
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE__SHIFT 0xe
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE_MASK 0x10000
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE__SHIFT 0x10
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA_MASK 0x20000
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA__SHIFT 0x11
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA_MASK 0x40000
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA__SHIFT 0x12
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF_MASK 0x80000
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF__SHIFT 0x13
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT_MASK 0x100000
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT__SHIFT 0x14
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR_MASK 0x200000
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR__SHIFT 0x15
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL_MASK 0x400000
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL__SHIFT 0x16
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA_MASK 0x1000000
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA__SHIFT 0x18
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE_MASK 0x2000000
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE__SHIFT 0x19
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE_MASK 0x4000000
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE__SHIFT 0x1a
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE_MASK 0x8000000
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE__SHIFT 0x1b
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_0_X__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_0_X__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_0_Y__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_0_Y__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_0_Z__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_0_Z__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_0_W__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_0_W__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_1_X__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_1_X__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_1_Y__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_1_Y__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_1_Z__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_1_Z__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_1_W__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_1_W__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_2_X__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_2_X__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_2_Y__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_2_Y__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_2_Z__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_2_Z__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_2_W__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_2_W__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_3_X__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_3_X__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_3_Y__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_3_Y__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_3_Z__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_3_Z__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_3_W__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_3_W__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_4_X__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_4_X__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_4_Y__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_4_Y__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_4_Z__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_4_Z__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_4_W__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_4_W__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_5_X__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_5_X__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_5_Y__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_5_Y__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_5_Z__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_5_Z__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_UCP_5_W__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_UCP_5_W__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_POINT_X_RAD__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_POINT_X_RAD__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_POINT_SIZE__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_POINT_SIZE__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER_MASK 0xffffffff
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER__SHIFT 0x0
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK 0x1
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA__SHIFT 0x0
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ_MASK 0x6
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT 0x1
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL_MASK 0x8
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL__SHIFT 0x3
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE_MASK 0x10
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE__SHIFT 0x4
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL_MASK 0x20
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL__SHIFT 0x5
+#define PA_CL_ENHANCE__ECO_SPARE3_MASK 0x10000000
+#define PA_CL_ENHANCE__ECO_SPARE3__SHIFT 0x1c
+#define PA_CL_ENHANCE__ECO_SPARE2_MASK 0x20000000
+#define PA_CL_ENHANCE__ECO_SPARE2__SHIFT 0x1d
+#define PA_CL_ENHANCE__ECO_SPARE1_MASK 0x40000000
+#define PA_CL_ENHANCE__ECO_SPARE1__SHIFT 0x1e
+#define PA_CL_ENHANCE__ECO_SPARE0_MASK 0x80000000
+#define PA_CL_ENHANCE__ECO_SPARE0__SHIFT 0x1f
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE_MASK 0x1
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE__SHIFT 0x0
+#define PA_SU_VTX_CNTL__PIX_CENTER_MASK 0x1
+#define PA_SU_VTX_CNTL__PIX_CENTER__SHIFT 0x0
+#define PA_SU_VTX_CNTL__ROUND_MODE_MASK 0x6
+#define PA_SU_VTX_CNTL__ROUND_MODE__SHIFT 0x1
+#define PA_SU_VTX_CNTL__QUANT_MODE_MASK 0x38
+#define PA_SU_VTX_CNTL__QUANT_MODE__SHIFT 0x3
+#define PA_SU_POINT_SIZE__HEIGHT_MASK 0xffff
+#define PA_SU_POINT_SIZE__HEIGHT__SHIFT 0x0
+#define PA_SU_POINT_SIZE__WIDTH_MASK 0xffff0000
+#define PA_SU_POINT_SIZE__WIDTH__SHIFT 0x10
+#define PA_SU_POINT_MINMAX__MIN_SIZE_MASK 0xffff
+#define PA_SU_POINT_MINMAX__MIN_SIZE__SHIFT 0x0
+#define PA_SU_POINT_MINMAX__MAX_SIZE_MASK 0xffff0000
+#define PA_SU_POINT_MINMAX__MAX_SIZE__SHIFT 0x10
+#define PA_SU_LINE_CNTL__WIDTH_MASK 0xffff
+#define PA_SU_LINE_CNTL__WIDTH__SHIFT 0x0
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET_MASK 0x3
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET__SHIFT 0x0
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH_MASK 0x4
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH__SHIFT 0x2
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM_MASK 0x8
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM__SHIFT 0x3
+#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST_MASK 0x10
+#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST__SHIFT 0x4
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE_MASK 0xffffffff
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE__SHIFT 0x0
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK 0x1
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT 0x0
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK 0x2
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT 0x1
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK 0x4
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT 0x2
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK 0x8
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT 0x3
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA_MASK 0x10
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA__SHIFT 0x4
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA_MASK 0x20
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA__SHIFT 0x5
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA_MASK 0x40
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA__SHIFT 0x6
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA_MASK 0x80
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA__SHIFT 0x7
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT_MASK 0xff00
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT__SHIFT 0x8
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION_MASK 0x40000000
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION__SHIFT 0x1e
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION_MASK 0x80000000
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION__SHIFT 0x1f
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT_MASK 0x1
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT__SHIFT 0x0
+#define PA_SU_SC_MODE_CNTL__CULL_BACK_MASK 0x2
+#define PA_SU_SC_MODE_CNTL__CULL_BACK__SHIFT 0x1
+#define PA_SU_SC_MODE_CNTL__FACE_MASK 0x4
+#define PA_SU_SC_MODE_CNTL__FACE__SHIFT 0x2
+#define PA_SU_SC_MODE_CNTL__POLY_MODE_MASK 0x18
+#define PA_SU_SC_MODE_CNTL__POLY_MODE__SHIFT 0x3
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE_MASK 0xe0
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE__SHIFT 0x5
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE_MASK 0x700
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE__SHIFT 0x8
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE_MASK 0x800
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE__SHIFT 0xb
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE_MASK 0x1000
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE__SHIFT 0xc
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE_MASK 0x2000
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE__SHIFT 0xd
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE_MASK 0x10000
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE__SHIFT 0x10
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST_MASK 0x80000
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST__SHIFT 0x13
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS_MASK 0x100000
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS__SHIFT 0x14
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA_MASK 0x200000
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA__SHIFT 0x15
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS_MASK 0xff
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS__SHIFT 0x0
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT_MASK 0x100
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT__SHIFT 0x8
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP_MASK 0xffffffff
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP__SHIFT 0x0
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE_MASK 0xffffffff
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE__SHIFT 0x0
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET_MASK 0xffffffff
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET__SHIFT 0x0
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE_MASK 0xffffffff
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE__SHIFT 0x0
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET_MASK 0xffffffff
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET__SHIFT 0x0
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X_MASK 0x1ff
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X__SHIFT 0x0
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y_MASK 0x1ff0000
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y__SHIFT 0x10
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE_MASK 0xffffff
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE__SHIFT 0x0
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffff
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffff
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffff
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffff
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES_MASK 0x7
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES__SHIFT 0x0
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN_MASK 0x10
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN__SHIFT 0x4
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST_MASK 0x1e000
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST__SHIFT 0xd
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES_MASK 0x700000
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES__SHIFT 0x14
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE_MASK 0x3000000
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE__SHIFT 0x18
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0_MASK 0xffff
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0__SHIFT 0x0
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0_MASK 0xffff0000
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0__SHIFT 0x10
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1_MASK 0xffff
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1__SHIFT 0x0
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1_MASK 0xffff0000
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1__SHIFT 0x10
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES_MASK 0x3
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y__SHIFT 0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X_MASK 0xf
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X__SHIFT 0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y_MASK 0xf0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y__SHIFT 0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X_MASK 0xf00
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X__SHIFT 0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y_MASK 0xf000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y__SHIFT 0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X_MASK 0xf0000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X__SHIFT 0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y_MASK 0xf00000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y__SHIFT 0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X_MASK 0xf000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X__SHIFT 0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y_MASK 0xf0000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y__SHIFT 0x1c
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0_MASK 0xf
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0__SHIFT 0x0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1_MASK 0xf0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1__SHIFT 0x4
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2_MASK 0xf00
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2__SHIFT 0x8
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3_MASK 0xf000
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3__SHIFT 0xc
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4_MASK 0xf0000
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4__SHIFT 0x10
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5_MASK 0xf00000
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5__SHIFT 0x14
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6_MASK 0xf000000
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6__SHIFT 0x18
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7_MASK 0xf0000000
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7__SHIFT 0x1c
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8_MASK 0xf
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8__SHIFT 0x0
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9_MASK 0xf0
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9__SHIFT 0x4
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10_MASK 0xf00
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10__SHIFT 0x8
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11_MASK 0xf000
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11__SHIFT 0xc
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12_MASK 0xf0000
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12__SHIFT 0x10
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13_MASK 0xf00000
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13__SHIFT 0x14
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14_MASK 0xf000000
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14__SHIFT 0x18
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15_MASK 0xf0000000
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15__SHIFT 0x1c
+#define PA_SC_CLIPRECT_0_TL__TL_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_0_TL__TL_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_0_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_0_TL__TL_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_0_BR__BR_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_0_BR__BR_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_0_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_0_BR__BR_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_1_TL__TL_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_1_TL__TL_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_1_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_1_TL__TL_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_1_BR__BR_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_1_BR__BR_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_1_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_1_BR__BR_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_2_TL__TL_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_2_TL__TL_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_2_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_2_TL__TL_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_2_BR__BR_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_2_BR__BR_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_2_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_2_BR__BR_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_3_TL__TL_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_3_TL__TL_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_3_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_3_TL__TL_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_3_BR__BR_X_MASK 0x7fff
+#define PA_SC_CLIPRECT_3_BR__BR_X__SHIFT 0x0
+#define PA_SC_CLIPRECT_3_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_CLIPRECT_3_BR__BR_Y__SHIFT 0x10
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE_MASK 0xffff
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE__SHIFT 0x0
+#define PA_SC_EDGERULE__ER_TRI_MASK 0xf
+#define PA_SC_EDGERULE__ER_TRI__SHIFT 0x0
+#define PA_SC_EDGERULE__ER_POINT_MASK 0xf0
+#define PA_SC_EDGERULE__ER_POINT__SHIFT 0x4
+#define PA_SC_EDGERULE__ER_RECT_MASK 0xf00
+#define PA_SC_EDGERULE__ER_RECT__SHIFT 0x8
+#define PA_SC_EDGERULE__ER_LINE_LR_MASK 0x3f000
+#define PA_SC_EDGERULE__ER_LINE_LR__SHIFT 0xc
+#define PA_SC_EDGERULE__ER_LINE_RL_MASK 0xfc0000
+#define PA_SC_EDGERULE__ER_LINE_RL__SHIFT 0x12
+#define PA_SC_EDGERULE__ER_LINE_TB_MASK 0xf000000
+#define PA_SC_EDGERULE__ER_LINE_TB__SHIFT 0x18
+#define PA_SC_EDGERULE__ER_LINE_BT_MASK 0xf0000000
+#define PA_SC_EDGERULE__ER_LINE_BT__SHIFT 0x1c
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH_MASK 0x200
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH__SHIFT 0x9
+#define PA_SC_LINE_CNTL__LAST_PIXEL_MASK 0x400
+#define PA_SC_LINE_CNTL__LAST_PIXEL__SHIFT 0xa
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA_MASK 0x800
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA__SHIFT 0xb
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA_MASK 0x1000
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA__SHIFT 0xc
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN_MASK 0xffff
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN__SHIFT 0x0
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT_MASK 0xff0000
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT__SHIFT 0x10
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER_MASK 0x10000000
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER__SHIFT 0x1c
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL_MASK 0x60000000
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL__SHIFT 0x1d
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE_MASK 0x1
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE__SHIFT 0x0
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE_MASK 0x2
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE__SHIFT 0x1
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE_MASK 0x4
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE__SHIFT 0x2
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR_MASK 0x8
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR__SHIFT 0x3
+#define PA_SC_MODE_CNTL_1__WALK_SIZE_MASK 0x1
+#define PA_SC_MODE_CNTL_1__WALK_SIZE__SHIFT 0x0
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT_MASK 0x2
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT__SHIFT 0x1
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST_MASK 0x4
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST__SHIFT 0x2
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE_MASK 0x8
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE__SHIFT 0x3
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE_MASK 0x70
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE__SHIFT 0x4
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE_MASK 0x80
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE__SHIFT 0x7
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE_MASK 0x100
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE__SHIFT 0x8
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE_MASK 0x200
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE__SHIFT 0x9
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR_MASK 0x400
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR__SHIFT 0xa
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT_MASK 0x800
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT__SHIFT 0xb
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET_MASK 0x1000
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET__SHIFT 0xc
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT_MASK 0x2000
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT__SHIFT 0xd
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z_MASK 0x4000
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z__SHIFT 0xe
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK_MASK 0x8000
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK__SHIFT 0xf
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE_MASK 0x10000
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE__SHIFT 0x10
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE_MASK 0x20000
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE__SHIFT 0x11
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE_MASK 0x40000
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE__SHIFT 0x12
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE_MASK 0x80000
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE__SHIFT 0x13
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_MASK 0xf00000
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE__SHIFT 0x14
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE_MASK 0x1000000
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE__SHIFT 0x18
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE_MASK 0x2000000
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE__SHIFT 0x19
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE_MASK 0x4000000
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE__SHIFT 0x1a
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE_MASK 0x8000000
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE__SHIFT 0x1b
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK_MASK 0x70000000
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK__SHIFT 0x1c
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK 0x3
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT 0x0
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK 0xc
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT 0x2
+#define PA_SC_RASTER_CONFIG__RB_XSEL2_MASK 0x30
+#define PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT 0x4
+#define PA_SC_RASTER_CONFIG__RB_XSEL_MASK 0x40
+#define PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT 0x6
+#define PA_SC_RASTER_CONFIG__RB_YSEL_MASK 0x80
+#define PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT 0x7
+#define PA_SC_RASTER_CONFIG__PKR_MAP_MASK 0x300
+#define PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT 0x8
+#define PA_SC_RASTER_CONFIG__PKR_XSEL_MASK 0xc00
+#define PA_SC_RASTER_CONFIG__PKR_XSEL__SHIFT 0xa
+#define PA_SC_RASTER_CONFIG__PKR_YSEL_MASK 0x3000
+#define PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT 0xc
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2_MASK 0xc000
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2__SHIFT 0xe
+#define PA_SC_RASTER_CONFIG__SC_MAP_MASK 0x30000
+#define PA_SC_RASTER_CONFIG__SC_MAP__SHIFT 0x10
+#define PA_SC_RASTER_CONFIG__SC_XSEL_MASK 0xc0000
+#define PA_SC_RASTER_CONFIG__SC_XSEL__SHIFT 0x12
+#define PA_SC_RASTER_CONFIG__SC_YSEL_MASK 0x300000
+#define PA_SC_RASTER_CONFIG__SC_YSEL__SHIFT 0x14
+#define PA_SC_RASTER_CONFIG__SE_MAP_MASK 0x3000000
+#define PA_SC_RASTER_CONFIG__SE_MAP__SHIFT 0x18
+#define PA_SC_RASTER_CONFIG__SE_XSEL_MASK 0xc000000
+#define PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT 0x1a
+#define PA_SC_RASTER_CONFIG__SE_YSEL_MASK 0x30000000
+#define PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT 0x1c
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP_MASK 0x3
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP__SHIFT 0x0
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL_MASK 0xc
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL__SHIFT 0x2
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL_MASK 0x30
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL__SHIFT 0x4
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE_MASK 0x3
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE__SHIFT 0x0
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE_MASK 0xc
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE__SHIFT 0x2
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X_MASK 0x7fff
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X__SHIFT 0x0
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y__SHIFT 0x10
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X_MASK 0x7fff
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X__SHIFT 0x0
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y__SHIFT 0x10
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X_MASK 0xffff
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X__SHIFT 0x0
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y_MASK 0xffff0000
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y__SHIFT 0x10
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X_MASK 0xffff
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X__SHIFT 0x0
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y_MASK 0xffff0000
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y__SHIFT 0x10
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET_MASK 0xffff
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET__SHIFT 0x0
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET_MASK 0xffff0000
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET__SHIFT 0x10
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X_MASK 0x7fff
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X__SHIFT 0x0
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y__SHIFT 0x10
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X_MASK 0x7fff
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X__SHIFT 0x0
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X_MASK 0x7fff
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X__SHIFT 0x0
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y_MASK 0x7fff0000
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y__SHIFT 0x10
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN_MASK 0xffffffff
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX_MASK 0xffffffff
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX__SHIFT 0x0
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK 0x1
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER__SHIFT 0x0
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX_MASK 0x2
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX__SHIFT 0x1
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX_MASK 0x4
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX__SHIFT 0x2
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS_MASK 0x8
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS__SHIFT 0x3
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID_MASK 0x10
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID__SHIFT 0x4
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX_MASK 0x20
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX__SHIFT 0x5
+#define PA_SC_ENHANCE__DISABLE_PW_BUBBLE_COLLAPSE_MASK 0xc0
+#define PA_SC_ENHANCE__DISABLE_PW_BUBBLE_COLLAPSE__SHIFT 0x6
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER_MASK 0x100
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER__SHIFT 0x8
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION_MASK 0x200
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION__SHIFT 0x9
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM_MASK 0x400
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM__SHIFT 0xa
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE_MASK 0x800
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE__SHIFT 0xb
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE_MASK 0x1000
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE__SHIFT 0xc
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE_MASK 0x2000
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE__SHIFT 0xd
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS_MASK 0x4000
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS__SHIFT 0xe
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE_MASK 0x8000
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE__SHIFT 0xf
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE_MASK 0x10000
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE__SHIFT 0x10
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE_MASK 0x20000
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE__SHIFT 0x11
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST_MASK 0x40000
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST__SHIFT 0x12
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING_MASK 0x80000
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING__SHIFT 0x13
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY_MASK 0x100000
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY__SHIFT 0x14
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING_MASK 0x200000
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING__SHIFT 0x15
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING_MASK 0x400000
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING__SHIFT 0x16
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS_MASK 0x800000
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS__SHIFT 0x17
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID_MASK 0x1000000
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID__SHIFT 0x18
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO_MASK 0x2000000
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO__SHIFT 0x19
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK 0x4000000
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT 0x1a
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING_MASK 0x8000000
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING__SHIFT 0x1b
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET_MASK 0x10000000
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET__SHIFT 0x1c
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET_MASK 0x20000000
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET__SHIFT 0x1d
+#define PA_SC_ENHANCE__ECO_SPARE1_MASK 0x40000000
+#define PA_SC_ENHANCE__ECO_SPARE1__SHIFT 0x1e
+#define PA_SC_ENHANCE__ECO_SPARE0_MASK 0x80000000
+#define PA_SC_ENHANCE__ECO_SPARE0__SHIFT 0x1f
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE_MASK 0x1
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE__SHIFT 0x0
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_MASK 0x6
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE__SHIFT 0x1
+#define PA_SC_ENHANCE_1__ENABLE_SC_BINNING_MASK 0x8
+#define PA_SC_ENHANCE_1__ENABLE_SC_BINNING__SHIFT 0x3
+#define PA_SC_ENHANCE_1__ECO_SPARE0_MASK 0x10
+#define PA_SC_ENHANCE_1__ECO_SPARE0__SHIFT 0x4
+#define PA_SC_ENHANCE_1__ECO_SPARE1_MASK 0x20
+#define PA_SC_ENHANCE_1__ECO_SPARE1__SHIFT 0x5
+#define PA_SC_ENHANCE_1__ECO_SPARE2_MASK 0x40
+#define PA_SC_ENHANCE_1__ECO_SPARE2__SHIFT 0x6
+#define PA_SC_ENHANCE_1__ECO_SPARE3_MASK 0x80
+#define PA_SC_ENHANCE_1__ECO_SPARE3__SHIFT 0x7
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0_MASK 0x1
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0__SHIFT 0x0
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1_MASK 0x2
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1__SHIFT 0x1
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE_MASK 0x3f
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT 0x0
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE_MASK 0x7fc0
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT 0x6
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE_MASK 0x1f8000
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT 0xf
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE_MASK 0xff800000
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT 0x17
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE_MASK 0x3f
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE__SHIFT 0x0
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE_MASK 0xfc0
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE__SHIFT 0x6
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE_MASK 0x3f000
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE__SHIFT 0xc
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE_MASK 0xfc0000
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE__SHIFT 0x12
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT_MASK 0xffff
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT 0x0
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT_MASK 0xffff0000
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT 0x10
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR_MASK 0xf
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR__SHIFT 0x0
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT_MASK 0xff00
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT__SHIFT 0x8
+#define PA_SC_SCREEN_EXTENT_MIN_0__X_MASK 0xffff
+#define PA_SC_SCREEN_EXTENT_MIN_0__X__SHIFT 0x0
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y_MASK 0xffff0000
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y__SHIFT 0x10
+#define PA_SC_SCREEN_EXTENT_MAX_0__X_MASK 0xffff
+#define PA_SC_SCREEN_EXTENT_MAX_0__X__SHIFT 0x0
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y_MASK 0xffff0000
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y__SHIFT 0x10
+#define PA_SC_SCREEN_EXTENT_MIN_1__X_MASK 0xffff
+#define PA_SC_SCREEN_EXTENT_MIN_1__X__SHIFT 0x0
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y_MASK 0xffff0000
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y__SHIFT 0x10
+#define PA_SC_SCREEN_EXTENT_MAX_1__X_MASK 0xffff
+#define PA_SC_SCREEN_EXTENT_MAX_1__X__SHIFT 0x0
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y_MASK 0xffff0000
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y__SHIFT 0x10
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL_MASK 0x3ff
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT 0x0
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK 0x1
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT 0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK 0x2
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT 0x1
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD_MASK 0x3fff
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD__SHIFT 0x0
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD_MASK 0x3fff
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD__SHIFT 0x0
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK 0xffff
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT 0x0
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT_MASK 0xffff
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT__SHIFT 0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK 0x1
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT 0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK 0x2
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT 0x1
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD_MASK 0x3fff
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD__SHIFT 0x0
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD_MASK 0x3fff
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD__SHIFT 0x0
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK 0xffff
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT 0x0
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT_MASK 0xffff
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT__SHIFT 0x0
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK 0x1
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT 0x0
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK 0x2
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT 0x1
+#define PA_SC_TRAP_SCREEN_H__X_COORD_MASK 0x3fff
+#define PA_SC_TRAP_SCREEN_H__X_COORD__SHIFT 0x0
+#define PA_SC_TRAP_SCREEN_V__Y_COORD_MASK 0x3fff
+#define PA_SC_TRAP_SCREEN_V__Y_COORD__SHIFT 0x0
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT_MASK 0xffff
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT 0x0
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT_MASK 0xffff
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT__SHIFT 0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK 0x1
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT 0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK 0x1
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT 0x0
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK 0x1
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT 0x0
+#define PA_CL_CNTL_STATUS__CL_BUSY_MASK 0x80000000
+#define PA_CL_CNTL_STATUS__CL_BUSY__SHIFT 0x1f
+#define PA_SU_CNTL_STATUS__SU_BUSY_MASK 0x80000000
+#define PA_SU_CNTL_STATUS__SU_BUSY__SHIFT 0x1f
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH_MASK 0x3ff
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH__SHIFT 0x0
+#define CGTT_PA_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_PA_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK 0x20000000
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT 0x1d
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK 0x40000000
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT 0x1e
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE_MASK 0x80000000
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE__SHIFT 0x1f
+#define CGTT_SC_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_SC_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_SC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX_MASK 0x1f
+#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX__SHIFT 0x0
+#define PA_SU_DEBUG_DATA__DATA_MASK 0xffffffff
+#define PA_SU_DEBUG_DATA__DATA__SHIFT 0x0
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX_MASK 0x3f
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX__SHIFT 0x0
+#define PA_SC_DEBUG_DATA__DATA_MASK 0xffffffff
+#define PA_SC_DEBUG_DATA__DATA__SHIFT 0x0
+#define CLIPPER_DEBUG_REG00__ALWAYS_ZERO_MASK 0xff
+#define CLIPPER_DEBUG_REG00__ALWAYS_ZERO__SHIFT 0x0
+#define CLIPPER_DEBUG_REG00__clip_ga_bc_fifo_write_MASK 0x100
+#define CLIPPER_DEBUG_REG00__clip_ga_bc_fifo_write__SHIFT 0x8
+#define CLIPPER_DEBUG_REG00__su_clip_baryc_free_MASK 0x600
+#define CLIPPER_DEBUG_REG00__su_clip_baryc_free__SHIFT 0x9
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_write_MASK 0x800
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_write__SHIFT 0xb
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_full_MASK 0x1000
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_full__SHIFT 0xc
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_empty_MASK 0x2000
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_empty__SHIFT 0xd
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_full_MASK 0x4000
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_full__SHIFT 0xe
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_empty_MASK 0x8000
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_empty__SHIFT 0xf
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_full_MASK 0x10000
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_full__SHIFT 0x10
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_empty_MASK 0x20000
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_empty__SHIFT 0x11
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_full_MASK 0x40000
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_full__SHIFT 0x12
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_empty_MASK 0x80000
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_empty__SHIFT 0x13
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_full_MASK 0x100000
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_full__SHIFT 0x14
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_fifo_empty_MASK 0x200000
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_fifo_empty__SHIFT 0x15
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_full_MASK 0x400000
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_full__SHIFT 0x16
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_empty_MASK 0x800000
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_empty__SHIFT 0x17
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_full_MASK 0x1000000
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_full__SHIFT 0x18
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_empty_MASK 0x2000000
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_empty__SHIFT 0x19
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_full_MASK 0x4000000
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_full__SHIFT 0x1a
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_empty_MASK 0x8000000
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_empty__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_full_MASK 0x10000000
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_full__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_write_MASK 0x20000000
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_write__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_write_MASK 0x40000000
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_write__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_write_MASK 0x80000000
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_write__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG01__ALWAYS_ZERO_MASK 0xff
+#define CLIPPER_DEBUG_REG01__ALWAYS_ZERO__SHIFT 0x0
+#define CLIPPER_DEBUG_REG01__clip_extra_bc_valid_MASK 0x700
+#define CLIPPER_DEBUG_REG01__clip_extra_bc_valid__SHIFT 0x8
+#define CLIPPER_DEBUG_REG01__clip_vert_vte_valid_MASK 0x3800
+#define CLIPPER_DEBUG_REG01__clip_vert_vte_valid__SHIFT 0xb
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_vertex_deallocate_MASK 0x1c000
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_vertex_deallocate__SHIFT 0xe
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_deallocate_slot_MASK 0xe0000
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_deallocate_slot__SHIFT 0x11
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_null_primitive_MASK 0x100000
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_null_primitive__SHIFT 0x14
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_2_MASK 0x200000
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_2__SHIFT 0x15
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_1_MASK 0x400000
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_1__SHIFT 0x16
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_0_MASK 0x800000
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_0__SHIFT 0x17
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_extra_bc_valid_MASK 0x1000000
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_extra_bc_valid__SHIFT 0x18
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vte_naninf_kill_MASK 0x2000000
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vte_naninf_kill__SHIFT 0x19
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vertex_store_indx_MASK 0xc000000
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vertex_store_indx__SHIFT 0x1a
+#define CLIPPER_DEBUG_REG01__clip_ga_bc_fifo_write_MASK 0x10000000
+#define CLIPPER_DEBUG_REG01__clip_ga_bc_fifo_write__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG01__clip_to_ga_fifo_write_MASK 0x20000000
+#define CLIPPER_DEBUG_REG01__clip_to_ga_fifo_write__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_advanceread_MASK 0x40000000
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_advanceread__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_empty_MASK 0x80000000
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_empty__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG02__clip_extra_bc_valid_MASK 0x7
+#define CLIPPER_DEBUG_REG02__clip_extra_bc_valid__SHIFT 0x0
+#define CLIPPER_DEBUG_REG02__clip_vert_vte_valid_MASK 0x38
+#define CLIPPER_DEBUG_REG02__clip_vert_vte_valid__SHIFT 0x3
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clip_seq_indx_MASK 0xc0
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clip_seq_indx__SHIFT 0x6
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_2_MASK 0xf00
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_2__SHIFT 0x8
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_1_MASK 0xf000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_1__SHIFT 0xc
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_0_MASK 0xf0000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_0__SHIFT 0x10
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_extra_bc_coords_MASK 0x100000
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_extra_bc_coords__SHIFT 0x14
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_vte_naninf_kill_MASK 0x200000
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_vte_naninf_kill__SHIFT 0x15
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_end_of_packet_MASK 0x400000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_end_of_packet__SHIFT 0x16
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_first_prim_of_slot_MASK 0x800000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_first_prim_of_slot__SHIFT 0x17
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clipped_prim_MASK 0x1000000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clipped_prim__SHIFT 0x18
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_null_primitive_MASK 0x2000000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_null_primitive__SHIFT 0x19
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_full_MASK 0x4000000
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_full__SHIFT 0x1a
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_full_MASK 0x8000000
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_full__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_write_MASK 0x10000000
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_write__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_write_MASK 0x20000000
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_write__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_advanceread_MASK 0x40000000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_advanceread__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_empty_MASK 0x80000000
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_empty__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_code_or_MASK 0x3fff
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_code_or__SHIFT 0x0
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_id_MASK 0xfc000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_id__SHIFT 0xe
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_state_var_indx_MASK 0x700000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_state_var_indx__SHIFT 0x14
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_primitive_MASK 0x800000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_primitive__SHIFT 0x17
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_deallocate_slot_MASK 0x7000000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_deallocate_slot__SHIFT 0x18
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_first_prim_of_slot_MASK 0x8000000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_end_of_packet_MASK 0x10000000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_end_of_packet__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_param_cache_indx_0_MASK 0x7fe
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_param_cache_indx_0__SHIFT 0x1
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_code_or_MASK 0x3fff
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_code_or__SHIFT 0x0
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_id_MASK 0xfc000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_id__SHIFT 0xe
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_state_var_indx_MASK 0x700000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_state_var_indx__SHIFT 0x14
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_primitive_MASK 0x800000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_primitive__SHIFT 0x17
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_deallocate_slot_MASK 0x7000000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_deallocate_slot__SHIFT 0x18
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_first_prim_of_slot_MASK 0x8000000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_end_of_packet_MASK 0x10000000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_end_of_packet__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_param_cache_indx_0_MASK 0x7fe
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_param_cache_indx_0__SHIFT 0x1
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_code_or_MASK 0x3fff
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_code_or__SHIFT 0x0
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_id_MASK 0xfc000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_id__SHIFT 0xe
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_state_var_indx_MASK 0x700000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_state_var_indx__SHIFT 0x14
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_primitive_MASK 0x800000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_primitive__SHIFT 0x17
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_deallocate_slot_MASK 0x7000000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_deallocate_slot__SHIFT 0x18
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_first_prim_of_slot_MASK 0x8000000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_end_of_packet_MASK 0x10000000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_end_of_packet__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_param_cache_indx_0_MASK 0x7fe
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_param_cache_indx_0__SHIFT 0x1
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_code_or_MASK 0x3fff
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_code_or__SHIFT 0x0
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_id_MASK 0xfc000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_id__SHIFT 0xe
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_state_var_indx_MASK 0x700000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_state_var_indx__SHIFT 0x14
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_primitive_MASK 0x800000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_primitive__SHIFT 0x17
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_deallocate_slot_MASK 0x7000000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_deallocate_slot__SHIFT 0x18
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_first_prim_of_slot_MASK 0x8000000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_end_of_packet_MASK 0x10000000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_end_of_packet__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_param_cache_indx_0_MASK 0x7fe
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_param_cache_indx_0__SHIFT 0x1
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_event_MASK 0x20000000
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_event__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_null_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_null_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_event_MASK 0x1
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_event__SHIFT 0x0
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_event_MASK 0x2
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_event__SHIFT 0x1
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_event_MASK 0x4
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_event__SHIFT 0x2
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_event_MASK 0x8
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_event__SHIFT 0x3
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_primitive_MASK 0x10
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_primitive__SHIFT 0x4
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_primitive_MASK 0x20
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_primitive__SHIFT 0x5
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_primitive_MASK 0x40
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_primitive__SHIFT 0x6
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_primitive_MASK 0x80
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_primitive__SHIFT 0x7
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf00
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x8
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf000
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0xc
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf0000
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x10
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf00000
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x14
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_prim_valid_MASK 0x1000000
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_prim_valid__SHIFT 0x18
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_prim_valid_MASK 0x2000000
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_prim_valid__SHIFT 0x19
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_prim_valid_MASK 0x4000000
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_prim_valid__SHIFT 0x1a
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_prim_valid_MASK 0x8000000
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_prim_valid__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG11__clipsm3_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x10000000
+#define CLIPPER_DEBUG_REG11__clipsm3_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG11__clipsm2_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x20000000
+#define CLIPPER_DEBUG_REG11__clipsm2_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG11__clipsm1_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x40000000
+#define CLIPPER_DEBUG_REG11__clipsm1_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG11__clipsm0_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x80000000
+#define CLIPPER_DEBUG_REG11__clipsm0_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG12__ALWAYS_ZERO_MASK 0xff
+#define CLIPPER_DEBUG_REG12__ALWAYS_ZERO__SHIFT 0x0
+#define CLIPPER_DEBUG_REG12__clip_priority_available_vte_out_clip_MASK 0x1f00
+#define CLIPPER_DEBUG_REG12__clip_priority_available_vte_out_clip__SHIFT 0x8
+#define CLIPPER_DEBUG_REG12__clip_priority_available_clip_verts_MASK 0x3e000
+#define CLIPPER_DEBUG_REG12__clip_priority_available_clip_verts__SHIFT 0xd
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_out_MASK 0xc0000
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_out__SHIFT 0x12
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_vert_MASK 0x300000
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_vert__SHIFT 0x14
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_load_MASK 0xc00000
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_load__SHIFT 0x16
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_clip_primitive_MASK 0x1000000
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_clip_primitive__SHIFT 0x18
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_prim_valid_MASK 0x2000000
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x19
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_clip_primitive_MASK 0x4000000
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_clip_primitive__SHIFT 0x1a
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_prim_valid_MASK 0x8000000
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_clip_primitive_MASK 0x10000000
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_clip_primitive__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_prim_valid_MASK 0x20000000
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_clip_primitive_MASK 0x40000000
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_clip_primitive__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG13__clprim_in_back_state_var_indx_MASK 0x7
+#define CLIPPER_DEBUG_REG13__clprim_in_back_state_var_indx__SHIFT 0x0
+#define CLIPPER_DEBUG_REG13__point_clip_candidate_MASK 0x8
+#define CLIPPER_DEBUG_REG13__point_clip_candidate__SHIFT 0x3
+#define CLIPPER_DEBUG_REG13__prim_nan_kill_MASK 0x10
+#define CLIPPER_DEBUG_REG13__prim_nan_kill__SHIFT 0x4
+#define CLIPPER_DEBUG_REG13__clprim_clip_primitive_MASK 0x20
+#define CLIPPER_DEBUG_REG13__clprim_clip_primitive__SHIFT 0x5
+#define CLIPPER_DEBUG_REG13__clprim_cull_primitive_MASK 0x40
+#define CLIPPER_DEBUG_REG13__clprim_cull_primitive__SHIFT 0x6
+#define CLIPPER_DEBUG_REG13__prim_back_valid_MASK 0x80
+#define CLIPPER_DEBUG_REG13__prim_back_valid__SHIFT 0x7
+#define CLIPPER_DEBUG_REG13__vertval_bits_vertex_cc_next_valid_MASK 0xf00
+#define CLIPPER_DEBUG_REG13__vertval_bits_vertex_cc_next_valid__SHIFT 0x8
+#define CLIPPER_DEBUG_REG13__clipcc_vertex_store_indx_MASK 0x3000
+#define CLIPPER_DEBUG_REG13__clipcc_vertex_store_indx__SHIFT 0xc
+#define CLIPPER_DEBUG_REG13__vte_out_orig_fifo_fifo_empty_MASK 0x4000
+#define CLIPPER_DEBUG_REG13__vte_out_orig_fifo_fifo_empty__SHIFT 0xe
+#define CLIPPER_DEBUG_REG13__clipcode_fifo_fifo_empty_MASK 0x8000
+#define CLIPPER_DEBUG_REG13__clipcode_fifo_fifo_empty__SHIFT 0xf
+#define CLIPPER_DEBUG_REG13__ccgen_to_clipcc_fifo_empty_MASK 0x10000
+#define CLIPPER_DEBUG_REG13__ccgen_to_clipcc_fifo_empty__SHIFT 0x10
+#define CLIPPER_DEBUG_REG13__clip_priority_seq_indx_out_cnt_MASK 0x1e0000
+#define CLIPPER_DEBUG_REG13__clip_priority_seq_indx_out_cnt__SHIFT 0x11
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_orig_vertices_MASK 0x600000
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_orig_vertices__SHIFT 0x15
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_clipsm_wait_MASK 0x800000
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_clipsm_wait__SHIFT 0x17
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_contents_MASK 0x1f000000
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_contents__SHIFT 0x18
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_full_MASK 0x20000000
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_full__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_advanceread_MASK 0x40000000
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_advanceread__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_write_MASK 0x80000000
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_write__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_2_MASK 0x3f
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_2__SHIFT 0x0
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_1_MASK 0xfc0
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_1__SHIFT 0x6
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_0_MASK 0x3f000
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_0__SHIFT 0xc
+#define CLIPPER_DEBUG_REG14__outputclprimtoclip_null_primitive_MASK 0x40000
+#define CLIPPER_DEBUG_REG14__outputclprimtoclip_null_primitive__SHIFT 0x12
+#define CLIPPER_DEBUG_REG14__clprim_in_back_end_of_packet_MASK 0x80000
+#define CLIPPER_DEBUG_REG14__clprim_in_back_end_of_packet__SHIFT 0x13
+#define CLIPPER_DEBUG_REG14__clprim_in_back_first_prim_of_slot_MASK 0x100000
+#define CLIPPER_DEBUG_REG14__clprim_in_back_first_prim_of_slot__SHIFT 0x14
+#define CLIPPER_DEBUG_REG14__clprim_in_back_deallocate_slot_MASK 0xe00000
+#define CLIPPER_DEBUG_REG14__clprim_in_back_deallocate_slot__SHIFT 0x15
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event_id_MASK 0x3f000000
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event_id__SHIFT 0x18
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event_MASK 0x40000000
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG14__prim_back_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG14__prim_back_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG15__vertval_bits_vertex_vertex_store_msb_MASK 0xffff
+#define CLIPPER_DEBUG_REG15__vertval_bits_vertex_vertex_store_msb__SHIFT 0x0
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_2_MASK 0x1f0000
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_2__SHIFT 0x10
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_1_MASK 0x3e00000
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_1__SHIFT 0x15
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_0_MASK 0x7c000000
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_0__SHIFT 0x1a
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG16__sm0_prim_end_state_MASK 0x7f
+#define CLIPPER_DEBUG_REG16__sm0_prim_end_state__SHIFT 0x0
+#define CLIPPER_DEBUG_REG16__sm0_ps_expand_MASK 0x80
+#define CLIPPER_DEBUG_REG16__sm0_ps_expand__SHIFT 0x7
+#define CLIPPER_DEBUG_REG16__sm0_clip_vert_cnt_MASK 0x1f00
+#define CLIPPER_DEBUG_REG16__sm0_clip_vert_cnt__SHIFT 0x8
+#define CLIPPER_DEBUG_REG16__sm0_vertex_clip_cnt_MASK 0x3e000
+#define CLIPPER_DEBUG_REG16__sm0_vertex_clip_cnt__SHIFT 0xd
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_1_MASK 0x40000
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_1__SHIFT 0x12
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_0_MASK 0x80000
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_0__SHIFT 0x13
+#define CLIPPER_DEBUG_REG16__sm0_current_state_MASK 0x7f00000
+#define CLIPPER_DEBUG_REG16__sm0_current_state__SHIFT 0x14
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_outsm_fifo_full_MASK 0x10000000
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_outsm_fifo_full__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG16__sm0_highest_priority_seq_MASK 0x20000000
+#define CLIPPER_DEBUG_REG16__sm0_highest_priority_seq__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG16__sm0_outputcliptoclipga_0_MASK 0x40000000
+#define CLIPPER_DEBUG_REG16__sm0_outputcliptoclipga_0__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG16__sm0_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG16__sm0_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG17__sm1_prim_end_state_MASK 0x7f
+#define CLIPPER_DEBUG_REG17__sm1_prim_end_state__SHIFT 0x0
+#define CLIPPER_DEBUG_REG17__sm1_ps_expand_MASK 0x80
+#define CLIPPER_DEBUG_REG17__sm1_ps_expand__SHIFT 0x7
+#define CLIPPER_DEBUG_REG17__sm1_clip_vert_cnt_MASK 0x1f00
+#define CLIPPER_DEBUG_REG17__sm1_clip_vert_cnt__SHIFT 0x8
+#define CLIPPER_DEBUG_REG17__sm1_vertex_clip_cnt_MASK 0x3e000
+#define CLIPPER_DEBUG_REG17__sm1_vertex_clip_cnt__SHIFT 0xd
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_1_MASK 0x40000
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_1__SHIFT 0x12
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_0_MASK 0x80000
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_0__SHIFT 0x13
+#define CLIPPER_DEBUG_REG17__sm1_current_state_MASK 0x7f00000
+#define CLIPPER_DEBUG_REG17__sm1_current_state__SHIFT 0x14
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_outsm_fifo_full_MASK 0x10000000
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_outsm_fifo_full__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG17__sm1_highest_priority_seq_MASK 0x20000000
+#define CLIPPER_DEBUG_REG17__sm1_highest_priority_seq__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG17__sm1_outputcliptoclipga_0_MASK 0x40000000
+#define CLIPPER_DEBUG_REG17__sm1_outputcliptoclipga_0__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG17__sm1_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG17__sm1_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG18__sm2_prim_end_state_MASK 0x7f
+#define CLIPPER_DEBUG_REG18__sm2_prim_end_state__SHIFT 0x0
+#define CLIPPER_DEBUG_REG18__sm2_ps_expand_MASK 0x80
+#define CLIPPER_DEBUG_REG18__sm2_ps_expand__SHIFT 0x7
+#define CLIPPER_DEBUG_REG18__sm2_clip_vert_cnt_MASK 0x1f00
+#define CLIPPER_DEBUG_REG18__sm2_clip_vert_cnt__SHIFT 0x8
+#define CLIPPER_DEBUG_REG18__sm2_vertex_clip_cnt_MASK 0x3e000
+#define CLIPPER_DEBUG_REG18__sm2_vertex_clip_cnt__SHIFT 0xd
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_1_MASK 0x40000
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_1__SHIFT 0x12
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_0_MASK 0x80000
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_0__SHIFT 0x13
+#define CLIPPER_DEBUG_REG18__sm2_current_state_MASK 0x7f00000
+#define CLIPPER_DEBUG_REG18__sm2_current_state__SHIFT 0x14
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_outsm_fifo_full_MASK 0x10000000
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_outsm_fifo_full__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG18__sm2_highest_priority_seq_MASK 0x20000000
+#define CLIPPER_DEBUG_REG18__sm2_highest_priority_seq__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG18__sm2_outputcliptoclipga_0_MASK 0x40000000
+#define CLIPPER_DEBUG_REG18__sm2_outputcliptoclipga_0__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG18__sm2_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG18__sm2_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define CLIPPER_DEBUG_REG19__sm3_prim_end_state_MASK 0x7f
+#define CLIPPER_DEBUG_REG19__sm3_prim_end_state__SHIFT 0x0
+#define CLIPPER_DEBUG_REG19__sm3_ps_expand_MASK 0x80
+#define CLIPPER_DEBUG_REG19__sm3_ps_expand__SHIFT 0x7
+#define CLIPPER_DEBUG_REG19__sm3_clip_vert_cnt_MASK 0x1f00
+#define CLIPPER_DEBUG_REG19__sm3_clip_vert_cnt__SHIFT 0x8
+#define CLIPPER_DEBUG_REG19__sm3_vertex_clip_cnt_MASK 0x3e000
+#define CLIPPER_DEBUG_REG19__sm3_vertex_clip_cnt__SHIFT 0xd
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_1_MASK 0x40000
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_1__SHIFT 0x12
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_0_MASK 0x80000
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_0__SHIFT 0x13
+#define CLIPPER_DEBUG_REG19__sm3_current_state_MASK 0x7f00000
+#define CLIPPER_DEBUG_REG19__sm3_current_state__SHIFT 0x14
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_outsm_fifo_full_MASK 0x10000000
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_outsm_fifo_full__SHIFT 0x1c
+#define CLIPPER_DEBUG_REG19__sm3_highest_priority_seq_MASK 0x20000000
+#define CLIPPER_DEBUG_REG19__sm3_highest_priority_seq__SHIFT 0x1d
+#define CLIPPER_DEBUG_REG19__sm3_outputcliptoclipga_0_MASK 0x40000000
+#define CLIPPER_DEBUG_REG19__sm3_outputcliptoclipga_0__SHIFT 0x1e
+#define CLIPPER_DEBUG_REG19__sm3_clprim_to_clip_prim_valid_MASK 0x80000000
+#define CLIPPER_DEBUG_REG19__sm3_clprim_to_clip_prim_valid__SHIFT 0x1f
+#define SXIFCCG_DEBUG_REG0__position_address_MASK 0x3f
+#define SXIFCCG_DEBUG_REG0__position_address__SHIFT 0x0
+#define SXIFCCG_DEBUG_REG0__point_address_MASK 0x1c0
+#define SXIFCCG_DEBUG_REG0__point_address__SHIFT 0x6
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_state_var_indx_MASK 0xe00
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_state_var_indx__SHIFT 0x9
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_req_mask_MASK 0xf000
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_req_mask__SHIFT 0xc
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_pci_MASK 0x3ff0000
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_pci__SHIFT 0x10
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_sel_MASK 0xc000000
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_sel__SHIFT 0x1a
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_sp_id_MASK 0x30000000
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_sp_id__SHIFT 0x1c
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_inc_MASK 0x40000000
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_inc__SHIFT 0x1e
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_advance_MASK 0x80000000
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_advance__SHIFT 0x1f
+#define SXIFCCG_DEBUG_REG1__available_positions_MASK 0x7f
+#define SXIFCCG_DEBUG_REG1__available_positions__SHIFT 0x0
+#define SXIFCCG_DEBUG_REG1__sx_receive_indx_MASK 0x380
+#define SXIFCCG_DEBUG_REG1__sx_receive_indx__SHIFT 0x7
+#define SXIFCCG_DEBUG_REG1__sx_pending_fifo_contents_MASK 0x7c00
+#define SXIFCCG_DEBUG_REG1__sx_pending_fifo_contents__SHIFT 0xa
+#define SXIFCCG_DEBUG_REG1__statevar_bits_vs_out_misc_vec_ena_MASK 0x8000
+#define SXIFCCG_DEBUG_REG1__statevar_bits_vs_out_misc_vec_ena__SHIFT 0xf
+#define SXIFCCG_DEBUG_REG1__statevar_bits_disable_sp_MASK 0xf0000
+#define SXIFCCG_DEBUG_REG1__statevar_bits_disable_sp__SHIFT 0x10
+#define SXIFCCG_DEBUG_REG1__aux_sel_MASK 0x300000
+#define SXIFCCG_DEBUG_REG1__aux_sel__SHIFT 0x14
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_1_MASK 0x400000
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_1__SHIFT 0x16
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_0_MASK 0x800000
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_0__SHIFT 0x17
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_1_MASK 0xf000000
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_1__SHIFT 0x18
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_0_MASK 0xf0000000
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_0__SHIFT 0x1c
+#define SXIFCCG_DEBUG_REG2__param_cache_base_MASK 0x7f
+#define SXIFCCG_DEBUG_REG2__param_cache_base__SHIFT 0x0
+#define SXIFCCG_DEBUG_REG2__sx_aux_MASK 0x180
+#define SXIFCCG_DEBUG_REG2__sx_aux__SHIFT 0x7
+#define SXIFCCG_DEBUG_REG2__sx_request_indx_MASK 0x7e00
+#define SXIFCCG_DEBUG_REG2__sx_request_indx__SHIFT 0x9
+#define SXIFCCG_DEBUG_REG2__req_active_verts_loaded_MASK 0x8000
+#define SXIFCCG_DEBUG_REG2__req_active_verts_loaded__SHIFT 0xf
+#define SXIFCCG_DEBUG_REG2__req_active_verts_MASK 0x7f0000
+#define SXIFCCG_DEBUG_REG2__req_active_verts__SHIFT 0x10
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_state_var_indx_MASK 0x3800000
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_state_var_indx__SHIFT 0x17
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_active_verts_MASK 0xfc000000
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_active_verts__SHIFT 0x1a
+#define SXIFCCG_DEBUG_REG3__ALWAYS_ZERO_MASK 0xff
+#define SXIFCCG_DEBUG_REG3__ALWAYS_ZERO__SHIFT 0x0
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_entriesavailable_MASK 0xf00
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_entriesavailable__SHIFT 0x8
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist1_vec_ena_MASK 0x1000
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist1_vec_ena__SHIFT 0xc
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist0_vec_ena_MASK 0x2000
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist0_vec_ena__SHIFT 0xd
+#define SXIFCCG_DEBUG_REG3__available_positions_MASK 0x1fc000
+#define SXIFCCG_DEBUG_REG3__available_positions__SHIFT 0xe
+#define SXIFCCG_DEBUG_REG3__current_state_MASK 0x600000
+#define SXIFCCG_DEBUG_REG3__current_state__SHIFT 0x15
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_empty_MASK 0x800000
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_empty__SHIFT 0x17
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_full_MASK 0x1000000
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_full__SHIFT 0x18
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_empty_MASK 0x2000000
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_empty__SHIFT 0x19
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_full_MASK 0x4000000
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_full__SHIFT 0x1a
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_empty_MASK 0x8000000
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_empty__SHIFT 0x1b
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_full_MASK 0x10000000
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_full__SHIFT 0x1c
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_fifo_full_MASK 0x20000000
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_fifo_full__SHIFT 0x1d
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_write_MASK 0x40000000
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_write__SHIFT 0x1e
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_write_MASK 0x80000000
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_write__SHIFT 0x1f
+#define SETUP_DEBUG_REG0__su_baryc_cntl_state_MASK 0x3
+#define SETUP_DEBUG_REG0__su_baryc_cntl_state__SHIFT 0x0
+#define SETUP_DEBUG_REG0__su_cntl_state_MASK 0x3c
+#define SETUP_DEBUG_REG0__su_cntl_state__SHIFT 0x2
+#define SETUP_DEBUG_REG0__pmode_state_MASK 0x3f00
+#define SETUP_DEBUG_REG0__pmode_state__SHIFT 0x8
+#define SETUP_DEBUG_REG0__ge_stallb_MASK 0x4000
+#define SETUP_DEBUG_REG0__ge_stallb__SHIFT 0xe
+#define SETUP_DEBUG_REG0__geom_enable_MASK 0x8000
+#define SETUP_DEBUG_REG0__geom_enable__SHIFT 0xf
+#define SETUP_DEBUG_REG0__su_clip_baryc_free_MASK 0x30000
+#define SETUP_DEBUG_REG0__su_clip_baryc_free__SHIFT 0x10
+#define SETUP_DEBUG_REG0__su_clip_rtr_MASK 0x40000
+#define SETUP_DEBUG_REG0__su_clip_rtr__SHIFT 0x12
+#define SETUP_DEBUG_REG0__pfifo_busy_MASK 0x80000
+#define SETUP_DEBUG_REG0__pfifo_busy__SHIFT 0x13
+#define SETUP_DEBUG_REG0__su_cntl_busy_MASK 0x100000
+#define SETUP_DEBUG_REG0__su_cntl_busy__SHIFT 0x14
+#define SETUP_DEBUG_REG0__geom_busy_MASK 0x200000
+#define SETUP_DEBUG_REG0__geom_busy__SHIFT 0x15
+#define SETUP_DEBUG_REG0__event_id_gated_MASK 0xfc00000
+#define SETUP_DEBUG_REG0__event_id_gated__SHIFT 0x16
+#define SETUP_DEBUG_REG0__event_gated_MASK 0x10000000
+#define SETUP_DEBUG_REG0__event_gated__SHIFT 0x1c
+#define SETUP_DEBUG_REG0__pmode_prim_gated_MASK 0x20000000
+#define SETUP_DEBUG_REG0__pmode_prim_gated__SHIFT 0x1d
+#define SETUP_DEBUG_REG0__su_dyn_sclk_vld_MASK 0x40000000
+#define SETUP_DEBUG_REG0__su_dyn_sclk_vld__SHIFT 0x1e
+#define SETUP_DEBUG_REG0__cl_dyn_sclk_vld_MASK 0x80000000
+#define SETUP_DEBUG_REG0__cl_dyn_sclk_vld__SHIFT 0x1f
+#define SETUP_DEBUG_REG1__y_sort0_gated_23_8_MASK 0xffff
+#define SETUP_DEBUG_REG1__y_sort0_gated_23_8__SHIFT 0x0
+#define SETUP_DEBUG_REG1__x_sort0_gated_23_8_MASK 0xffff0000
+#define SETUP_DEBUG_REG1__x_sort0_gated_23_8__SHIFT 0x10
+#define SETUP_DEBUG_REG2__y_sort1_gated_23_8_MASK 0xffff
+#define SETUP_DEBUG_REG2__y_sort1_gated_23_8__SHIFT 0x0
+#define SETUP_DEBUG_REG2__x_sort1_gated_23_8_MASK 0xffff0000
+#define SETUP_DEBUG_REG2__x_sort1_gated_23_8__SHIFT 0x10
+#define SETUP_DEBUG_REG3__y_sort2_gated_23_8_MASK 0xffff
+#define SETUP_DEBUG_REG3__y_sort2_gated_23_8__SHIFT 0x0
+#define SETUP_DEBUG_REG3__x_sort2_gated_23_8_MASK 0xffff0000
+#define SETUP_DEBUG_REG3__x_sort2_gated_23_8__SHIFT 0x10
+#define SETUP_DEBUG_REG4__attr_indx_sort0_gated_MASK 0x3fff
+#define SETUP_DEBUG_REG4__attr_indx_sort0_gated__SHIFT 0x0
+#define SETUP_DEBUG_REG4__null_prim_gated_MASK 0x4000
+#define SETUP_DEBUG_REG4__null_prim_gated__SHIFT 0xe
+#define SETUP_DEBUG_REG4__backfacing_gated_MASK 0x8000
+#define SETUP_DEBUG_REG4__backfacing_gated__SHIFT 0xf
+#define SETUP_DEBUG_REG4__st_indx_gated_MASK 0x70000
+#define SETUP_DEBUG_REG4__st_indx_gated__SHIFT 0x10
+#define SETUP_DEBUG_REG4__clipped_gated_MASK 0x80000
+#define SETUP_DEBUG_REG4__clipped_gated__SHIFT 0x13
+#define SETUP_DEBUG_REG4__dealloc_slot_gated_MASK 0x700000
+#define SETUP_DEBUG_REG4__dealloc_slot_gated__SHIFT 0x14
+#define SETUP_DEBUG_REG4__xmajor_gated_MASK 0x800000
+#define SETUP_DEBUG_REG4__xmajor_gated__SHIFT 0x17
+#define SETUP_DEBUG_REG4__diamond_rule_gated_MASK 0x3000000
+#define SETUP_DEBUG_REG4__diamond_rule_gated__SHIFT 0x18
+#define SETUP_DEBUG_REG4__type_gated_MASK 0x1c000000
+#define SETUP_DEBUG_REG4__type_gated__SHIFT 0x1a
+#define SETUP_DEBUG_REG4__fpov_gated_MASK 0x60000000
+#define SETUP_DEBUG_REG4__fpov_gated__SHIFT 0x1d
+#define SETUP_DEBUG_REG4__eop_gated_MASK 0x80000000
+#define SETUP_DEBUG_REG4__eop_gated__SHIFT 0x1f
+#define SETUP_DEBUG_REG5__attr_indx_sort2_gated_MASK 0x3fff
+#define SETUP_DEBUG_REG5__attr_indx_sort2_gated__SHIFT 0x0
+#define SETUP_DEBUG_REG5__attr_indx_sort1_gated_MASK 0xfffc000
+#define SETUP_DEBUG_REG5__attr_indx_sort1_gated__SHIFT 0xe
+#define SETUP_DEBUG_REG5__provoking_vtx_gated_MASK 0x30000000
+#define SETUP_DEBUG_REG5__provoking_vtx_gated__SHIFT 0x1c
+#define SETUP_DEBUG_REG5__valid_prim_gated_MASK 0x40000000
+#define SETUP_DEBUG_REG5__valid_prim_gated__SHIFT 0x1e
+#define SETUP_DEBUG_REG5__pa_reg_sclk_vld_MASK 0x80000000
+#define SETUP_DEBUG_REG5__pa_reg_sclk_vld__SHIFT 0x1f
+#define PA_SC_DEBUG_REG0__REG0_FIELD0_MASK 0x3
+#define PA_SC_DEBUG_REG0__REG0_FIELD0__SHIFT 0x0
+#define PA_SC_DEBUG_REG0__REG0_FIELD1_MASK 0xc
+#define PA_SC_DEBUG_REG0__REG0_FIELD1__SHIFT 0x2
+#define PA_SC_DEBUG_REG1__REG1_FIELD0_MASK 0x3
+#define PA_SC_DEBUG_REG1__REG1_FIELD0__SHIFT 0x0
+#define PA_SC_DEBUG_REG1__REG1_FIELD1_MASK 0xc
+#define PA_SC_DEBUG_REG1__REG1_FIELD1__SHIFT 0x2
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK 0x1
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT 0x0
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK 0x2
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT 0x1
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK 0x4
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT 0x2
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK 0x8
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT 0x3
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK 0x10
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT 0x4
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK 0x20
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT 0x5
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK 0x40
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT 0x6
+#define COMPUTE_DISPATCH_INITIATOR__DISPATCH_CACHE_CNTL_MASK 0x380
+#define COMPUTE_DISPATCH_INITIATOR__DISPATCH_CACHE_CNTL__SHIFT 0x7
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL_MASK 0x400
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL__SHIFT 0xa
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL_MASK 0x800
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL__SHIFT 0xb
+#define COMPUTE_DISPATCH_INITIATOR__DATA_ATC_MASK 0x1000
+#define COMPUTE_DISPATCH_INITIATOR__DATA_ATC__SHIFT 0xc
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK 0x4000
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT 0xe
+#define COMPUTE_DIM_X__SIZE_MASK 0xffffffff
+#define COMPUTE_DIM_X__SIZE__SHIFT 0x0
+#define COMPUTE_DIM_Y__SIZE_MASK 0xffffffff
+#define COMPUTE_DIM_Y__SIZE__SHIFT 0x0
+#define COMPUTE_DIM_Z__SIZE_MASK 0xffffffff
+#define COMPUTE_DIM_Z__SIZE__SHIFT 0x0
+#define COMPUTE_START_X__START_MASK 0xffffffff
+#define COMPUTE_START_X__START__SHIFT 0x0
+#define COMPUTE_START_Y__START_MASK 0xffffffff
+#define COMPUTE_START_Y__START__SHIFT 0x0
+#define COMPUTE_START_Z__START_MASK 0xffffffff
+#define COMPUTE_START_Z__START__SHIFT 0x0
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK 0xffff
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT 0x0
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK 0xffff0000
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT 0x10
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK 0xffff
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT 0x0
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK 0xffff0000
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT 0x10
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK 0xffff
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT 0x0
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK 0xffff0000
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT 0x10
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE_MASK 0x1
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE__SHIFT 0x0
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE_MASK 0x1
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE__SHIFT 0x0
+#define COMPUTE_PGM_LO__DATA_MASK 0xffffffff
+#define COMPUTE_PGM_LO__DATA__SHIFT 0x0
+#define COMPUTE_PGM_HI__DATA_MASK 0xff
+#define COMPUTE_PGM_HI__DATA__SHIFT 0x0
+#define COMPUTE_PGM_HI__INST_ATC_MASK 0x100
+#define COMPUTE_PGM_HI__INST_ATC__SHIFT 0x8
+#define COMPUTE_TBA_LO__DATA_MASK 0xffffffff
+#define COMPUTE_TBA_LO__DATA__SHIFT 0x0
+#define COMPUTE_TBA_HI__DATA_MASK 0xff
+#define COMPUTE_TBA_HI__DATA__SHIFT 0x0
+#define COMPUTE_TMA_LO__DATA_MASK 0xffffffff
+#define COMPUTE_TMA_LO__DATA__SHIFT 0x0
+#define COMPUTE_TMA_HI__DATA_MASK 0xff
+#define COMPUTE_TMA_HI__DATA__SHIFT 0x0
+#define COMPUTE_PGM_RSRC1__VGPRS_MASK 0x3f
+#define COMPUTE_PGM_RSRC1__VGPRS__SHIFT 0x0
+#define COMPUTE_PGM_RSRC1__SGPRS_MASK 0x3c0
+#define COMPUTE_PGM_RSRC1__SGPRS__SHIFT 0x6
+#define COMPUTE_PGM_RSRC1__PRIORITY_MASK 0xc00
+#define COMPUTE_PGM_RSRC1__PRIORITY__SHIFT 0xa
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK 0xff000
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT 0xc
+#define COMPUTE_PGM_RSRC1__PRIV_MASK 0x100000
+#define COMPUTE_PGM_RSRC1__PRIV__SHIFT 0x14
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP_MASK 0x200000
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP__SHIFT 0x15
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE_MASK 0x400000
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE__SHIFT 0x16
+#define COMPUTE_PGM_RSRC1__IEEE_MODE_MASK 0x800000
+#define COMPUTE_PGM_RSRC1__IEEE_MODE__SHIFT 0x17
+#define COMPUTE_PGM_RSRC1__BULKY_MASK 0x1000000
+#define COMPUTE_PGM_RSRC1__BULKY__SHIFT 0x18
+#define COMPUTE_PGM_RSRC1__CDBG_USER_MASK 0x2000000
+#define COMPUTE_PGM_RSRC1__CDBG_USER__SHIFT 0x19
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK 0x1
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT 0x0
+#define COMPUTE_PGM_RSRC2__USER_SGPR_MASK 0x3e
+#define COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT 0x1
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK 0x40
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT 0x6
+#define COMPUTE_PGM_RSRC2__TGID_X_EN_MASK 0x80
+#define COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT 0x7
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK 0x100
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT 0x8
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK 0x200
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT 0x9
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK 0x400
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT 0xa
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK 0x1800
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT 0xb
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK 0x6000
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT 0xd
+#define COMPUTE_PGM_RSRC2__LDS_SIZE_MASK 0xff8000
+#define COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT 0xf
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MASK 0x7f000000
+#define COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT 0x18
+#define COMPUTE_VMID__DATA_MASK 0xf
+#define COMPUTE_VMID__DATA__SHIFT 0x0
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK 0x3ff
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT 0x0
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK 0xf000
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT 0xc
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK 0x3f0000
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT 0x10
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK 0x400000
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT 0x16
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK 0x800000
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT 0x17
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK 0x7000000
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT 0x18
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN_MASK 0xffff
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN__SHIFT 0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN_MASK 0xffff0000
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN__SHIFT 0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN_MASK 0xffff
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN__SHIFT 0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN_MASK 0xffff0000
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN__SHIFT 0x10
+#define COMPUTE_TMPRING_SIZE__WAVES_MASK 0xfff
+#define COMPUTE_TMPRING_SIZE__WAVES__SHIFT 0x0
+#define COMPUTE_TMPRING_SIZE__WAVESIZE_MASK 0x1fff000
+#define COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT 0xc
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN_MASK 0xffff
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN__SHIFT 0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN_MASK 0xffff0000
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN__SHIFT 0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN_MASK 0xffff
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN__SHIFT 0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN_MASK 0xffff0000
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN__SHIFT 0x10
+#define COMPUTE_RESTART_X__RESTART_MASK 0xffffffff
+#define COMPUTE_RESTART_X__RESTART__SHIFT 0x0
+#define COMPUTE_RESTART_Y__RESTART_MASK 0xffffffff
+#define COMPUTE_RESTART_Y__RESTART__SHIFT 0x0
+#define COMPUTE_RESTART_Z__RESTART_MASK 0xffffffff
+#define COMPUTE_RESTART_Z__RESTART__SHIFT 0x0
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE_MASK 0x1
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE__SHIFT 0x0
+#define COMPUTE_MISC_RESERVED__SEND_SEID_MASK 0x3
+#define COMPUTE_MISC_RESERVED__SEND_SEID__SHIFT 0x0
+#define COMPUTE_MISC_RESERVED__RESERVED2_MASK 0x4
+#define COMPUTE_MISC_RESERVED__RESERVED2__SHIFT 0x2
+#define COMPUTE_MISC_RESERVED__RESERVED3_MASK 0x8
+#define COMPUTE_MISC_RESERVED__RESERVED3__SHIFT 0x3
+#define COMPUTE_MISC_RESERVED__RESERVED4_MASK 0x10
+#define COMPUTE_MISC_RESERVED__RESERVED4__SHIFT 0x4
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE_MASK 0x1ffe0
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE__SHIFT 0x5
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID_MASK 0xffffffff
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID__SHIFT 0x0
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID_MASK 0xffffffff
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID__SHIFT 0x0
+#define COMPUTE_RELAUNCH__PAYLOAD_MASK 0x3fffffff
+#define COMPUTE_RELAUNCH__PAYLOAD__SHIFT 0x0
+#define COMPUTE_RELAUNCH__IS_EVENT_MASK 0x40000000
+#define COMPUTE_RELAUNCH__IS_EVENT__SHIFT 0x1e
+#define COMPUTE_RELAUNCH__IS_STATE_MASK 0x80000000
+#define COMPUTE_RELAUNCH__IS_STATE__SHIFT 0x1f
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR_MASK 0xffffffff
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR__SHIFT 0x0
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR_MASK 0xffff
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR__SHIFT 0x0
+#define COMPUTE_WAVE_RESTORE_CONTROL__ATC_MASK 0x1
+#define COMPUTE_WAVE_RESTORE_CONTROL__ATC__SHIFT 0x0
+#define COMPUTE_WAVE_RESTORE_CONTROL__MTYPE_MASK 0x6
+#define COMPUTE_WAVE_RESTORE_CONTROL__MTYPE__SHIFT 0x1
+#define COMPUTE_USER_DATA_0__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_0__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_1__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_1__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_2__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_2__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_3__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_3__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_4__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_4__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_5__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_5__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_6__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_6__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_7__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_7__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_8__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_8__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_9__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_9__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_10__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_10__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_11__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_11__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_12__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_12__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_13__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_13__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_14__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_14__DATA__SHIFT 0x0
+#define COMPUTE_USER_DATA_15__DATA_MASK 0xffffffff
+#define COMPUTE_USER_DATA_15__DATA__SHIFT 0x0
+#define COMPUTE_NOWHERE__DATA_MASK 0xffffffff
+#define COMPUTE_NOWHERE__DATA__SHIFT 0x0
+#define CSPRIV_CONNECT__DOORBELL_OFFSET_MASK 0x1fffff
+#define CSPRIV_CONNECT__DOORBELL_OFFSET__SHIFT 0x0
+#define CSPRIV_CONNECT__QUEUE_ID_MASK 0xe00000
+#define CSPRIV_CONNECT__QUEUE_ID__SHIFT 0x15
+#define CSPRIV_CONNECT__VMID_MASK 0x3c000000
+#define CSPRIV_CONNECT__VMID__SHIFT 0x1a
+#define CSPRIV_CONNECT__UNORD_DISP_MASK 0x80000000
+#define CSPRIV_CONNECT__UNORD_DISP__SHIFT 0x1f
+#define CSPRIV_THREAD_TRACE_TG0__TGID_X_MASK 0xffffffff
+#define CSPRIV_THREAD_TRACE_TG0__TGID_X__SHIFT 0x0
+#define CSPRIV_THREAD_TRACE_TG1__TGID_Y_MASK 0xffffffff
+#define CSPRIV_THREAD_TRACE_TG1__TGID_Y__SHIFT 0x0
+#define CSPRIV_THREAD_TRACE_TG2__TGID_Z_MASK 0xffffffff
+#define CSPRIV_THREAD_TRACE_TG2__TGID_Z__SHIFT 0x0
+#define CSPRIV_THREAD_TRACE_TG3__WAVE_ID_BASE_MASK 0xfff
+#define CSPRIV_THREAD_TRACE_TG3__WAVE_ID_BASE__SHIFT 0x0
+#define CSPRIV_THREAD_TRACE_TG3__THREADS_IN_GROUP_MASK 0xfff000
+#define CSPRIV_THREAD_TRACE_TG3__THREADS_IN_GROUP__SHIFT 0xc
+#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_X_FLAG_MASK 0x1000000
+#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_X_FLAG__SHIFT 0x18
+#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Y_FLAG_MASK 0x2000000
+#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Y_FLAG__SHIFT 0x19
+#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Z_FLAG_MASK 0x4000000
+#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Z_FLAG__SHIFT 0x1a
+#define CSPRIV_THREAD_TRACE_TG3__LAST_TG_MASK 0x8000000
+#define CSPRIV_THREAD_TRACE_TG3__LAST_TG__SHIFT 0x1b
+#define CSPRIV_THREAD_TRACE_TG3__FIRST_TG_MASK 0x10000000
+#define CSPRIV_THREAD_TRACE_TG3__FIRST_TG__SHIFT 0x1c
+#define CSPRIV_THREAD_TRACE_EVENT__EVENT_ID_MASK 0x1f
+#define CSPRIV_THREAD_TRACE_EVENT__EVENT_ID__SHIFT 0x0
+#define RLC_CNTL__RLC_ENABLE_F32_MASK 0x1
+#define RLC_CNTL__RLC_ENABLE_F32__SHIFT 0x0
+#define RLC_CNTL__FORCE_RETRY_MASK 0x2
+#define RLC_CNTL__FORCE_RETRY__SHIFT 0x1
+#define RLC_CNTL__READ_CACHE_DISABLE_MASK 0x4
+#define RLC_CNTL__READ_CACHE_DISABLE__SHIFT 0x2
+#define RLC_CNTL__RLC_STEP_F32_MASK 0x8
+#define RLC_CNTL__RLC_STEP_F32__SHIFT 0x3
+#define RLC_CNTL__SOFT_RESET_DEBUG_MODE_MASK 0x10
+#define RLC_CNTL__SOFT_RESET_DEBUG_MODE__SHIFT 0x4
+#define RLC_CNTL__RESERVED_MASK 0xffffff00
+#define RLC_CNTL__RESERVED__SHIFT 0x8
+#define RLC_DEBUG_SELECT__SELECT_MASK 0xff
+#define RLC_DEBUG_SELECT__SELECT__SHIFT 0x0
+#define RLC_DEBUG_SELECT__RESERVED_MASK 0xffffff00
+#define RLC_DEBUG_SELECT__RESERVED__SHIFT 0x8
+#define RLC_DEBUG__DATA_MASK 0xffffffff
+#define RLC_DEBUG__DATA__SHIFT 0x0
+#define RLC_MC_CNTL__WRREQ_SWAP_MASK 0x3
+#define RLC_MC_CNTL__WRREQ_SWAP__SHIFT 0x0
+#define RLC_MC_CNTL__WRREQ_TRAN_MASK 0x4
+#define RLC_MC_CNTL__WRREQ_TRAN__SHIFT 0x2
+#define RLC_MC_CNTL__WRREQ_PRIV_MASK 0x8
+#define RLC_MC_CNTL__WRREQ_PRIV__SHIFT 0x3
+#define RLC_MC_CNTL__WRNFO_STALL_MASK 0x10
+#define RLC_MC_CNTL__WRNFO_STALL__SHIFT 0x4
+#define RLC_MC_CNTL__WRNFO_URG_MASK 0x1e0
+#define RLC_MC_CNTL__WRNFO_URG__SHIFT 0x5
+#define RLC_MC_CNTL__WRREQ_DW_IMASK_MASK 0x1e00
+#define RLC_MC_CNTL__WRREQ_DW_IMASK__SHIFT 0x9
+#define RLC_MC_CNTL__RESERVED_B_MASK 0xfe000
+#define RLC_MC_CNTL__RESERVED_B__SHIFT 0xd
+#define RLC_MC_CNTL__RDNFO_URG_MASK 0xf00000
+#define RLC_MC_CNTL__RDNFO_URG__SHIFT 0x14
+#define RLC_MC_CNTL__RDREQ_SWAP_MASK 0x3000000
+#define RLC_MC_CNTL__RDREQ_SWAP__SHIFT 0x18
+#define RLC_MC_CNTL__RDREQ_TRAN_MASK 0x4000000
+#define RLC_MC_CNTL__RDREQ_TRAN__SHIFT 0x1a
+#define RLC_MC_CNTL__RDREQ_PRIV_MASK 0x8000000
+#define RLC_MC_CNTL__RDREQ_PRIV__SHIFT 0x1b
+#define RLC_MC_CNTL__RDNFO_STALL_MASK 0x10000000
+#define RLC_MC_CNTL__RDNFO_STALL__SHIFT 0x1c
+#define RLC_MC_CNTL__RESERVED_MASK 0xe0000000
+#define RLC_MC_CNTL__RESERVED__SHIFT 0x1d
+#define RLC_STAT__RLC_BUSY_MASK 0x1
+#define RLC_STAT__RLC_BUSY__SHIFT 0x0
+#define RLC_STAT__RLC_GPM_BUSY_MASK 0x2
+#define RLC_STAT__RLC_GPM_BUSY__SHIFT 0x1
+#define RLC_STAT__RLC_SPM_BUSY_MASK 0x4
+#define RLC_STAT__RLC_SPM_BUSY__SHIFT 0x2
+#define RLC_STAT__RLC_SRM_BUSY_MASK 0x8
+#define RLC_STAT__RLC_SRM_BUSY__SHIFT 0x3
+#define RLC_STAT__RESERVED_MASK 0xfffffff0
+#define RLC_STAT__RESERVED__SHIFT 0x4
+#define RLC_SAFE_MODE__CMD_MASK 0x1
+#define RLC_SAFE_MODE__CMD__SHIFT 0x0
+#define RLC_SAFE_MODE__MESSAGE_MASK 0x1e
+#define RLC_SAFE_MODE__MESSAGE__SHIFT 0x1
+#define RLC_SAFE_MODE__RESERVED1_MASK 0xe0
+#define RLC_SAFE_MODE__RESERVED1__SHIFT 0x5
+#define RLC_SAFE_MODE__RESPONSE_MASK 0xf00
+#define RLC_SAFE_MODE__RESPONSE__SHIFT 0x8
+#define RLC_SAFE_MODE__RESERVED_MASK 0xfffff000
+#define RLC_SAFE_MODE__RESERVED__SHIFT 0xc
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK 0x1
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN__SHIFT 0x0
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN_MASK 0x2
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN__SHIFT 0x1
+#define RLC_MEM_SLP_CNTL__RESERVED_MASK 0x7c
+#define RLC_MEM_SLP_CNTL__RESERVED__SHIFT 0x2
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE_MASK 0x80
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE__SHIFT 0x7
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY_MASK 0xff00
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY__SHIFT 0x8
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY_MASK 0xff0000
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY__SHIFT 0x10
+#define RLC_MEM_SLP_CNTL__RESERVED1_MASK 0xff000000
+#define RLC_MEM_SLP_CNTL__RESERVED1__SHIFT 0x18
+#define SMU_RLC_RESPONSE__RESP_MASK 0xffffffff
+#define SMU_RLC_RESPONSE__RESP__SHIFT 0x0
+#define RLC_RLCV_SAFE_MODE__CMD_MASK 0x1
+#define RLC_RLCV_SAFE_MODE__CMD__SHIFT 0x0
+#define RLC_RLCV_SAFE_MODE__MESSAGE_MASK 0x1e
+#define RLC_RLCV_SAFE_MODE__MESSAGE__SHIFT 0x1
+#define RLC_RLCV_SAFE_MODE__RESERVED1_MASK 0xe0
+#define RLC_RLCV_SAFE_MODE__RESERVED1__SHIFT 0x5
+#define RLC_RLCV_SAFE_MODE__RESPONSE_MASK 0xf00
+#define RLC_RLCV_SAFE_MODE__RESPONSE__SHIFT 0x8
+#define RLC_RLCV_SAFE_MODE__RESERVED_MASK 0xfffff000
+#define RLC_RLCV_SAFE_MODE__RESERVED__SHIFT 0xc
+#define RLC_SMU_SAFE_MODE__CMD_MASK 0x1
+#define RLC_SMU_SAFE_MODE__CMD__SHIFT 0x0
+#define RLC_SMU_SAFE_MODE__MESSAGE_MASK 0x1e
+#define RLC_SMU_SAFE_MODE__MESSAGE__SHIFT 0x1
+#define RLC_SMU_SAFE_MODE__RESERVED1_MASK 0xe0
+#define RLC_SMU_SAFE_MODE__RESERVED1__SHIFT 0x5
+#define RLC_SMU_SAFE_MODE__RESPONSE_MASK 0xf00
+#define RLC_SMU_SAFE_MODE__RESPONSE__SHIFT 0x8
+#define RLC_SMU_SAFE_MODE__RESERVED_MASK 0xfffff000
+#define RLC_SMU_SAFE_MODE__RESERVED__SHIFT 0xc
+#define RLC_RLCV_COMMAND__CMD_MASK 0xf
+#define RLC_RLCV_COMMAND__CMD__SHIFT 0x0
+#define RLC_RLCV_COMMAND__RESERVED_MASK 0xfffffff0
+#define RLC_RLCV_COMMAND__RESERVED__SHIFT 0x4
+#define RLC_CLK_CNTL__RLC_SRM_CLK_CNTL_MASK 0x1
+#define RLC_CLK_CNTL__RLC_SRM_CLK_CNTL__SHIFT 0x0
+#define RLC_CLK_CNTL__RLC_SPM_CLK_CNTL_MASK 0x2
+#define RLC_CLK_CNTL__RLC_SPM_CLK_CNTL__SHIFT 0x1
+#define RLC_CLK_CNTL__RESERVED_MASK 0xfffffffc
+#define RLC_CLK_CNTL__RESERVED__SHIFT 0x2
+#define RLC_PERFMON_CLK_CNTL__PERFMON_CLOCK_STATE_MASK 0x1
+#define RLC_PERFMON_CLK_CNTL__PERFMON_CLOCK_STATE__SHIFT 0x0
+#define RLC_PERFMON_CNTL__PERFMON_STATE_MASK 0x7
+#define RLC_PERFMON_CNTL__PERFMON_STATE__SHIFT 0x0
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK 0x400
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT 0xa
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0xff
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0xff
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define CGTT_RLC_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_RLC_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f
+#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK 0x1
+#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE__SHIFT 0x0
+#define RLC_LB_CNTL__LB_CNT_CP_BUSY_MASK 0x2
+#define RLC_LB_CNTL__LB_CNT_CP_BUSY__SHIFT 0x1
+#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK 0x4
+#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE__SHIFT 0x2
+#define RLC_LB_CNTL__LB_CNT_REG_INC_MASK 0x8
+#define RLC_LB_CNTL__LB_CNT_REG_INC__SHIFT 0x3
+#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST_MASK 0xff0
+#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST__SHIFT 0x4
+#define RLC_LB_CNTL__RESERVED_MASK 0xfffff000
+#define RLC_LB_CNTL__RESERVED__SHIFT 0xc
+#define RLC_LB_CNTR_MAX__LB_CNTR_MAX_MASK 0xffffffff
+#define RLC_LB_CNTR_MAX__LB_CNTR_MAX__SHIFT 0x0
+#define RLC_LB_CNTR_INIT__LB_CNTR_INIT_MASK 0xffffffff
+#define RLC_LB_CNTR_INIT__LB_CNTR_INIT__SHIFT 0x0
+#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR_MASK 0xffffffff
+#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR__SHIFT 0x0
+#define RLC_JUMP_TABLE_RESTORE__ADDR_MASK 0xffffffff
+#define RLC_JUMP_TABLE_RESTORE__ADDR__SHIFT 0x0
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE_MASK 0xff
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE__SHIFT 0x0
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK 0xff00
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT 0x8
+#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE_MASK 0xffff0000
+#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE__SHIFT 0x10
+#define RLC_GPM_DEBUG_SELECT__SELECT_MASK 0xff
+#define RLC_GPM_DEBUG_SELECT__SELECT__SHIFT 0x0
+#define RLC_GPM_DEBUG_SELECT__F32_DEBUG_SELECT_MASK 0x300
+#define RLC_GPM_DEBUG_SELECT__F32_DEBUG_SELECT__SHIFT 0x8
+#define RLC_GPM_DEBUG_SELECT__RESERVED_MASK 0xfffffc00
+#define RLC_GPM_DEBUG_SELECT__RESERVED__SHIFT 0xa
+#define RLC_GPM_DEBUG__DATA_MASK 0xffffffff
+#define RLC_GPM_DEBUG__DATA__SHIFT 0x0
+#define RLC_GPM_DEBUG_INST_A__INST_A_MASK 0xffffffff
+#define RLC_GPM_DEBUG_INST_A__INST_A__SHIFT 0x0
+#define RLC_GPM_DEBUG_INST_B__INST_B_MASK 0xffffffff
+#define RLC_GPM_DEBUG_INST_B__INST_B__SHIFT 0x0
+#define RLC_GPM_DEBUG_INST_ADDR__ADRR_A_MASK 0xffff
+#define RLC_GPM_DEBUG_INST_ADDR__ADRR_A__SHIFT 0x0
+#define RLC_GPM_DEBUG_INST_ADDR__ADDR_B_MASK 0xffff0000
+#define RLC_GPM_DEBUG_INST_ADDR__ADDR_B__SHIFT 0x10
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR_MASK 0xfff
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0
+#define RLC_GPM_UCODE_ADDR__RESERVED_MASK 0xfffff000
+#define RLC_GPM_UCODE_ADDR__RESERVED__SHIFT 0xc
+#define RLC_GPM_UCODE_DATA__UCODE_DATA_MASK 0xffffffff
+#define RLC_GPM_UCODE_DATA__UCODE_DATA__SHIFT 0x0
+#define GPU_BIST_CONTROL__STOP_ON_FAIL_HW_MASK 0x1
+#define GPU_BIST_CONTROL__STOP_ON_FAIL_HW__SHIFT 0x0
+#define GPU_BIST_CONTROL__STOP_ON_FAIL_CU_HARV_MASK 0x2
+#define GPU_BIST_CONTROL__STOP_ON_FAIL_CU_HARV__SHIFT 0x1
+#define GPU_BIST_CONTROL__CU_HARV_LOOP_COUNT_MASK 0x3c
+#define GPU_BIST_CONTROL__CU_HARV_LOOP_COUNT__SHIFT 0x2
+#define GPU_BIST_CONTROL__RESERVED_MASK 0xffff80
+#define GPU_BIST_CONTROL__RESERVED__SHIFT 0x7
+#define GPU_BIST_CONTROL__GLOBAL_LOOP_COUNT_MASK 0xff000000
+#define GPU_BIST_CONTROL__GLOBAL_LOOP_COUNT__SHIFT 0x18
+#define RLC_ROM_CNTL__USE_ROM_MASK 0x1
+#define RLC_ROM_CNTL__USE_ROM__SHIFT 0x0
+#define RLC_ROM_CNTL__SLP_MODE_EN_MASK 0x2
+#define RLC_ROM_CNTL__SLP_MODE_EN__SHIFT 0x1
+#define RLC_ROM_CNTL__EFUSE_DISTRIB_EN_MASK 0x4
+#define RLC_ROM_CNTL__EFUSE_DISTRIB_EN__SHIFT 0x2
+#define RLC_ROM_CNTL__HELLOWORLD_EN_MASK 0x8
+#define RLC_ROM_CNTL__HELLOWORLD_EN__SHIFT 0x3
+#define RLC_ROM_CNTL__CU_HARVEST_EN_MASK 0x10
+#define RLC_ROM_CNTL__CU_HARVEST_EN__SHIFT 0x4
+#define RLC_ROM_CNTL__RESERVED_MASK 0xffffffe0
+#define RLC_ROM_CNTL__RESERVED__SHIFT 0x5
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB_MASK 0xffffffff
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB__SHIFT 0x0
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB_MASK 0xffffffff
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB__SHIFT 0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE_MASK 0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE__SHIFT 0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED_MASK 0xfffffffe
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED__SHIFT 0x1
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS_MASK 0xffffffff
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS__SHIFT 0x0
+#define RLC_GPM_STAT__RLC_BUSY_MASK 0x1
+#define RLC_GPM_STAT__RLC_BUSY__SHIFT 0x0
+#define RLC_GPM_STAT__GFX_POWER_STATUS_MASK 0x2
+#define RLC_GPM_STAT__GFX_POWER_STATUS__SHIFT 0x1
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK 0x4
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS__SHIFT 0x2
+#define RLC_GPM_STAT__GFX_LS_STATUS_MASK 0x8
+#define RLC_GPM_STAT__GFX_LS_STATUS__SHIFT 0x3
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS_MASK 0x10
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS__SHIFT 0x4
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED_MASK 0x20
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED__SHIFT 0x5
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED_MASK 0x40
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED__SHIFT 0x6
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED_MASK 0x80
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED__SHIFT 0x7
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED_MASK 0x100
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED__SHIFT 0x8
+#define RLC_GPM_STAT__SAVING_REGISTERS_MASK 0x200
+#define RLC_GPM_STAT__SAVING_REGISTERS__SHIFT 0x9
+#define RLC_GPM_STAT__RESTORING_REGISTERS_MASK 0x400
+#define RLC_GPM_STAT__RESTORING_REGISTERS__SHIFT 0xa
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE_MASK 0x800
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE__SHIFT 0xb
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE_MASK 0x1000
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE__SHIFT 0xc
+#define RLC_GPM_STAT__STATIC_CU_POWERING_UP_MASK 0x2000
+#define RLC_GPM_STAT__STATIC_CU_POWERING_UP__SHIFT 0xd
+#define RLC_GPM_STAT__STATIC_CU_POWERING_DOWN_MASK 0x4000
+#define RLC_GPM_STAT__STATIC_CU_POWERING_DOWN__SHIFT 0xe
+#define RLC_GPM_STAT__DYN_CU_POWERING_UP_MASK 0x8000
+#define RLC_GPM_STAT__DYN_CU_POWERING_UP__SHIFT 0xf
+#define RLC_GPM_STAT__DYN_CU_POWERING_DOWN_MASK 0x10000
+#define RLC_GPM_STAT__DYN_CU_POWERING_DOWN__SHIFT 0x10
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE_MASK 0x20000
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT 0x11
+#define RLC_GPM_STAT__RESERVED_MASK 0xfc0000
+#define RLC_GPM_STAT__RESERVED__SHIFT 0x12
+#define RLC_GPM_STAT__PG_ERROR_STATUS_MASK 0xff000000
+#define RLC_GPM_STAT__PG_ERROR_STATUS__SHIFT 0x18
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK 0x3f
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL__SHIFT 0x0
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED_MASK 0xffffffc0
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED__SHIFT 0x6
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32_MASK 0xffffffff
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32__SHIFT 0x0
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK 0x1
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE__SHIFT 0x0
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK 0x2
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC__SHIFT 0x1
+#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK 0x4
+#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE__SHIFT 0x2
+#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK 0x8
+#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE__SHIFT 0x3
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK 0x10
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE__SHIFT 0x4
+#define RLC_PG_CNTL__RESERVED_MASK 0x3fe0
+#define RLC_PG_CNTL__RESERVED__SHIFT 0x5
+#define RLC_PG_CNTL__PG_OVERRIDE_MASK 0x4000
+#define RLC_PG_CNTL__PG_OVERRIDE__SHIFT 0xe
+#define RLC_PG_CNTL__CP_PG_DISABLE_MASK 0x8000
+#define RLC_PG_CNTL__CP_PG_DISABLE__SHIFT 0xf
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE_MASK 0x10000
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE__SHIFT 0x10
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK 0x20000
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE__SHIFT 0x11
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK 0x40000
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT 0x12
+#define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE_MASK 0x80000
+#define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE__SHIFT 0x13
+#define RLC_PG_CNTL__RESERVED1_MASK 0xf00000
+#define RLC_PG_CNTL__RESERVED1__SHIFT 0x14
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK 0xff
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT 0x0
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK 0xff00
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY__SHIFT 0x8
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY_MASK 0xff0000
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY__SHIFT 0x10
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY_MASK 0xff000000
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY__SHIFT 0x18
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE_MASK 0x1
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE__SHIFT 0x0
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE_MASK 0x2
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE__SHIFT 0x1
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE_MASK 0x4
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE__SHIFT 0x2
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE_MASK 0x8
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE__SHIFT 0x3
+#define RLC_GPM_THREAD_ENABLE__RESERVED_MASK 0xfffffff0
+#define RLC_GPM_THREAD_ENABLE__RESERVED__SHIFT 0x4
+#define RLC_GPM_VMID_THREAD0__RLC_VMID_MASK 0xf
+#define RLC_GPM_VMID_THREAD0__RLC_VMID__SHIFT 0x0
+#define RLC_GPM_VMID_THREAD0__RESERVED0_MASK 0xf0
+#define RLC_GPM_VMID_THREAD0__RESERVED0__SHIFT 0x4
+#define RLC_GPM_VMID_THREAD0__RLC_QUEUEID_MASK 0x700
+#define RLC_GPM_VMID_THREAD0__RLC_QUEUEID__SHIFT 0x8
+#define RLC_GPM_VMID_THREAD0__RESERVED1_MASK 0xfffff800
+#define RLC_GPM_VMID_THREAD0__RESERVED1__SHIFT 0xb
+#define RLC_GPM_VMID_THREAD1__RLC_VMID_MASK 0xf
+#define RLC_GPM_VMID_THREAD1__RLC_VMID__SHIFT 0x0
+#define RLC_GPM_VMID_THREAD1__RESERVED0_MASK 0xf0
+#define RLC_GPM_VMID_THREAD1__RESERVED0__SHIFT 0x4
+#define RLC_GPM_VMID_THREAD1__RLC_QUEUEID_MASK 0x700
+#define RLC_GPM_VMID_THREAD1__RLC_QUEUEID__SHIFT 0x8
+#define RLC_GPM_VMID_THREAD1__RESERVED1_MASK 0xfffff800
+#define RLC_GPM_VMID_THREAD1__RESERVED1__SHIFT 0xb
+#define RLC_CGTT_MGCG_OVERRIDE__OVERRIDE_MASK 0xffffffff
+#define RLC_CGTT_MGCG_OVERRIDE__OVERRIDE__SHIFT 0x0
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK 0x1
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN__SHIFT 0x0
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK 0x2
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN__SHIFT 0x1
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK 0xfc
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT 0x2
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK 0x7ffff00
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT 0x8
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER_MASK 0x8000000
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER__SHIFT 0x1b
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL_MASK 0x10000000
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL__SHIFT 0x1c
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE_MASK 0x60000000
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE__SHIFT 0x1d
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN_MASK 0x80000000
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN__SHIFT 0x1f
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT_MASK 0xf
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT__SHIFT 0x0
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT_MASK 0xf0
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT__SHIFT 0x4
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT_MASK 0xf00
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT__SHIFT 0x8
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT_MASK 0xf000
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT__SHIFT 0xc
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT_MASK 0xfff0000
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT__SHIFT 0x10
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT_MASK 0xf0000000
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT__SHIFT 0x1c
+#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK_MASK 0xffffffff
+#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK__SHIFT 0x0
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK_MASK 0xffffffff
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK__SHIFT 0x0
+#define RLC_PG_DELAY__POWER_UP_DELAY_MASK 0xff
+#define RLC_PG_DELAY__POWER_UP_DELAY__SHIFT 0x0
+#define RLC_PG_DELAY__POWER_DOWN_DELAY_MASK 0xff00
+#define RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT 0x8
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY_MASK 0xff0000
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT 0x10
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY_MASK 0xff000000
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT 0x18
+#define RLC_CU_STATUS__WORK_PENDING_MASK 0xffffffff
+#define RLC_CU_STATUS__WORK_PENDING__SHIFT 0x0
+#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK_MASK 0xffffffff
+#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK__SHIFT 0x0
+#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK_MASK 0xffffffff
+#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK__SHIFT 0x0
+#define RLC_LB_PARAMS__SKIP_L2_CHECK_MASK 0x1
+#define RLC_LB_PARAMS__SKIP_L2_CHECK__SHIFT 0x0
+#define RLC_LB_PARAMS__FIFO_SAMPLES_MASK 0xfe
+#define RLC_LB_PARAMS__FIFO_SAMPLES__SHIFT 0x1
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLES_MASK 0xff00
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLES__SHIFT 0x8
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL_MASK 0xffff0000
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL__SHIFT 0x10
+#define RLC_THREAD1_DELAY__CU_IDEL_DELAY_MASK 0xff
+#define RLC_THREAD1_DELAY__CU_IDEL_DELAY__SHIFT 0x0
+#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY_MASK 0xff00
+#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY__SHIFT 0x8
+#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY_MASK 0xff0000
+#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY__SHIFT 0x10
+#define RLC_THREAD1_DELAY__SPARE_MASK 0xff000000
+#define RLC_THREAD1_DELAY__SPARE__SHIFT 0x18
+#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK_MASK 0xffffffff
+#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK__SHIFT 0x0
+#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK 0xff
+#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT 0x0
+#define RLC_MAX_PG_CU__SPARE_MASK 0xffffff00
+#define RLC_MAX_PG_CU__SPARE__SHIFT 0x8
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK 0x1
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN__SHIFT 0x0
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN_MASK 0x2
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN__SHIFT 0x1
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN_MASK 0x4
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN__SHIFT 0x2
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK 0x7fff8
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT 0x3
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK 0xfff80000
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD__SHIFT 0x13
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE_MASK 0x1
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE__SHIFT 0x0
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE_MASK 0xfffffffe
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE__SHIFT 0x1
+#define RLC_SERDES_RD_MASTER_INDEX__CU_ID_MASK 0xf
+#define RLC_SERDES_RD_MASTER_INDEX__CU_ID__SHIFT 0x0
+#define RLC_SERDES_RD_MASTER_INDEX__SH_ID_MASK 0x30
+#define RLC_SERDES_RD_MASTER_INDEX__SH_ID__SHIFT 0x4
+#define RLC_SERDES_RD_MASTER_INDEX__SE_ID_MASK 0x1c0
+#define RLC_SERDES_RD_MASTER_INDEX__SE_ID__SHIFT 0x6
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID_MASK 0x200
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID__SHIFT 0x9
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_MASK 0x400
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU__SHIFT 0xa
+#define RLC_SERDES_RD_MASTER_INDEX__NON_SE_MASK 0x7800
+#define RLC_SERDES_RD_MASTER_INDEX__NON_SE__SHIFT 0xb
+#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID_MASK 0x18000
+#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID__SHIFT 0xf
+#define RLC_SERDES_RD_MASTER_INDEX__SPARE_MASK 0xfffe0000
+#define RLC_SERDES_RD_MASTER_INDEX__SPARE__SHIFT 0x11
+#define RLC_SERDES_RD_DATA_0__DATA_MASK 0xffffffff
+#define RLC_SERDES_RD_DATA_0__DATA__SHIFT 0x0
+#define RLC_SERDES_RD_DATA_1__DATA_MASK 0xffffffff
+#define RLC_SERDES_RD_DATA_1__DATA__SHIFT 0x0
+#define RLC_SERDES_RD_DATA_2__DATA_MASK 0xffffffff
+#define RLC_SERDES_RD_DATA_2__DATA__SHIFT 0x0
+#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK_MASK 0xffffffff
+#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK__SHIFT 0x0
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK_MASK 0xffff
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK__SHIFT 0x0
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK_MASK 0x10000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK__SHIFT 0x10
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_GFX_MASTER_MASK_MASK 0x20000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_GFX_MASTER_MASK__SHIFT 0x11
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK_MASK 0x40000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK__SHIFT 0x12
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK_MASK 0x80000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK__SHIFT 0x13
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK_MASK 0x100000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK__SHIFT 0x14
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK_MASK 0x200000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK__SHIFT 0x15
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK_MASK 0x400000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK__SHIFT 0x16
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK_MASK 0x800000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK__SHIFT 0x17
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED_MASK 0xff000000
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED__SHIFT 0x18
+#define RLC_SERDES_WR_CTRL__BPM_ADDR_MASK 0xff
+#define RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT 0x0
+#define RLC_SERDES_WR_CTRL__POWER_DOWN_MASK 0x100
+#define RLC_SERDES_WR_CTRL__POWER_DOWN__SHIFT 0x8
+#define RLC_SERDES_WR_CTRL__POWER_UP_MASK 0x200
+#define RLC_SERDES_WR_CTRL__POWER_UP__SHIFT 0x9
+#define RLC_SERDES_WR_CTRL__P1_SELECT_MASK 0x400
+#define RLC_SERDES_WR_CTRL__P1_SELECT__SHIFT 0xa
+#define RLC_SERDES_WR_CTRL__P2_SELECT_MASK 0x800
+#define RLC_SERDES_WR_CTRL__P2_SELECT__SHIFT 0xb
+#define RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK 0x1000
+#define RLC_SERDES_WR_CTRL__WRITE_COMMAND__SHIFT 0xc
+#define RLC_SERDES_WR_CTRL__READ_COMMAND_MASK 0x2000
+#define RLC_SERDES_WR_CTRL__READ_COMMAND__SHIFT 0xd
+#define RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK 0x4000
+#define RLC_SERDES_WR_CTRL__RDDATA_RESET__SHIFT 0xe
+#define RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK 0x8000
+#define RLC_SERDES_WR_CTRL__SHORT_FORMAT__SHIFT 0xf
+#define RLC_SERDES_WR_CTRL__BPM_DATA_MASK 0x3ff0000
+#define RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT 0x10
+#define RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK 0x4000000
+#define RLC_SERDES_WR_CTRL__SRBM_OVERRIDE__SHIFT 0x1a
+#define RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK 0x8000000
+#define RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR__SHIFT 0x1b
+#define RLC_SERDES_WR_CTRL__REG_ADDR_MASK 0xf0000000
+#define RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT 0x1c
+#define RLC_SERDES_WR_DATA__DATA_MASK 0xffffffff
+#define RLC_SERDES_WR_DATA__DATA__SHIFT 0x0
+#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY_MASK 0xffffffff
+#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY__SHIFT 0x0
+#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK 0xffff
+#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY__SHIFT 0x0
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK 0x10000
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY__SHIFT 0x10
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_GFX_MASTER_BUSY_MASK 0x20000
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_GFX_MASTER_BUSY__SHIFT 0x11
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK 0x40000
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY__SHIFT 0x12
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK 0x80000
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY__SHIFT 0x13
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY_MASK 0x100000
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY__SHIFT 0x14
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY_MASK 0x200000
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY__SHIFT 0x15
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY_MASK 0x400000
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY__SHIFT 0x16
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY_MASK 0x800000
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY__SHIFT 0x17
+#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED_MASK 0xff000000
+#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED__SHIFT 0x18
+#define RLC_GPM_GENERAL_0__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_0__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_1__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_1__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_2__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_2__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_3__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_3__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_4__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_4__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_5__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_5__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_6__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_6__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_7__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_7__DATA__SHIFT 0x0
+#define RLC_GPM_SCRATCH_ADDR__ADDR_MASK 0x1ff
+#define RLC_GPM_SCRATCH_ADDR__ADDR__SHIFT 0x0
+#define RLC_GPM_SCRATCH_ADDR__RESERVED_MASK 0xfffffe00
+#define RLC_GPM_SCRATCH_ADDR__RESERVED__SHIFT 0x9
+#define RLC_GPM_SCRATCH_DATA__DATA_MASK 0xffffffff
+#define RLC_GPM_SCRATCH_DATA__DATA__SHIFT 0x0
+#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK_MASK 0xffffffff
+#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK__SHIFT 0x0
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL_MASK 0xf
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL__SHIFT 0x0
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX_MASK 0xf0
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX__SHIFT 0x4
+#define RLC_GPM_PERF_COUNT_0__SH_INDEX_MASK 0xf00
+#define RLC_GPM_PERF_COUNT_0__SH_INDEX__SHIFT 0x8
+#define RLC_GPM_PERF_COUNT_0__CU_INDEX_MASK 0xf000
+#define RLC_GPM_PERF_COUNT_0__CU_INDEX__SHIFT 0xc
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL_MASK 0x30000
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL__SHIFT 0x10
+#define RLC_GPM_PERF_COUNT_0__UNUSED_MASK 0xc0000
+#define RLC_GPM_PERF_COUNT_0__UNUSED__SHIFT 0x12
+#define RLC_GPM_PERF_COUNT_0__ENABLE_MASK 0x100000
+#define RLC_GPM_PERF_COUNT_0__ENABLE__SHIFT 0x14
+#define RLC_GPM_PERF_COUNT_0__RESERVED_MASK 0xffe00000
+#define RLC_GPM_PERF_COUNT_0__RESERVED__SHIFT 0x15
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL_MASK 0xf
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL__SHIFT 0x0
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX_MASK 0xf0
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX__SHIFT 0x4
+#define RLC_GPM_PERF_COUNT_1__SH_INDEX_MASK 0xf00
+#define RLC_GPM_PERF_COUNT_1__SH_INDEX__SHIFT 0x8
+#define RLC_GPM_PERF_COUNT_1__CU_INDEX_MASK 0xf000
+#define RLC_GPM_PERF_COUNT_1__CU_INDEX__SHIFT 0xc
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL_MASK 0x30000
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL__SHIFT 0x10
+#define RLC_GPM_PERF_COUNT_1__UNUSED_MASK 0xc0000
+#define RLC_GPM_PERF_COUNT_1__UNUSED__SHIFT 0x12
+#define RLC_GPM_PERF_COUNT_1__ENABLE_MASK 0x100000
+#define RLC_GPM_PERF_COUNT_1__ENABLE__SHIFT 0x14
+#define RLC_GPM_PERF_COUNT_1__RESERVED_MASK 0xffe00000
+#define RLC_GPM_PERF_COUNT_1__RESERVED__SHIFT 0x15
+#define RLC_GPR_REG1__DATA_MASK 0xffffffff
+#define RLC_GPR_REG1__DATA__SHIFT 0x0
+#define RLC_GPR_REG2__DATA_MASK 0xffffffff
+#define RLC_GPR_REG2__DATA__SHIFT 0x0
+#define RLC_MGCG_CTRL__MGCG_EN_MASK 0x1
+#define RLC_MGCG_CTRL__MGCG_EN__SHIFT 0x0
+#define RLC_MGCG_CTRL__SILICON_EN_MASK 0x2
+#define RLC_MGCG_CTRL__SILICON_EN__SHIFT 0x1
+#define RLC_MGCG_CTRL__SIMULATION_EN_MASK 0x4
+#define RLC_MGCG_CTRL__SIMULATION_EN__SHIFT 0x2
+#define RLC_MGCG_CTRL__ON_DELAY_MASK 0x78
+#define RLC_MGCG_CTRL__ON_DELAY__SHIFT 0x3
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS_MASK 0x7f80
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS__SHIFT 0x7
+#define RLC_MGCG_CTRL__GC_CAC_MGCG_CLK_CNTL_MASK 0x8000
+#define RLC_MGCG_CTRL__GC_CAC_MGCG_CLK_CNTL__SHIFT 0xf
+#define RLC_MGCG_CTRL__SE_CAC_MGCG_CLK_CNTL_MASK 0x10000
+#define RLC_MGCG_CTRL__SE_CAC_MGCG_CLK_CNTL__SHIFT 0x10
+#define RLC_MGCG_CTRL__SPARE_MASK 0xfffe0000
+#define RLC_MGCG_CTRL__SPARE__SHIFT 0x11
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET_MASK 0x1
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET__SHIFT 0x0
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET_MASK 0x2
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET__SHIFT 0x1
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET_MASK 0x4
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET__SHIFT 0x2
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET_MASK 0x8
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET__SHIFT 0x3
+#define RLC_GPM_THREAD_RESET__RESERVED_MASK 0xfffffff0
+#define RLC_GPM_THREAD_RESET__RESERVED__SHIFT 0x4
+#define RLC_SPM_VMID__RLC_SPM_VMID_MASK 0xf
+#define RLC_SPM_VMID__RLC_SPM_VMID__SHIFT 0x0
+#define RLC_SPM_VMID__RESERVED_MASK 0xfffffff0
+#define RLC_SPM_VMID__RESERVED__SHIFT 0x4
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL_MASK 0x1
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL__SHIFT 0x0
+#define RLC_SPM_INT_CNTL__RESERVED_MASK 0xfffffffe
+#define RLC_SPM_INT_CNTL__RESERVED__SHIFT 0x1
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS_MASK 0x1
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS__SHIFT 0x0
+#define RLC_SPM_INT_STATUS__RESERVED_MASK 0xfffffffe
+#define RLC_SPM_INT_STATUS__RESERVED__SHIFT 0x1
+#define RLC_SPM_DEBUG_SELECT__SELECT_MASK 0xff
+#define RLC_SPM_DEBUG_SELECT__SELECT__SHIFT 0x0
+#define RLC_SPM_DEBUG_SELECT__RESERVED_MASK 0x7f00
+#define RLC_SPM_DEBUG_SELECT__RESERVED__SHIFT 0x8
+#define RLC_SPM_DEBUG_SELECT__RLC_SPM_DEBUG_MODE_MASK 0x8000
+#define RLC_SPM_DEBUG_SELECT__RLC_SPM_DEBUG_MODE__SHIFT 0xf
+#define RLC_SPM_DEBUG_SELECT__RLC_SPM_NUM_SAMPLE_MASK 0xffff0000
+#define RLC_SPM_DEBUG_SELECT__RLC_SPM_NUM_SAMPLE__SHIFT 0x10
+#define RLC_SPM_DEBUG__DATA_MASK 0xffffffff
+#define RLC_SPM_DEBUG__DATA__SHIFT 0x0
+#define RLC_SMU_MESSAGE__CMD_MASK 0xffffffff
+#define RLC_SMU_MESSAGE__CMD__SHIFT 0x0
+#define RLC_GPM_LOG_SIZE__SIZE_MASK 0xffffffff
+#define RLC_GPM_LOG_SIZE__SIZE__SHIFT 0x0
+#define RLC_GPM_LOG_CONT__CONT_MASK 0xffffffff
+#define RLC_GPM_LOG_CONT__CONT__SHIFT 0x0
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK 0xff
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT 0x0
+#define RLC_PG_DELAY_3__RESERVED_MASK 0xffffff00
+#define RLC_PG_DELAY_3__RESERVED__SHIFT 0x8
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE_MASK 0xffffffff
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE__SHIFT 0x0
+#define RLC_GPM_INT_DISABLE_TH1__DISABLE_MASK 0xffffffff
+#define RLC_GPM_INT_DISABLE_TH1__DISABLE__SHIFT 0x0
+#define RLC_GPM_INT_FORCE_TH0__FORCE_MASK 0xffffffff
+#define RLC_GPM_INT_FORCE_TH0__FORCE__SHIFT 0x0
+#define RLC_GPM_INT_FORCE_TH1__FORCE_MASK 0xffffffff
+#define RLC_GPM_INT_FORCE_TH1__FORCE__SHIFT 0x0
+#define RLC_SRM_CNTL__SRM_ENABLE_MASK 0x1
+#define RLC_SRM_CNTL__SRM_ENABLE__SHIFT 0x0
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK 0x2
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR__SHIFT 0x1
+#define RLC_SRM_CNTL__RESERVED_MASK 0xfffffffc
+#define RLC_SRM_CNTL__RESERVED__SHIFT 0x2
+#define RLC_SRM_DEBUG_SELECT__SELECT_MASK 0xff
+#define RLC_SRM_DEBUG_SELECT__SELECT__SHIFT 0x0
+#define RLC_SRM_DEBUG_SELECT__RESERVED_MASK 0xffffff00
+#define RLC_SRM_DEBUG_SELECT__RESERVED__SHIFT 0x8
+#define RLC_SRM_DEBUG__DATA_MASK 0xffffffff
+#define RLC_SRM_DEBUG__DATA__SHIFT 0x0
+#define RLC_SRM_ARAM_ADDR__ADDR_MASK 0x3ff
+#define RLC_SRM_ARAM_ADDR__ADDR__SHIFT 0x0
+#define RLC_SRM_ARAM_ADDR__RESERVED_MASK 0xfffffc00
+#define RLC_SRM_ARAM_ADDR__RESERVED__SHIFT 0xa
+#define RLC_SRM_ARAM_DATA__DATA_MASK 0xffffffff
+#define RLC_SRM_ARAM_DATA__DATA__SHIFT 0x0
+#define RLC_SRM_DRAM_ADDR__ADDR_MASK 0x3ff
+#define RLC_SRM_DRAM_ADDR__ADDR__SHIFT 0x0
+#define RLC_SRM_DRAM_ADDR__RESERVED_MASK 0xfffffc00
+#define RLC_SRM_DRAM_ADDR__RESERVED__SHIFT 0xa
+#define RLC_SRM_DRAM_DATA__DATA_MASK 0xffffffff
+#define RLC_SRM_DRAM_DATA__DATA__SHIFT 0x0
+#define RLC_SRM_GPM_COMMAND__OP_MASK 0x1
+#define RLC_SRM_GPM_COMMAND__OP__SHIFT 0x0
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_MASK 0x2
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL__SHIFT 0x1
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM_MASK 0x1c
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM__SHIFT 0x2
+#define RLC_SRM_GPM_COMMAND__SIZE_MASK 0x1ffe0
+#define RLC_SRM_GPM_COMMAND__SIZE__SHIFT 0x5
+#define RLC_SRM_GPM_COMMAND__START_OFFSET_MASK 0x1ffe0000
+#define RLC_SRM_GPM_COMMAND__START_OFFSET__SHIFT 0x11
+#define RLC_SRM_GPM_COMMAND__RESERVED1_MASK 0x60000000
+#define RLC_SRM_GPM_COMMAND__RESERVED1__SHIFT 0x1d
+#define RLC_SRM_GPM_COMMAND__DEST_MEMORY_MASK 0x80000000
+#define RLC_SRM_GPM_COMMAND__DEST_MEMORY__SHIFT 0x1f
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY_MASK 0x1
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY__SHIFT 0x0
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL_MASK 0x2
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL__SHIFT 0x1
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED_MASK 0xfffffffc
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED__SHIFT 0x2
+#define RLC_SRM_RLCV_COMMAND__OP_MASK 0x1
+#define RLC_SRM_RLCV_COMMAND__OP__SHIFT 0x0
+#define RLC_SRM_RLCV_COMMAND__RESERVED_MASK 0xe
+#define RLC_SRM_RLCV_COMMAND__RESERVED__SHIFT 0x1
+#define RLC_SRM_RLCV_COMMAND__SIZE_MASK 0xfff0
+#define RLC_SRM_RLCV_COMMAND__SIZE__SHIFT 0x4
+#define RLC_SRM_RLCV_COMMAND__START_OFFSET_MASK 0xfff0000
+#define RLC_SRM_RLCV_COMMAND__START_OFFSET__SHIFT 0x10
+#define RLC_SRM_RLCV_COMMAND__RESERVED1_MASK 0x70000000
+#define RLC_SRM_RLCV_COMMAND__RESERVED1__SHIFT 0x1c
+#define RLC_SRM_RLCV_COMMAND__DEST_MEMORY_MASK 0x80000000
+#define RLC_SRM_RLCV_COMMAND__DEST_MEMORY__SHIFT 0x1f
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_EMPTY_MASK 0x1
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_EMPTY__SHIFT 0x0
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_FULL_MASK 0x2
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_FULL__SHIFT 0x1
+#define RLC_SRM_RLCV_COMMAND_STATUS__RESERVED_MASK 0xfffffffc
+#define RLC_SRM_RLCV_COMMAND_STATUS__RESERVED__SHIFT 0x2
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_0__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_0__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_1__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_1__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_2__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_2__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_3__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_3__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_4__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_4__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_5__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_5__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_6__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_6__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS_MASK 0xffff
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_7__RESERVED_MASK 0xffff0000
+#define RLC_SRM_INDEX_CNTL_ADDR_7__RESERVED__SHIFT 0x10
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA__SHIFT 0x0
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA_MASK 0xffffffff
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA__SHIFT 0x0
+#define RLC_SRM_STAT__SRM_STATUS_MASK 0x1
+#define RLC_SRM_STAT__SRM_STATUS__SHIFT 0x0
+#define RLC_SRM_STAT__RESERVED_MASK 0xfffffffe
+#define RLC_SRM_STAT__RESERVED__SHIFT 0x1
+#define RLC_SRM_GPM_ABORT__ABORT_MASK 0x1
+#define RLC_SRM_GPM_ABORT__ABORT__SHIFT 0x0
+#define RLC_SRM_GPM_ABORT__RESERVED_MASK 0xfffffffe
+#define RLC_SRM_GPM_ABORT__RESERVED__SHIFT 0x1
+#define RLC_CSIB_ADDR_LO__ADDRESS_MASK 0xffffffff
+#define RLC_CSIB_ADDR_LO__ADDRESS__SHIFT 0x0
+#define RLC_CSIB_ADDR_HI__ADDRESS_MASK 0xffff
+#define RLC_CSIB_ADDR_HI__ADDRESS__SHIFT 0x0
+#define RLC_CSIB_LENGTH__LENGTH_MASK 0xffffffff
+#define RLC_CSIB_LENGTH__LENGTH__SHIFT 0x0
+#define RLC_CP_RESPONSE0__RESPONSE_MASK 0xffffffff
+#define RLC_CP_RESPONSE0__RESPONSE__SHIFT 0x0
+#define RLC_CP_RESPONSE1__RESPONSE_MASK 0xffffffff
+#define RLC_CP_RESPONSE1__RESPONSE__SHIFT 0x0
+#define RLC_CP_RESPONSE2__RESPONSE_MASK 0xffffffff
+#define RLC_CP_RESPONSE2__RESPONSE__SHIFT 0x0
+#define RLC_CP_RESPONSE3__RESPONSE_MASK 0xffffffff
+#define RLC_CP_RESPONSE3__RESPONSE__SHIFT 0x0
+#define RLC_SMU_COMMAND__CMD_MASK 0xffffffff
+#define RLC_SMU_COMMAND__CMD__SHIFT 0x0
+#define RLC_CP_SCHEDULERS__scheduler0_MASK 0xff
+#define RLC_CP_SCHEDULERS__scheduler0__SHIFT 0x0
+#define RLC_CP_SCHEDULERS__scheduler1_MASK 0xff00
+#define RLC_CP_SCHEDULERS__scheduler1__SHIFT 0x8
+#define RLC_CP_SCHEDULERS__scheduler2_MASK 0xff0000
+#define RLC_CP_SCHEDULERS__scheduler2__SHIFT 0x10
+#define RLC_CP_SCHEDULERS__scheduler3_MASK 0xff000000
+#define RLC_CP_SCHEDULERS__scheduler3__SHIFT 0x18
+#define RLC_SMU_ARGUMENT_1__ARG_MASK 0xffffffff
+#define RLC_SMU_ARGUMENT_1__ARG__SHIFT 0x0
+#define RLC_SMU_ARGUMENT_2__ARG_MASK 0xffffffff
+#define RLC_SMU_ARGUMENT_2__ARG__SHIFT 0x0
+#define RLC_GPM_GENERAL_8__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_8__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_9__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_9__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_10__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_10__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_11__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_11__DATA__SHIFT 0x0
+#define RLC_GPM_GENERAL_12__DATA_MASK 0xffffffff
+#define RLC_GPM_GENERAL_12__DATA__SHIFT 0x0
+#define RLC_SPM_PERFMON_CNTL__RESERVED1_MASK 0xfff
+#define RLC_SPM_PERFMON_CNTL__RESERVED1__SHIFT 0x0
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE_MASK 0x3000
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE__SHIFT 0xc
+#define RLC_SPM_PERFMON_CNTL__RESERVED_MASK 0xc000
+#define RLC_SPM_PERFMON_CNTL__RESERVED__SHIFT 0xe
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_MASK 0xffff0000
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL__SHIFT 0x10
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO_MASK 0xffffffff
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO__SHIFT 0x0
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI_MASK 0xffff
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI__SHIFT 0x0
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED_MASK 0xffff0000
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED__SHIFT 0x10
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE_MASK 0xffffffff
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE__SHIFT 0x0
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE_MASK 0xff
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE__SHIFT 0x0
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1_MASK 0x700
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1__SHIFT 0x8
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE_MASK 0xf800
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE__SHIFT 0xb
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE_MASK 0x1f0000
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE__SHIFT 0x10
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE_MASK 0x3e00000
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE__SHIFT 0x15
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE_MASK 0x7c000000
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE__SHIFT 0x1a
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED_MASK 0x80000000
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED__SHIFT 0x1f
+#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK 0xffffffff
+#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT 0x0
+#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA_MASK 0xffffffff
+#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT 0x0
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK 0xffffffff
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT 0x0
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA_MASK 0xffffffff
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT 0x0
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR_MASK 0xffffffff
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR__SHIFT 0x0
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD_MASK 0xffffffff
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD__SHIFT 0x0
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE_MASK 0x1
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE__SHIFT 0x0
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED_MASK 0xfffe
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED__SHIFT 0x1
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM_MASK 0xffff0000
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM__SHIFT 0x10
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP_MASK 0xffffffff
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP__SHIFT 0x0
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID_MASK 0xf
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID__SHIFT 0x0
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_MASK 0x7ffffff0
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED__SHIFT 0x4
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF_MASK 0x80000000
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF__SHIFT 0x1f
+#define SPI_PS_INPUT_CNTL_0__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_0__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_0__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_0__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_0__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_0__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_1__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_1__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_1__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_1__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_1__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_1__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_2__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_2__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_2__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_2__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_2__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_2__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_3__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_3__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_3__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_3__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_3__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_3__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_4__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_4__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_4__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_4__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_4__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_4__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_5__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_5__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_5__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_5__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_5__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_5__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_6__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_6__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_6__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_6__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_6__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_6__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_7__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_7__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_7__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_7__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_7__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_7__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_8__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_8__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_8__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_8__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_8__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_8__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_9__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_9__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_9__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_9__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_9__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_9__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_10__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_10__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_10__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_10__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_10__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_10__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_11__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_11__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_11__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_11__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_11__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_11__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_12__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_12__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_12__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_12__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_12__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_12__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_13__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_13__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_13__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_13__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_13__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_13__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_14__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_14__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_14__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_14__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_14__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_14__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_15__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_15__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_15__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_15__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_15__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_15__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_16__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_16__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_16__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_16__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_16__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_16__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_17__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_17__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_17__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_17__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_17__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_17__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_18__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_18__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_18__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_18__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_18__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_18__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_19__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_19__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_19__CYL_WRAP_MASK 0x1e000
+#define SPI_PS_INPUT_CNTL_19__CYL_WRAP__SHIFT 0xd
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_MASK 0x20000
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX__SHIFT 0x11
+#define SPI_PS_INPUT_CNTL_19__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_19__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1_MASK 0x800000
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1__SHIFT 0x17
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_20__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_20__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_20__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_20__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_21__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_21__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_21__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_21__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_22__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_22__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_22__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_22__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_23__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_23__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_23__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_23__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_24__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_24__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_24__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_24__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_25__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_25__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_25__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_25__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_26__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_26__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_26__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_26__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_27__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_27__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_27__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_27__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_28__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_28__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_28__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_28__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_29__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_29__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_29__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_29__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_30__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_30__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_30__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_30__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID__SHIFT 0x19
+#define SPI_PS_INPUT_CNTL_31__OFFSET_MASK 0x3f
+#define SPI_PS_INPUT_CNTL_31__OFFSET__SHIFT 0x0
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_MASK 0x300
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL__SHIFT 0x8
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE_MASK 0x400
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE__SHIFT 0xa
+#define SPI_PS_INPUT_CNTL_31__DUP_MASK 0x40000
+#define SPI_PS_INPUT_CNTL_31__DUP__SHIFT 0x12
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE_MASK 0x80000
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE__SHIFT 0x13
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1_MASK 0x100000
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1__SHIFT 0x14
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1_MASK 0x600000
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1__SHIFT 0x15
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID_MASK 0x1000000
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID__SHIFT 0x18
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID_MASK 0x2000000
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID__SHIFT 0x19
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT_MASK 0x3e
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT__SHIFT 0x1
+#define SPI_VS_OUT_CONFIG__VS_HALF_PACK_MASK 0x40
+#define SPI_VS_OUT_CONFIG__VS_HALF_PACK__SHIFT 0x6
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA_MASK 0x1
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA__SHIFT 0x0
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA_MASK 0x2
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA__SHIFT 0x1
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA_MASK 0x4
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA__SHIFT 0x2
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA_MASK 0x8
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA__SHIFT 0x3
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA_MASK 0x10
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA__SHIFT 0x4
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA_MASK 0x20
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA__SHIFT 0x5
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA_MASK 0x40
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA__SHIFT 0x6
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA_MASK 0x80
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA__SHIFT 0x7
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA_MASK 0x100
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA__SHIFT 0x8
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA_MASK 0x200
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA__SHIFT 0x9
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA_MASK 0x400
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA__SHIFT 0xa
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA_MASK 0x800
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA__SHIFT 0xb
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA_MASK 0x1000
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA__SHIFT 0xc
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA_MASK 0x2000
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA__SHIFT 0xd
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA_MASK 0x4000
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA__SHIFT 0xe
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA_MASK 0x8000
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA__SHIFT 0xf
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA_MASK 0x1
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA__SHIFT 0x0
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA_MASK 0x2
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA__SHIFT 0x1
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA_MASK 0x4
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA__SHIFT 0x2
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA_MASK 0x8
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA__SHIFT 0x3
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA_MASK 0x10
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA__SHIFT 0x4
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA_MASK 0x20
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA__SHIFT 0x5
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA_MASK 0x40
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA__SHIFT 0x6
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA_MASK 0x80
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA__SHIFT 0x7
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA_MASK 0x100
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA__SHIFT 0x8
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA_MASK 0x200
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA__SHIFT 0x9
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA_MASK 0x400
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA__SHIFT 0xa
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA_MASK 0x800
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA__SHIFT 0xb
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA_MASK 0x1000
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA__SHIFT 0xc
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA_MASK 0x2000
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA__SHIFT 0xd
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA_MASK 0x4000
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA__SHIFT 0xe
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA_MASK 0x8000
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA__SHIFT 0xf
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA_MASK 0x1
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA__SHIFT 0x0
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA_MASK 0x2
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA__SHIFT 0x1
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X_MASK 0x1c
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X__SHIFT 0x2
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y_MASK 0xe0
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y__SHIFT 0x5
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z_MASK 0x700
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z__SHIFT 0x8
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W_MASK 0x3800
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W__SHIFT 0xb
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1_MASK 0x4000
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1__SHIFT 0xe
+#define SPI_PS_IN_CONTROL__NUM_INTERP_MASK 0x3f
+#define SPI_PS_IN_CONTROL__NUM_INTERP__SHIFT 0x0
+#define SPI_PS_IN_CONTROL__PARAM_GEN_MASK 0x40
+#define SPI_PS_IN_CONTROL__PARAM_GEN__SHIFT 0x6
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE_MASK 0x4000
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE__SHIFT 0xe
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL_MASK 0x1
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL__SHIFT 0x0
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL_MASK 0x10
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL__SHIFT 0x4
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL_MASK 0x100
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL__SHIFT 0x8
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL_MASK 0x1000
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL__SHIFT 0xc
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION_MASK 0x30000
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION__SHIFT 0x10
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC_MASK 0x100000
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC__SHIFT 0x14
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS_MASK 0x1000000
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS__SHIFT 0x18
+#define SPI_TMPRING_SIZE__WAVES_MASK 0xfff
+#define SPI_TMPRING_SIZE__WAVES__SHIFT 0x0
+#define SPI_TMPRING_SIZE__WAVESIZE_MASK 0x1fff000
+#define SPI_TMPRING_SIZE__WAVESIZE__SHIFT 0xc
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT_MASK 0xf
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT__SHIFT 0x0
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT_MASK 0xf0
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT__SHIFT 0x4
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT_MASK 0xf00
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT__SHIFT 0x8
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT_MASK 0xf000
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT__SHIFT 0xc
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT_MASK 0xf
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT__SHIFT 0x0
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT_MASK 0xf
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT__SHIFT 0x0
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT_MASK 0xf0
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT__SHIFT 0x4
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT_MASK 0xf00
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT__SHIFT 0x8
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT_MASK 0xf000
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT__SHIFT 0xc
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT_MASK 0xf0000
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT__SHIFT 0x10
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT_MASK 0xf00000
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT__SHIFT 0x14
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT_MASK 0xf000000
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT__SHIFT 0x18
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT_MASK 0xf0000000
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT__SHIFT 0x1c
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0_MASK 0x7
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0__SHIFT 0x0
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1_MASK 0x38
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1__SHIFT 0x3
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2_MASK 0x1c0
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2__SHIFT 0x6
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3_MASK 0xe00
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3__SHIFT 0x9
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT_MASK 0x3000
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT__SHIFT 0xc
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT_MASK 0xc000
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT__SHIFT 0xe
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT_MASK 0x30000
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT__SHIFT 0x10
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT_MASK 0xc0000
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT__SHIFT 0x12
+#define SPI_ARB_CYCLES_0__TS0_DURATION_MASK 0xffff
+#define SPI_ARB_CYCLES_0__TS0_DURATION__SHIFT 0x0
+#define SPI_ARB_CYCLES_0__TS1_DURATION_MASK 0xffff0000
+#define SPI_ARB_CYCLES_0__TS1_DURATION__SHIFT 0x10
+#define SPI_ARB_CYCLES_1__TS2_DURATION_MASK 0xffff
+#define SPI_ARB_CYCLES_1__TS2_DURATION__SHIFT 0x0
+#define SPI_ARB_CYCLES_1__TS3_DURATION_MASK 0xffff0000
+#define SPI_ARB_CYCLES_1__TS3_DURATION__SHIFT 0x10
+#define SPI_CDBG_SYS_GFX__PS_EN_MASK 0x1
+#define SPI_CDBG_SYS_GFX__PS_EN__SHIFT 0x0
+#define SPI_CDBG_SYS_GFX__VS_EN_MASK 0x2
+#define SPI_CDBG_SYS_GFX__VS_EN__SHIFT 0x1
+#define SPI_CDBG_SYS_GFX__GS_EN_MASK 0x4
+#define SPI_CDBG_SYS_GFX__GS_EN__SHIFT 0x2
+#define SPI_CDBG_SYS_GFX__ES_EN_MASK 0x8
+#define SPI_CDBG_SYS_GFX__ES_EN__SHIFT 0x3
+#define SPI_CDBG_SYS_GFX__HS_EN_MASK 0x10
+#define SPI_CDBG_SYS_GFX__HS_EN__SHIFT 0x4
+#define SPI_CDBG_SYS_GFX__LS_EN_MASK 0x20
+#define SPI_CDBG_SYS_GFX__LS_EN__SHIFT 0x5
+#define SPI_CDBG_SYS_GFX__CS_EN_MASK 0x40
+#define SPI_CDBG_SYS_GFX__CS_EN__SHIFT 0x6
+#define SPI_CDBG_SYS_HP3D__PS_EN_MASK 0x1
+#define SPI_CDBG_SYS_HP3D__PS_EN__SHIFT 0x0
+#define SPI_CDBG_SYS_HP3D__VS_EN_MASK 0x2
+#define SPI_CDBG_SYS_HP3D__VS_EN__SHIFT 0x1
+#define SPI_CDBG_SYS_HP3D__GS_EN_MASK 0x4
+#define SPI_CDBG_SYS_HP3D__GS_EN__SHIFT 0x2
+#define SPI_CDBG_SYS_HP3D__ES_EN_MASK 0x8
+#define SPI_CDBG_SYS_HP3D__ES_EN__SHIFT 0x3
+#define SPI_CDBG_SYS_HP3D__HS_EN_MASK 0x10
+#define SPI_CDBG_SYS_HP3D__HS_EN__SHIFT 0x4
+#define SPI_CDBG_SYS_HP3D__LS_EN_MASK 0x20
+#define SPI_CDBG_SYS_HP3D__LS_EN__SHIFT 0x5
+#define SPI_CDBG_SYS_CS0__PIPE0_MASK 0xff
+#define SPI_CDBG_SYS_CS0__PIPE0__SHIFT 0x0
+#define SPI_CDBG_SYS_CS0__PIPE1_MASK 0xff00
+#define SPI_CDBG_SYS_CS0__PIPE1__SHIFT 0x8
+#define SPI_CDBG_SYS_CS0__PIPE2_MASK 0xff0000
+#define SPI_CDBG_SYS_CS0__PIPE2__SHIFT 0x10
+#define SPI_CDBG_SYS_CS0__PIPE3_MASK 0xff000000
+#define SPI_CDBG_SYS_CS0__PIPE3__SHIFT 0x18
+#define SPI_CDBG_SYS_CS1__PIPE0_MASK 0xff
+#define SPI_CDBG_SYS_CS1__PIPE0__SHIFT 0x0
+#define SPI_CDBG_SYS_CS1__PIPE1_MASK 0xff00
+#define SPI_CDBG_SYS_CS1__PIPE1__SHIFT 0x8
+#define SPI_CDBG_SYS_CS1__PIPE2_MASK 0xff0000
+#define SPI_CDBG_SYS_CS1__PIPE2__SHIFT 0x10
+#define SPI_CDBG_SYS_CS1__PIPE3_MASK 0xff000000
+#define SPI_CDBG_SYS_CS1__PIPE3__SHIFT 0x18
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_GFX__LS_GRP_VALUE_MASK 0xf80
+#define SPI_WCL_PIPE_PERCENT_GFX__LS_GRP_VALUE__SHIFT 0x7
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE_MASK 0x1f000
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE__SHIFT 0xc
+#define SPI_WCL_PIPE_PERCENT_GFX__ES_GRP_VALUE_MASK 0x3e0000
+#define SPI_WCL_PIPE_PERCENT_GFX__ES_GRP_VALUE__SHIFT 0x11
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE_MASK 0x7c00000
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE__SHIFT 0x16
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_HP3D__LS_GRP_VALUE_MASK 0xf80
+#define SPI_WCL_PIPE_PERCENT_HP3D__LS_GRP_VALUE__SHIFT 0x7
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE_MASK 0x1f000
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE__SHIFT 0xc
+#define SPI_WCL_PIPE_PERCENT_HP3D__ES_GRP_VALUE_MASK 0x3e0000
+#define SPI_WCL_PIPE_PERCENT_HP3D__ES_GRP_VALUE__SHIFT 0x11
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE_MASK 0x7c00000
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE__SHIFT 0x16
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE__SHIFT 0x0
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE_MASK 0x7f
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE__SHIFT 0x0
+#define SPI_GDBG_WAVE_CNTL__STALL_RA_MASK 0x1
+#define SPI_GDBG_WAVE_CNTL__STALL_RA__SHIFT 0x0
+#define SPI_GDBG_WAVE_CNTL__STALL_VMID_MASK 0x1fffe
+#define SPI_GDBG_WAVE_CNTL__STALL_VMID__SHIFT 0x1
+#define SPI_GDBG_TRAP_CONFIG__ME_SEL_MASK 0x3
+#define SPI_GDBG_TRAP_CONFIG__ME_SEL__SHIFT 0x0
+#define SPI_GDBG_TRAP_CONFIG__PIPE_SEL_MASK 0xc
+#define SPI_GDBG_TRAP_CONFIG__PIPE_SEL__SHIFT 0x2
+#define SPI_GDBG_TRAP_CONFIG__QUEUE_SEL_MASK 0x70
+#define SPI_GDBG_TRAP_CONFIG__QUEUE_SEL__SHIFT 0x4
+#define SPI_GDBG_TRAP_CONFIG__ME_MATCH_MASK 0x80
+#define SPI_GDBG_TRAP_CONFIG__ME_MATCH__SHIFT 0x7
+#define SPI_GDBG_TRAP_CONFIG__PIPE_MATCH_MASK 0x100
+#define SPI_GDBG_TRAP_CONFIG__PIPE_MATCH__SHIFT 0x8
+#define SPI_GDBG_TRAP_CONFIG__QUEUE_MATCH_MASK 0x200
+#define SPI_GDBG_TRAP_CONFIG__QUEUE_MATCH__SHIFT 0x9
+#define SPI_GDBG_TRAP_CONFIG__TRAP_EN_MASK 0x8000
+#define SPI_GDBG_TRAP_CONFIG__TRAP_EN__SHIFT 0xf
+#define SPI_GDBG_TRAP_CONFIG__VMID_SEL_MASK 0xffff0000
+#define SPI_GDBG_TRAP_CONFIG__VMID_SEL__SHIFT 0x10
+#define SPI_GDBG_TRAP_MASK__EXCP_EN_MASK 0x1ff
+#define SPI_GDBG_TRAP_MASK__EXCP_EN__SHIFT 0x0
+#define SPI_GDBG_TRAP_MASK__REPLACE_MASK 0x200
+#define SPI_GDBG_TRAP_MASK__REPLACE__SHIFT 0x9
+#define SPI_GDBG_TBA_LO__MEM_BASE_MASK 0xffffffff
+#define SPI_GDBG_TBA_LO__MEM_BASE__SHIFT 0x0
+#define SPI_GDBG_TBA_HI__MEM_BASE_MASK 0xff
+#define SPI_GDBG_TBA_HI__MEM_BASE__SHIFT 0x0
+#define SPI_GDBG_TMA_LO__MEM_BASE_MASK 0xffffffff
+#define SPI_GDBG_TMA_LO__MEM_BASE__SHIFT 0x0
+#define SPI_GDBG_TMA_HI__MEM_BASE_MASK 0xff
+#define SPI_GDBG_TMA_HI__MEM_BASE__SHIFT 0x0
+#define SPI_GDBG_TRAP_DATA0__DATA_MASK 0xffffffff
+#define SPI_GDBG_TRAP_DATA0__DATA__SHIFT 0x0
+#define SPI_GDBG_TRAP_DATA1__DATA_MASK 0xffffffff
+#define SPI_GDBG_TRAP_DATA1__DATA__SHIFT 0x0
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_MASK 0x1
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET__SHIFT 0x0
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID_MASK 0x2
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID__SHIFT 0x1
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID_MASK 0x4
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID__SHIFT 0x2
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE_MASK 0x8
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE__SHIFT 0x3
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY_MASK 0x10
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY__SHIFT 0x4
+#define SPI_COMPUTE_QUEUE_RESET__RESET_MASK 0x1
+#define SPI_COMPUTE_QUEUE_RESET__RESET__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_0__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_0__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_1__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_1__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_2__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_2__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_3__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_3__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_4__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_4__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_5__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_5__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_6__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_6__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_7__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_7__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_8__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_8__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_9__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_9__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_10__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_10__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_11__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_11__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_12__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_12__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_13__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_13__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_14__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_14__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR_MASK 0xf
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR_MASK 0xf0
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR__SHIFT 0x4
+#define SPI_RESOURCE_RESERVE_CU_15__LDS_MASK 0xf00
+#define SPI_RESOURCE_RESERVE_CU_15__LDS__SHIFT 0x8
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES_MASK 0x7000
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES__SHIFT 0xc
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS_MASK 0x78000
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS__SHIFT 0xf
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_12__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_12__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_13__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_13__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_14__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_14__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN_MASK 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN__SHIFT 0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK_MASK 0xfffe
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK__SHIFT 0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK_MASK 0xff0000
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK__SHIFT 0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_15__RESERVE_SPACE_ONLY_MASK 0x1000000
+#define SPI_RESOURCE_RESERVE_EN_CU_15__RESERVE_SPACE_ONLY__SHIFT 0x18
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE_MASK 0x1
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE__SHIFT 0x0
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN_MASK 0x2
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN__SHIFT 0x1
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN_MASK 0x4
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN__SHIFT 0x2
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY_MASK 0x40000000
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY__SHIFT 0x1e
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY_MASK 0x80000000
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY__SHIFT 0x1f
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID_MASK 0xfff
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT 0x0
+#define SPI_START_PHASE__VGPR_START_PHASE_MASK 0x3
+#define SPI_START_PHASE__VGPR_START_PHASE__SHIFT 0x0
+#define SPI_START_PHASE__SGPR_START_PHASE_MASK 0xc
+#define SPI_START_PHASE__SGPR_START_PHASE__SHIFT 0x2
+#define SPI_START_PHASE__WAVE_START_PHASE_MASK 0x30
+#define SPI_START_PHASE__WAVE_START_PHASE__SHIFT 0x4
+#define SPI_GFX_CNTL__RESET_COUNTS_MASK 0x1
+#define SPI_GFX_CNTL__RESET_COUNTS__SHIFT 0x0
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY_MASK 0x1fffff
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY__SHIFT 0x0
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER_MASK 0xe00000
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER__SHIFT 0x15
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS_MASK 0x1000000
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS__SHIFT 0x18
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS_MASK 0x2000000
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS__SHIFT 0x19
+#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET_MASK 0x4000000
+#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET__SHIFT 0x1a
+#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL_MASK 0x8000000
+#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL__SHIFT 0x1b
+#define SPI_DEBUG_CNTL__DEBUG_GRBM_OVERRIDE_MASK 0x1
+#define SPI_DEBUG_CNTL__DEBUG_GRBM_OVERRIDE__SHIFT 0x0
+#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL_MASK 0xe
+#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL__SHIFT 0x1
+#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL_MASK 0x3f0
+#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL__SHIFT 0x4
+#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL_MASK 0xfc00
+#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL__SHIFT 0xa
+#define SPI_DEBUG_CNTL__DEBUG_SH_SEL_MASK 0x10000
+#define SPI_DEBUG_CNTL__DEBUG_SH_SEL__SHIFT 0x10
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0_MASK 0x20000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0__SHIFT 0x11
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1_MASK 0x40000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1__SHIFT 0x12
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_2_MASK 0x80000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_2__SHIFT 0x13
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3_MASK 0x100000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3__SHIFT 0x14
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_4_MASK 0x200000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_4__SHIFT 0x15
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_5_MASK 0x400000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_5__SHIFT 0x16
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_6_MASK 0x800000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_6__SHIFT 0x17
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_7_MASK 0x1000000
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_7__SHIFT 0x18
+#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL_MASK 0xe000000
+#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL__SHIFT 0x19
+#define SPI_DEBUG_CNTL__DEBUG_REG_EN_MASK 0x80000000
+#define SPI_DEBUG_CNTL__DEBUG_REG_EN__SHIFT 0x1f
+#define SPI_DEBUG_READ__DATA_MASK 0xffffff
+#define SPI_DEBUG_READ__DATA__SHIFT 0x0
+#define SPI_DSM_CNTL__Sel_DSM_SPI_Irritator_data0_MASK 0x1
+#define SPI_DSM_CNTL__Sel_DSM_SPI_Irritator_data0__SHIFT 0x0
+#define SPI_DSM_CNTL__Sel_DSM_SPI_Irritator_data1_MASK 0x2
+#define SPI_DSM_CNTL__Sel_DSM_SPI_Irritator_data1__SHIFT 0x1
+#define SPI_DSM_CNTL__SPI_Enable_Single_Write_MASK 0x4
+#define SPI_DSM_CNTL__SPI_Enable_Single_Write__SHIFT 0x2
+#define SPI_DSM_CNTL__UNUSED_MASK 0xfffffff8
+#define SPI_DSM_CNTL__UNUSED__SHIFT 0x3
+#define SPI_EDC_CNT__SED_MASK 0xff
+#define SPI_EDC_CNT__SED__SHIFT 0x0
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK 0xffc00
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT 0xa
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1_MASK 0xffc00
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT 0xa
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK 0x3ff
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT 0x0
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK 0xffc00
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT 0xa
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK 0x3ff
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT 0x0
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK 0xffc00
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT 0xa
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0xff
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x0
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0xff
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x0
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN_MASK 0xf
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN__SHIFT 0x0
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX_MASK 0xf0
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX__SHIFT 0x4
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN_MASK 0xf00
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN__SHIFT 0x8
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX_MASK 0xf000
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX__SHIFT 0xc
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN_MASK 0xf0000
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN__SHIFT 0x10
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX_MASK 0xf00000
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX__SHIFT 0x14
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN_MASK 0xf000000
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN__SHIFT 0x18
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX_MASK 0xf0000000
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX__SHIFT 0x1c
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY_MASK 0xf
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT 0x0
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW_MASK 0x10
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW__SHIFT 0x4
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE_MASK 0x40
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE__SHIFT 0x6
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT_MASK 0x80
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT__SHIFT 0x7
+#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE_MASK 0x100
+#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE__SHIFT 0x8
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE_MASK 0x200
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE__SHIFT 0x9
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT_MASK 0x3c00
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT__SHIFT 0xa
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE_MASK 0xffff0000
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE__SHIFT 0x10
+#define SPI_DEBUG_BUSY__LS_BUSY_MASK 0x1
+#define SPI_DEBUG_BUSY__LS_BUSY__SHIFT 0x0
+#define SPI_DEBUG_BUSY__HS_BUSY_MASK 0x2
+#define SPI_DEBUG_BUSY__HS_BUSY__SHIFT 0x1
+#define SPI_DEBUG_BUSY__ES_BUSY_MASK 0x4
+#define SPI_DEBUG_BUSY__ES_BUSY__SHIFT 0x2
+#define SPI_DEBUG_BUSY__GS_BUSY_MASK 0x8
+#define SPI_DEBUG_BUSY__GS_BUSY__SHIFT 0x3
+#define SPI_DEBUG_BUSY__VS_BUSY_MASK 0x10
+#define SPI_DEBUG_BUSY__VS_BUSY__SHIFT 0x4
+#define SPI_DEBUG_BUSY__PS0_BUSY_MASK 0x20
+#define SPI_DEBUG_BUSY__PS0_BUSY__SHIFT 0x5
+#define SPI_DEBUG_BUSY__PS1_BUSY_MASK 0x40
+#define SPI_DEBUG_BUSY__PS1_BUSY__SHIFT 0x6
+#define SPI_DEBUG_BUSY__CSG_BUSY_MASK 0x80
+#define SPI_DEBUG_BUSY__CSG_BUSY__SHIFT 0x7
+#define SPI_DEBUG_BUSY__CS0_BUSY_MASK 0x100
+#define SPI_DEBUG_BUSY__CS0_BUSY__SHIFT 0x8
+#define SPI_DEBUG_BUSY__CS1_BUSY_MASK 0x200
+#define SPI_DEBUG_BUSY__CS1_BUSY__SHIFT 0x9
+#define SPI_DEBUG_BUSY__CS2_BUSY_MASK 0x400
+#define SPI_DEBUG_BUSY__CS2_BUSY__SHIFT 0xa
+#define SPI_DEBUG_BUSY__CS3_BUSY_MASK 0x800
+#define SPI_DEBUG_BUSY__CS3_BUSY__SHIFT 0xb
+#define SPI_DEBUG_BUSY__CS4_BUSY_MASK 0x1000
+#define SPI_DEBUG_BUSY__CS4_BUSY__SHIFT 0xc
+#define SPI_DEBUG_BUSY__CS5_BUSY_MASK 0x2000
+#define SPI_DEBUG_BUSY__CS5_BUSY__SHIFT 0xd
+#define SPI_DEBUG_BUSY__CS6_BUSY_MASK 0x4000
+#define SPI_DEBUG_BUSY__CS6_BUSY__SHIFT 0xe
+#define SPI_DEBUG_BUSY__CS7_BUSY_MASK 0x8000
+#define SPI_DEBUG_BUSY__CS7_BUSY__SHIFT 0xf
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY_MASK 0x10000
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY__SHIFT 0x10
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY_MASK 0x20000
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY__SHIFT 0x11
+#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY_MASK 0x40000
+#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY__SHIFT 0x12
+#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY_MASK 0x80000
+#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY__SHIFT 0x13
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY_MASK 0x100000
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY__SHIFT 0x14
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY_MASK 0x200000
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY__SHIFT 0x15
+#define SPI_DEBUG_BUSY__GRBM_BUSY_MASK 0x400000
+#define SPI_DEBUG_BUSY__GRBM_BUSY__SHIFT 0x16
+#define SPI_DEBUG_BUSY__SPIS_BUSY_MASK 0x800000
+#define SPI_DEBUG_BUSY__SPIS_BUSY__SHIFT 0x17
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD_MASK 0xf
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD__SHIFT 0x0
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD_MASK 0xf0
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD__SHIFT 0x4
+#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY_MASK 0xf
+#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY__SHIFT 0x0
+#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY_MASK 0xff0
+#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY__SHIFT 0x4
+#define CGTS_SM_CTRL_REG__MGCG_ENABLED_MASK 0x1000
+#define CGTS_SM_CTRL_REG__MGCG_ENABLED__SHIFT 0xc
+#define CGTS_SM_CTRL_REG__BASE_MODE_MASK 0x10000
+#define CGTS_SM_CTRL_REG__BASE_MODE__SHIFT 0x10
+#define CGTS_SM_CTRL_REG__SM_MODE_MASK 0xe0000
+#define CGTS_SM_CTRL_REG__SM_MODE__SHIFT 0x11
+#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK 0x100000
+#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE__SHIFT 0x14
+#define CGTS_SM_CTRL_REG__OVERRIDE_MASK 0x200000
+#define CGTS_SM_CTRL_REG__OVERRIDE__SHIFT 0x15
+#define CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK 0x400000
+#define CGTS_SM_CTRL_REG__LS_OVERRIDE__SHIFT 0x16
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK 0x800000
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN__SHIFT 0x17
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_MASK 0xff000000
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT 0x18
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL_MASK 0x1f
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL__SHIFT 0x0
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL_MASK 0x1f00
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL__SHIFT 0x8
+#define CGTS_RD_REG__READ_DATA_MASK 0x3fff
+#define CGTS_RD_REG__READ_DATA__SHIFT 0x0
+#define CGTS_TCC_DISABLE__TCC_DISABLE_MASK 0xffff0000
+#define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT 0x10
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK 0xffff0000
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT 0x10
+#define CGTS_CU0_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU0_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU0_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU0_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC__SHIFT 0x10
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU0_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU0_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU0_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU0_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU1_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU1_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU1_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU1_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU1_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU1_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU1_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU1_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU1_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU1_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU1_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU1_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU1_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU1_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU1_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU1_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU1_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU1_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU2_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU2_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU2_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU2_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU2_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU2_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU2_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU2_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU2_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU2_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU2_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU2_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU2_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU2_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU2_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU2_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU2_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU2_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU3_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU3_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU3_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU3_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU3_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU3_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU3_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU3_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU3_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU3_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU3_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU3_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU3_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU3_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU3_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU3_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU3_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU3_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU4_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU4_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU4_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU4_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC__SHIFT 0x10
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU4_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU4_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU4_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU4_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU5_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU5_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU5_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU5_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU5_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU5_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU5_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU5_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU5_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU5_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU5_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU5_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU5_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU5_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU5_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU5_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU5_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU5_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU6_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU6_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU6_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU6_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU6_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU6_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU6_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU6_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU6_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU6_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU6_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU6_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU6_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU6_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU6_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU6_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU6_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU6_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU7_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU7_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU7_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU7_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU7_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU7_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU7_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU7_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU7_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU7_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU7_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU7_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU7_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU7_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU7_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU7_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU7_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU7_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU8_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU8_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU8_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU8_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC__SHIFT 0x10
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU8_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU8_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU8_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU8_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU9_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU9_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU9_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU9_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU9_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU9_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU9_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU9_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU9_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU9_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU9_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU9_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU9_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU9_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU9_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU9_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU9_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU9_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU10_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU10_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU10_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU10_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU10_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU10_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU10_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU10_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU10_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU10_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU10_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU10_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU10_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU10_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU10_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU10_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU10_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU10_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU11_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU11_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU11_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU11_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU11_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU11_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU11_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU11_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU11_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU11_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU11_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU11_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU11_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU11_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU11_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU11_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU11_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU11_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU12_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU12_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU12_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU12_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC__SHIFT 0x10
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU12_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU12_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU12_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU12_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU13_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU13_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU13_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU13_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU13_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU13_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU13_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU13_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU13_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU13_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU13_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU13_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU13_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU13_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU13_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU13_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU13_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU13_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU14_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU14_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU14_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU14_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU14_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU14_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU14_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU14_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU14_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU14_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU14_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU14_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU14_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU14_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU14_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU14_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU14_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU14_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU15_SP0_CTRL_REG__SP00_MASK 0x7f
+#define CGTS_CU15_SP0_CTRL_REG__SP00__SHIFT 0x0
+#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80
+#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7
+#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU15_SP0_CTRL_REG__SP01_MASK 0x7f0000
+#define CGTS_CU15_SP0_CTRL_REG__SP01__SHIFT 0x10
+#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000
+#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17
+#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_MASK 0x7f
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU15_TA_CTRL_REG__TA_MASK 0x7f
+#define CGTS_CU15_TA_CTRL_REG__TA__SHIFT 0x0
+#define CGTS_CU15_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80
+#define CGTS_CU15_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7
+#define CGTS_CU15_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU15_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU15_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU15_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU15_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU15_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU15_SP1_CTRL_REG__SP10_MASK 0x7f
+#define CGTS_CU15_SP1_CTRL_REG__SP10__SHIFT 0x0
+#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80
+#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7
+#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU15_SP1_CTRL_REG__SP11_MASK 0x7f0000
+#define CGTS_CU15_SP1_CTRL_REG__SP11__SHIFT 0x10
+#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000
+#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17
+#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_MASK 0x7f
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD__SHIFT 0x0
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP__SHIFT 0x10
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b
+#define CGTT_SPI_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_SPI_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST_MASK 0xfc0000
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT 0x12
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE_MASK 0x1000000
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT 0x18
+#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE_MASK 0x4000000
+#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE__SHIFT 0x1a
+#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE_MASK 0x8000000
+#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE__SHIFT 0x1b
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE_MASK 0x10000000
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE__SHIFT 0x1c
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE_MASK 0x20000000
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE__SHIFT 0x1d
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE_MASK 0x40000000
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE__SHIFT 0x1e
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define CGTT_PC_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_PC_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST_MASK 0xfc0000
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT 0x12
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE_MASK 0x1000000
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT 0x18
+#define CGTT_PC_CLK_CTRL__BACK_CLK_ON_OVERRIDE_MASK 0x2000000
+#define CGTT_PC_CLK_CTRL__BACK_CLK_ON_OVERRIDE__SHIFT 0x19
+#define CGTT_PC_CLK_CTRL__FRONT_CLK_ON_OVERRIDE_MASK 0x4000000
+#define CGTT_PC_CLK_CTRL__FRONT_CLK_ON_OVERRIDE__SHIFT 0x1a
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE_MASK 0x8000000
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x1b
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x1c
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x1d
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x1e
+#define CGTT_PC_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_PC_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define CGTT_BCI_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_BCI_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_BCI_CLK_CTRL__RESERVED_MASK 0xfff000
+#define CGTT_BCI_CLK_CTRL__RESERVED__SHIFT 0xc
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE_MASK 0x1000000
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE__SHIFT 0x18
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE_MASK 0x2000000
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE__SHIFT 0x19
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE_MASK 0x4000000
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE__SHIFT 0x1a
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE_MASK 0x8000000
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x1b
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x1c
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x1d
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x1e
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD_MASK 0xf
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD__SHIFT 0x0
+#define SPI_WF_LIFETIME_CNTL__EN_MASK 0x10
+#define SPI_WF_LIFETIME_CNTL__EN__SHIFT 0x4
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN_MASK 0x80000000
+#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_1__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_1__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_3__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_3__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_5__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_5__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_8__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_8__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_10__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_10__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_12__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_12__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT__SHIFT 0x0
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT_MASK 0x80000000
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT__SHIFT 0x1f
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE_MASK 0x7fffffff
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE__SHIFT 0x0
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN_MASK 0x80000000
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN__SHIFT 0x1f
+#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY_MASK 0x1
+#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY__SHIFT 0x0
+#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY_MASK 0x2
+#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY__SHIFT 0x1
+#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY_MASK 0x4
+#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY__SHIFT 0x2
+#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY_MASK 0x8
+#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY__SHIFT 0x3
+#define SPI_SLAVE_DEBUG_BUSY__VS_VTX_BUSY_MASK 0x10
+#define SPI_SLAVE_DEBUG_BUSY__VS_VTX_BUSY__SHIFT 0x4
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY_MASK 0x20
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY__SHIFT 0x5
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY_MASK 0x40
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY__SHIFT 0x6
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC10_BUSY_MASK 0x80
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC10_BUSY__SHIFT 0x7
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC11_BUSY_MASK 0x100
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC11_BUSY__SHIFT 0x8
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY_MASK 0x200
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY__SHIFT 0x9
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY_MASK 0x400
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY__SHIFT 0xa
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC02_BUSY_MASK 0x800
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC02_BUSY__SHIFT 0xb
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC03_BUSY_MASK 0x1000
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC03_BUSY__SHIFT 0xc
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC10_BUSY_MASK 0x2000
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC10_BUSY__SHIFT 0xd
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC11_BUSY_MASK 0x4000
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC11_BUSY__SHIFT 0xe
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC12_BUSY_MASK 0x8000
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC12_BUSY__SHIFT 0xf
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC13_BUSY_MASK 0x10000
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC13_BUSY__SHIFT 0x10
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER0_BUSY_MASK 0x20000
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER0_BUSY__SHIFT 0x11
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER1_BUSY_MASK 0x40000
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER1_BUSY__SHIFT 0x12
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC0_BUSY_MASK 0x80000
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC0_BUSY__SHIFT 0x13
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC1_BUSY_MASK 0x100000
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC1_BUSY__SHIFT 0x14
+#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY_MASK 0x200000
+#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY__SHIFT 0x15
+#define SPI_SLAVE_DEBUG_BUSY__SAVE_CTX_BUSY_MASK 0x400000
+#define SPI_SLAVE_DEBUG_BUSY__SAVE_CTX_BUSY__SHIFT 0x16
+#define SPI_LB_CTR_CTRL__LOAD_MASK 0x1
+#define SPI_LB_CTR_CTRL__LOAD__SHIFT 0x0
+#define SPI_LB_CU_MASK__CU_MASK_MASK 0xffff
+#define SPI_LB_CU_MASK__CU_MASK__SHIFT 0x0
+#define SPI_LB_DATA_REG__CNT_DATA_MASK 0xffffffff
+#define SPI_LB_DATA_REG__CNT_DATA__SHIFT 0x0
+#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK_MASK 0xffff
+#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK__SHIFT 0x0
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS_MASK 0xff
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS__SHIFT 0x0
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS_MASK 0xff00
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS__SHIFT 0x8
+#define SPI_GDS_CREDITS__UNUSED_MASK 0xffff0000
+#define SPI_GDS_CREDITS__UNUSED__SHIFT 0x10
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE_MASK 0xffff
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE__SHIFT 0x0
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE_MASK 0xffff0000
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE__SHIFT 0x10
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE_MASK 0xffff
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE__SHIFT 0x0
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE_MASK 0xffff0000
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE__SHIFT 0x10
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE_MASK 0xffffffff
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT__SHIFT 0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT_MASK 0x7ff
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT__SHIFT 0x0
+#define BCI_DEBUG_READ__DATA_MASK 0xffffff
+#define BCI_DEBUG_READ__DATA__SHIFT 0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK 0xffffffff
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT 0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK 0xff
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT 0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK 0xffffffff
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT 0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK 0xff
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT 0x0
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK 0x3f
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT 0x0
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK 0x3c0
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT 0x6
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK 0xffffffff
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT 0x0
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK 0xff
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT 0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK 0xffffffff
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT 0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK 0xff
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT 0x0
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK 0x3f
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT 0x0
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK 0x3c0
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT 0x6
+#define SPI_SHADER_TBA_LO_PS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TBA_LO_PS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TBA_HI_PS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TBA_HI_PS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_LO_PS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TMA_LO_PS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_HI_PS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TMA_HI_PS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS_MASK 0x3f
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS_MASK 0x3c0
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY_MASK 0xc00
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY__SHIFT 0xa
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE_MASK 0xff000
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE__SHIFT 0xc
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV_MASK 0x100000
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP_MASK 0x200000
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP__SHIFT 0x15
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE_MASK 0x400000
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE_MASK 0x800000
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE__SHIFT 0x17
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE_MASK 0x1000000
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE__SHIFT 0x18
+#define SPI_SHADER_PGM_RSRC1_PS__CACHE_CTL_MASK 0xe000000
+#define SPI_SHADER_PGM_RSRC1_PS__CACHE_CTL__SHIFT 0x19
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER_MASK 0x10000000
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER__SHIFT 0x1c
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN_MASK 0x80
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE_MASK 0xff00
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE__SHIFT 0x8
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN_MASK 0x1ff0000
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN__SHIFT 0x10
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN_MASK 0xffff
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT_MASK 0x3f0000
+#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT__SHIFT 0x10
+#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD_MASK 0x3c00000
+#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD__SHIFT 0x16
+#define SPI_SHADER_USER_DATA_PS_0__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_0__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_1__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_1__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_2__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_2__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_3__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_3__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_4__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_4__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_5__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_5__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_6__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_6__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_7__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_7__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_8__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_8__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_9__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_9__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_10__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_10__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_11__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_11__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_12__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_12__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_13__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_13__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_14__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_14__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_PS_15__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_PS_15__DATA__SHIFT 0x0
+#define SPI_SHADER_TBA_LO_VS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TBA_LO_VS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TBA_HI_VS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TBA_HI_VS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_LO_VS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TMA_LO_VS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_HI_VS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TMA_HI_VS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_LO_VS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_PGM_LO_VS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_HI_VS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_PGM_HI_VS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_VS__VGPRS_MASK 0x3f
+#define SPI_SHADER_PGM_RSRC1_VS__VGPRS__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_VS__SGPRS_MASK 0x3c0
+#define SPI_SHADER_PGM_RSRC1_VS__SGPRS__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY_MASK 0xc00
+#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY__SHIFT 0xa
+#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE_MASK 0xff000
+#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE__SHIFT 0xc
+#define SPI_SHADER_PGM_RSRC1_VS__PRIV_MASK 0x100000
+#define SPI_SHADER_PGM_RSRC1_VS__PRIV__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP_MASK 0x200000
+#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP__SHIFT 0x15
+#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE_MASK 0x400000
+#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE_MASK 0x800000
+#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE__SHIFT 0x17
+#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT_MASK 0x3000000
+#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT__SHIFT 0x18
+#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE_MASK 0x4000000
+#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE__SHIFT 0x1a
+#define SPI_SHADER_PGM_RSRC1_VS__CACHE_CTL_MASK 0x38000000
+#define SPI_SHADER_PGM_RSRC1_VS__CACHE_CTL__SHIFT 0x1b
+#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER_MASK 0x40000000
+#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER__SHIFT 0x1e
+#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN_MASK 0x80
+#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN_MASK 0x100
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN__SHIFT 0x8
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN_MASK 0x200
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN__SHIFT 0x9
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN_MASK 0x400
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN__SHIFT 0xa
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN_MASK 0x800
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN__SHIFT 0xb
+#define SPI_SHADER_PGM_RSRC2_VS__SO_EN_MASK 0x1000
+#define SPI_SHADER_PGM_RSRC2_VS__SO_EN__SHIFT 0xc
+#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN_MASK 0x3fe000
+#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN__SHIFT 0xd
+#define SPI_SHADER_PGM_RSRC2_VS__DISPATCH_DRAW_EN_MASK 0x1000000
+#define SPI_SHADER_PGM_RSRC2_VS__DISPATCH_DRAW_EN__SHIFT 0x18
+#define SPI_SHADER_PGM_RSRC3_VS__CU_EN_MASK 0xffff
+#define SPI_SHADER_PGM_RSRC3_VS__CU_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT_MASK 0x3f0000
+#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT__SHIFT 0x10
+#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD_MASK 0x3c00000
+#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD__SHIFT 0x16
+#define SPI_SHADER_LATE_ALLOC_VS__LIMIT_MASK 0x3f
+#define SPI_SHADER_LATE_ALLOC_VS__LIMIT__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_0__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_0__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_1__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_1__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_2__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_2__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_3__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_3__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_4__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_4__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_5__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_5__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_6__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_6__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_7__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_7__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_8__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_8__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_9__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_9__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_10__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_10__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_11__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_11__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_12__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_12__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_13__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_13__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_14__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_14__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_VS_15__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_VS_15__DATA__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_ES_VS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_ES_VS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_ES_VS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_ES_VS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_ES_VS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_ES_VS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_ES_VS__OC_LDS_EN_MASK 0x80
+#define SPI_SHADER_PGM_RSRC2_ES_VS__OC_LDS_EN__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_ES_VS__EXCP_EN_MASK 0x1ff00
+#define SPI_SHADER_PGM_RSRC2_ES_VS__EXCP_EN__SHIFT 0x8
+#define SPI_SHADER_PGM_RSRC2_ES_VS__LDS_SIZE_MASK 0x1ff00000
+#define SPI_SHADER_PGM_RSRC2_ES_VS__LDS_SIZE__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC2_LS_VS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_LS_VS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_LS_VS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_LS_VS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_LS_VS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_LS_VS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_LS_VS__LDS_SIZE_MASK 0xff80
+#define SPI_SHADER_PGM_RSRC2_LS_VS__LDS_SIZE__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_LS_VS__EXCP_EN_MASK 0x1ff0000
+#define SPI_SHADER_PGM_RSRC2_LS_VS__EXCP_EN__SHIFT 0x10
+#define SPI_SHADER_TBA_LO_GS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TBA_LO_GS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TBA_HI_GS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TBA_HI_GS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_LO_GS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TMA_LO_GS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_HI_GS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TMA_HI_GS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS_MASK 0x3f
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS_MASK 0x3c0
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY_MASK 0xc00
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY__SHIFT 0xa
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE_MASK 0xff000
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE__SHIFT 0xc
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV_MASK 0x100000
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP_MASK 0x200000
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP__SHIFT 0x15
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE_MASK 0x400000
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE_MASK 0x800000
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE__SHIFT 0x17
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE_MASK 0x1000000
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE__SHIFT 0x18
+#define SPI_SHADER_PGM_RSRC1_GS__CACHE_CTL_MASK 0xe000000
+#define SPI_SHADER_PGM_RSRC1_GS__CACHE_CTL__SHIFT 0x19
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER_MASK 0x10000000
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER__SHIFT 0x1c
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN_MASK 0xff80
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN_MASK 0xffff
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT_MASK 0x3f0000
+#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT__SHIFT 0x10
+#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD_MASK 0x3c00000
+#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC3_GS__GROUP_FIFO_DEPTH_MASK 0xfc000000
+#define SPI_SHADER_PGM_RSRC3_GS__GROUP_FIFO_DEPTH__SHIFT 0x1a
+#define SPI_SHADER_USER_DATA_GS_0__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_0__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_1__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_1__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_2__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_2__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_3__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_3__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_4__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_4__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_5__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_5__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_6__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_6__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_7__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_7__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_8__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_8__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_9__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_9__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_10__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_10__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_11__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_11__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_12__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_12__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_13__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_13__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_14__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_14__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_GS_15__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_GS_15__DATA__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_ES_GS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_ES_GS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_ES_GS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_ES_GS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_ES_GS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_ES_GS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_ES_GS__OC_LDS_EN_MASK 0x80
+#define SPI_SHADER_PGM_RSRC2_ES_GS__OC_LDS_EN__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_ES_GS__EXCP_EN_MASK 0x1ff00
+#define SPI_SHADER_PGM_RSRC2_ES_GS__EXCP_EN__SHIFT 0x8
+#define SPI_SHADER_PGM_RSRC2_ES_GS__LDS_SIZE_MASK 0x1ff00000
+#define SPI_SHADER_PGM_RSRC2_ES_GS__LDS_SIZE__SHIFT 0x14
+#define SPI_SHADER_TBA_LO_ES__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TBA_LO_ES__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TBA_HI_ES__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TBA_HI_ES__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_LO_ES__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TMA_LO_ES__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_HI_ES__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TMA_HI_ES__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE_MASK 0xff
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_ES__VGPRS_MASK 0x3f
+#define SPI_SHADER_PGM_RSRC1_ES__VGPRS__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_ES__SGPRS_MASK 0x3c0
+#define SPI_SHADER_PGM_RSRC1_ES__SGPRS__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC1_ES__PRIORITY_MASK 0xc00
+#define SPI_SHADER_PGM_RSRC1_ES__PRIORITY__SHIFT 0xa
+#define SPI_SHADER_PGM_RSRC1_ES__FLOAT_MODE_MASK 0xff000
+#define SPI_SHADER_PGM_RSRC1_ES__FLOAT_MODE__SHIFT 0xc
+#define SPI_SHADER_PGM_RSRC1_ES__PRIV_MASK 0x100000
+#define SPI_SHADER_PGM_RSRC1_ES__PRIV__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC1_ES__DX10_CLAMP_MASK 0x200000
+#define SPI_SHADER_PGM_RSRC1_ES__DX10_CLAMP__SHIFT 0x15
+#define SPI_SHADER_PGM_RSRC1_ES__DEBUG_MODE_MASK 0x400000
+#define SPI_SHADER_PGM_RSRC1_ES__DEBUG_MODE__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC1_ES__IEEE_MODE_MASK 0x800000
+#define SPI_SHADER_PGM_RSRC1_ES__IEEE_MODE__SHIFT 0x17
+#define SPI_SHADER_PGM_RSRC1_ES__VGPR_COMP_CNT_MASK 0x3000000
+#define SPI_SHADER_PGM_RSRC1_ES__VGPR_COMP_CNT__SHIFT 0x18
+#define SPI_SHADER_PGM_RSRC1_ES__CU_GROUP_ENABLE_MASK 0x4000000
+#define SPI_SHADER_PGM_RSRC1_ES__CU_GROUP_ENABLE__SHIFT 0x1a
+#define SPI_SHADER_PGM_RSRC1_ES__CACHE_CTL_MASK 0x38000000
+#define SPI_SHADER_PGM_RSRC1_ES__CACHE_CTL__SHIFT 0x1b
+#define SPI_SHADER_PGM_RSRC1_ES__CDBG_USER_MASK 0x40000000
+#define SPI_SHADER_PGM_RSRC1_ES__CDBG_USER__SHIFT 0x1e
+#define SPI_SHADER_PGM_RSRC2_ES__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_ES__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_ES__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_ES__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_ES__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_ES__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_ES__OC_LDS_EN_MASK 0x80
+#define SPI_SHADER_PGM_RSRC2_ES__OC_LDS_EN__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_ES__EXCP_EN_MASK 0x1ff00
+#define SPI_SHADER_PGM_RSRC2_ES__EXCP_EN__SHIFT 0x8
+#define SPI_SHADER_PGM_RSRC2_ES__LDS_SIZE_MASK 0x1ff00000
+#define SPI_SHADER_PGM_RSRC2_ES__LDS_SIZE__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC3_ES__CU_EN_MASK 0xffff
+#define SPI_SHADER_PGM_RSRC3_ES__CU_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC3_ES__WAVE_LIMIT_MASK 0x3f0000
+#define SPI_SHADER_PGM_RSRC3_ES__WAVE_LIMIT__SHIFT 0x10
+#define SPI_SHADER_PGM_RSRC3_ES__LOCK_LOW_THRESHOLD_MASK 0x3c00000
+#define SPI_SHADER_PGM_RSRC3_ES__LOCK_LOW_THRESHOLD__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC3_ES__GROUP_FIFO_DEPTH_MASK 0xfc000000
+#define SPI_SHADER_PGM_RSRC3_ES__GROUP_FIFO_DEPTH__SHIFT 0x1a
+#define SPI_SHADER_USER_DATA_ES_0__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_0__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_1__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_1__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_2__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_2__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_3__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_3__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_4__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_4__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_5__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_5__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_6__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_6__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_7__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_7__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_8__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_8__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_9__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_9__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_10__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_10__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_11__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_11__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_12__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_12__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_13__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_13__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_14__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_14__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_ES_15__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_ES_15__DATA__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_LS_ES__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_LS_ES__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_LS_ES__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_LS_ES__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_LS_ES__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_LS_ES__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_LS_ES__LDS_SIZE_MASK 0xff80
+#define SPI_SHADER_PGM_RSRC2_LS_ES__LDS_SIZE__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_LS_ES__EXCP_EN_MASK 0x1ff0000
+#define SPI_SHADER_PGM_RSRC2_LS_ES__EXCP_EN__SHIFT 0x10
+#define SPI_SHADER_TBA_LO_HS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TBA_LO_HS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TBA_HI_HS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TBA_HI_HS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_LO_HS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TMA_LO_HS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_HI_HS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TMA_HI_HS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS_MASK 0x3f
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS_MASK 0x3c0
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY_MASK 0xc00
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY__SHIFT 0xa
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE_MASK 0xff000
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE__SHIFT 0xc
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV_MASK 0x100000
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP_MASK 0x200000
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP__SHIFT 0x15
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE_MASK 0x400000
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE_MASK 0x800000
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE__SHIFT 0x17
+#define SPI_SHADER_PGM_RSRC1_HS__CACHE_CTL_MASK 0x7000000
+#define SPI_SHADER_PGM_RSRC1_HS__CACHE_CTL__SHIFT 0x18
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER_MASK 0x8000000
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER__SHIFT 0x1b
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN_MASK 0x80
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN_MASK 0x100
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN__SHIFT 0x8
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN_MASK 0x3fe00
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN__SHIFT 0x9
+#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT_MASK 0x3f
+#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD_MASK 0x3c0
+#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC3_HS__GROUP_FIFO_DEPTH_MASK 0xfc00
+#define SPI_SHADER_PGM_RSRC3_HS__GROUP_FIFO_DEPTH__SHIFT 0xa
+#define SPI_SHADER_USER_DATA_HS_0__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_0__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_1__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_1__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_2__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_2__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_3__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_3__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_4__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_4__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_5__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_5__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_6__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_6__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_7__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_7__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_8__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_8__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_9__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_9__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_10__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_10__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_11__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_11__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_12__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_12__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_13__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_13__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_14__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_14__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_HS_15__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_HS_15__DATA__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_LS_HS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_LS_HS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_LS_HS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_LS_HS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_LS_HS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_LS_HS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_LS_HS__LDS_SIZE_MASK 0xff80
+#define SPI_SHADER_PGM_RSRC2_LS_HS__LDS_SIZE__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_LS_HS__EXCP_EN_MASK 0x1ff0000
+#define SPI_SHADER_PGM_RSRC2_LS_HS__EXCP_EN__SHIFT 0x10
+#define SPI_SHADER_TBA_LO_LS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TBA_LO_LS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TBA_HI_LS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TBA_HI_LS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_LO_LS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_TMA_LO_LS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_TMA_HI_LS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_TMA_HI_LS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE_MASK 0xffffffff
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE_MASK 0xff
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_LS__VGPRS_MASK 0x3f
+#define SPI_SHADER_PGM_RSRC1_LS__VGPRS__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC1_LS__SGPRS_MASK 0x3c0
+#define SPI_SHADER_PGM_RSRC1_LS__SGPRS__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC1_LS__PRIORITY_MASK 0xc00
+#define SPI_SHADER_PGM_RSRC1_LS__PRIORITY__SHIFT 0xa
+#define SPI_SHADER_PGM_RSRC1_LS__FLOAT_MODE_MASK 0xff000
+#define SPI_SHADER_PGM_RSRC1_LS__FLOAT_MODE__SHIFT 0xc
+#define SPI_SHADER_PGM_RSRC1_LS__PRIV_MASK 0x100000
+#define SPI_SHADER_PGM_RSRC1_LS__PRIV__SHIFT 0x14
+#define SPI_SHADER_PGM_RSRC1_LS__DX10_CLAMP_MASK 0x200000
+#define SPI_SHADER_PGM_RSRC1_LS__DX10_CLAMP__SHIFT 0x15
+#define SPI_SHADER_PGM_RSRC1_LS__DEBUG_MODE_MASK 0x400000
+#define SPI_SHADER_PGM_RSRC1_LS__DEBUG_MODE__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC1_LS__IEEE_MODE_MASK 0x800000
+#define SPI_SHADER_PGM_RSRC1_LS__IEEE_MODE__SHIFT 0x17
+#define SPI_SHADER_PGM_RSRC1_LS__VGPR_COMP_CNT_MASK 0x3000000
+#define SPI_SHADER_PGM_RSRC1_LS__VGPR_COMP_CNT__SHIFT 0x18
+#define SPI_SHADER_PGM_RSRC1_LS__CACHE_CTL_MASK 0x1c000000
+#define SPI_SHADER_PGM_RSRC1_LS__CACHE_CTL__SHIFT 0x1a
+#define SPI_SHADER_PGM_RSRC1_LS__CDBG_USER_MASK 0x20000000
+#define SPI_SHADER_PGM_RSRC1_LS__CDBG_USER__SHIFT 0x1d
+#define SPI_SHADER_PGM_RSRC2_LS__SCRATCH_EN_MASK 0x1
+#define SPI_SHADER_PGM_RSRC2_LS__SCRATCH_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC2_LS__USER_SGPR_MASK 0x3e
+#define SPI_SHADER_PGM_RSRC2_LS__USER_SGPR__SHIFT 0x1
+#define SPI_SHADER_PGM_RSRC2_LS__TRAP_PRESENT_MASK 0x40
+#define SPI_SHADER_PGM_RSRC2_LS__TRAP_PRESENT__SHIFT 0x6
+#define SPI_SHADER_PGM_RSRC2_LS__LDS_SIZE_MASK 0xff80
+#define SPI_SHADER_PGM_RSRC2_LS__LDS_SIZE__SHIFT 0x7
+#define SPI_SHADER_PGM_RSRC2_LS__EXCP_EN_MASK 0x1ff0000
+#define SPI_SHADER_PGM_RSRC2_LS__EXCP_EN__SHIFT 0x10
+#define SPI_SHADER_PGM_RSRC3_LS__CU_EN_MASK 0xffff
+#define SPI_SHADER_PGM_RSRC3_LS__CU_EN__SHIFT 0x0
+#define SPI_SHADER_PGM_RSRC3_LS__WAVE_LIMIT_MASK 0x3f0000
+#define SPI_SHADER_PGM_RSRC3_LS__WAVE_LIMIT__SHIFT 0x10
+#define SPI_SHADER_PGM_RSRC3_LS__LOCK_LOW_THRESHOLD_MASK 0x3c00000
+#define SPI_SHADER_PGM_RSRC3_LS__LOCK_LOW_THRESHOLD__SHIFT 0x16
+#define SPI_SHADER_PGM_RSRC3_LS__GROUP_FIFO_DEPTH_MASK 0xfc000000
+#define SPI_SHADER_PGM_RSRC3_LS__GROUP_FIFO_DEPTH__SHIFT 0x1a
+#define SPI_SHADER_USER_DATA_LS_0__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_0__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_1__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_1__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_2__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_2__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_3__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_3__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_4__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_4__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_5__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_5__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_6__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_6__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_7__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_7__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_8__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_8__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_9__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_9__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_10__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_10__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_11__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_11__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_12__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_12__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_13__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_13__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_14__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_14__DATA__SHIFT 0x0
+#define SPI_SHADER_USER_DATA_LS_15__DATA_MASK 0xffffffff
+#define SPI_SHADER_USER_DATA_LS_15__DATA__SHIFT 0x0
+#define SQ_CONFIG__UNUSED_MASK 0xff
+#define SQ_CONFIG__UNUSED__SHIFT 0x0
+#define SQ_CONFIG__DEBUG_EN_MASK 0x100
+#define SQ_CONFIG__DEBUG_EN__SHIFT 0x8
+#define SQ_CONFIG__DEBUG_SINGLE_MEMOP_MASK 0x200
+#define SQ_CONFIG__DEBUG_SINGLE_MEMOP__SHIFT 0x9
+#define SQ_CONFIG__DEBUG_ONE_INST_CLAUSE_MASK 0x400
+#define SQ_CONFIG__DEBUG_ONE_INST_CLAUSE__SHIFT 0xa
+#define SQ_CONFIG__EARLY_TA_DONE_DISABLE_MASK 0x1000
+#define SQ_CONFIG__EARLY_TA_DONE_DISABLE__SHIFT 0xc
+#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE_MASK 0x2000
+#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE__SHIFT 0xd
+#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE_MASK 0x4000
+#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE__SHIFT 0xe
+#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE_MASK 0x8000
+#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE__SHIFT 0xf
+#define SQ_CONFIG__DISABLE_VMEM_SOFT_CLAUSE_MASK 0x10000
+#define SQ_CONFIG__DISABLE_VMEM_SOFT_CLAUSE__SHIFT 0x10
+#define SQ_CONFIG__DISABLE_SMEM_SOFT_CLAUSE_MASK 0x20000
+#define SQ_CONFIG__DISABLE_SMEM_SOFT_CLAUSE__SHIFT 0x11
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_VS_MASK 0x40000
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_VS__SHIFT 0x12
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_VS_MASK 0x180000
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_VS__SHIFT 0x13
+#define SQ_CONFIG__REPLAY_SLEEP_CNT_MASK 0x1e00000
+#define SQ_CONFIG__REPLAY_SLEEP_CNT__SHIFT 0x15
+#define SQC_CONFIG__INST_CACHE_SIZE_MASK 0x3
+#define SQC_CONFIG__INST_CACHE_SIZE__SHIFT 0x0
+#define SQC_CONFIG__DATA_CACHE_SIZE_MASK 0xc
+#define SQC_CONFIG__DATA_CACHE_SIZE__SHIFT 0x2
+#define SQC_CONFIG__MISS_FIFO_DEPTH_MASK 0x30
+#define SQC_CONFIG__MISS_FIFO_DEPTH__SHIFT 0x4
+#define SQC_CONFIG__HIT_FIFO_DEPTH_MASK 0x40
+#define SQC_CONFIG__HIT_FIFO_DEPTH__SHIFT 0x6
+#define SQC_CONFIG__FORCE_ALWAYS_MISS_MASK 0x80
+#define SQC_CONFIG__FORCE_ALWAYS_MISS__SHIFT 0x7
+#define SQC_CONFIG__FORCE_IN_ORDER_MASK 0x100
+#define SQC_CONFIG__FORCE_IN_ORDER__SHIFT 0x8
+#define SQC_CONFIG__IDENTITY_HASH_BANK_MASK 0x200
+#define SQC_CONFIG__IDENTITY_HASH_BANK__SHIFT 0x9
+#define SQC_CONFIG__IDENTITY_HASH_SET_MASK 0x400
+#define SQC_CONFIG__IDENTITY_HASH_SET__SHIFT 0xa
+#define SQC_CONFIG__PER_VMID_INV_DISABLE_MASK 0x800
+#define SQC_CONFIG__PER_VMID_INV_DISABLE__SHIFT 0xb
+#define SQC_CONFIG__EVICT_LRU_MASK 0x3000
+#define SQC_CONFIG__EVICT_LRU__SHIFT 0xc
+#define SQC_CONFIG__FORCE_2_BANK_MASK 0x4000
+#define SQC_CONFIG__FORCE_2_BANK__SHIFT 0xe
+#define SQC_CONFIG__FORCE_1_BANK_MASK 0x8000
+#define SQC_CONFIG__FORCE_1_BANK__SHIFT 0xf
+#define SQC_CONFIG__LS_DISABLE_CLOCKS_MASK 0xff0000
+#define SQC_CONFIG__LS_DISABLE_CLOCKS__SHIFT 0x10
+#define SQC_CACHES__TARGET_INST_MASK 0x1
+#define SQC_CACHES__TARGET_INST__SHIFT 0x0
+#define SQC_CACHES__TARGET_DATA_MASK 0x2
+#define SQC_CACHES__TARGET_DATA__SHIFT 0x1
+#define SQC_CACHES__INVALIDATE_MASK 0x4
+#define SQC_CACHES__INVALIDATE__SHIFT 0x2
+#define SQC_CACHES__WRITEBACK_MASK 0x8
+#define SQC_CACHES__WRITEBACK__SHIFT 0x3
+#define SQC_CACHES__VOL_MASK 0x10
+#define SQC_CACHES__VOL__SHIFT 0x4
+#define SQC_CACHES__COMPLETE_MASK 0x10000
+#define SQC_CACHES__COMPLETE__SHIFT 0x10
+#define SQC_WRITEBACK__DWB_MASK 0x1
+#define SQC_WRITEBACK__DWB__SHIFT 0x0
+#define SQC_WRITEBACK__DIRTY_MASK 0x2
+#define SQC_WRITEBACK__DIRTY__SHIFT 0x1
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKA_MASK 0x3
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKA__SHIFT 0x0
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKA_MASK 0x4
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKA__SHIFT 0x2
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKB_MASK 0x18
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKB__SHIFT 0x3
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKB_MASK 0x20
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKB__SHIFT 0x5
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKC_MASK 0xc0
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKC__SHIFT 0x6
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKC_MASK 0x100
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKC__SHIFT 0x8
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKD_MASK 0x600
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_BANKD__SHIFT 0x9
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKD_MASK 0x800
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_BANKD__SHIFT 0xb
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_GATCL1_MASK 0x3000
+#define SQC_DSM_CNTL__SEL_DATA_ICACHE_GATCL1__SHIFT 0xc
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_GATCL1_MASK 0x4000
+#define SQC_DSM_CNTL__EN_SINGLE_WR_ICACHE_GATCL1__SHIFT 0xe
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKA_MASK 0x18000
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKA__SHIFT 0xf
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKA_MASK 0x20000
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKA__SHIFT 0x11
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKB_MASK 0xc0000
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKB__SHIFT 0x12
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKB_MASK 0x100000
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKB__SHIFT 0x14
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKC_MASK 0x600000
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKC__SHIFT 0x15
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKC_MASK 0x800000
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKC__SHIFT 0x17
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKD_MASK 0x3000000
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_BANKD__SHIFT 0x18
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKD_MASK 0x4000000
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_BANKD__SHIFT 0x1a
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_GATCL1_MASK 0x18000000
+#define SQC_DSM_CNTL__SEL_DATA_DCACHE_GATCL1__SHIFT 0x1b
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_GATCL1_MASK 0x20000000
+#define SQC_DSM_CNTL__EN_SINGLE_WR_DCACHE_GATCL1__SHIFT 0x1d
+#define SQ_RANDOM_WAVE_PRI__RET_MASK 0x7f
+#define SQ_RANDOM_WAVE_PRI__RET__SHIFT 0x0
+#define SQ_RANDOM_WAVE_PRI__RUI_MASK 0x380
+#define SQ_RANDOM_WAVE_PRI__RUI__SHIFT 0x7
+#define SQ_RANDOM_WAVE_PRI__RNG_MASK 0x1ffc00
+#define SQ_RANDOM_WAVE_PRI__RNG__SHIFT 0xa
+#define SQ_REG_CREDITS__SRBM_CREDITS_MASK 0x3f
+#define SQ_REG_CREDITS__SRBM_CREDITS__SHIFT 0x0
+#define SQ_REG_CREDITS__CMD_CREDITS_MASK 0xf00
+#define SQ_REG_CREDITS__CMD_CREDITS__SHIFT 0x8
+#define SQ_REG_CREDITS__REG_BUSY_MASK 0x10000000
+#define SQ_REG_CREDITS__REG_BUSY__SHIFT 0x1c
+#define SQ_REG_CREDITS__SRBM_OVERFLOW_MASK 0x20000000
+#define SQ_REG_CREDITS__SRBM_OVERFLOW__SHIFT 0x1d
+#define SQ_REG_CREDITS__IMMED_OVERFLOW_MASK 0x40000000
+#define SQ_REG_CREDITS__IMMED_OVERFLOW__SHIFT 0x1e
+#define SQ_REG_CREDITS__CMD_OVERFLOW_MASK 0x80000000
+#define SQ_REG_CREDITS__CMD_OVERFLOW__SHIFT 0x1f
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE_MASK 0xf
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE__SHIFT 0x0
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE_MASK 0xf00
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE__SHIFT 0x8
+#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE_MASK 0x30000
+#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE__SHIFT 0x10
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE_MASK 0xc0000
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE__SHIFT 0x12
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0_MASK 0x1
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0__SHIFT 0x0
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1_MASK 0x2
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1__SHIFT 0x1
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0_MASK 0x4
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0__SHIFT 0x2
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1_MASK 0x8
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1__SHIFT 0x3
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0_MASK 0x100
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0__SHIFT 0x8
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1_MASK 0x200
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1__SHIFT 0x9
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE_MASK 0x400
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE__SHIFT 0xa
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0_MASK 0x10000
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0__SHIFT 0x10
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1_MASK 0x20000
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1__SHIFT 0x11
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01_MASK 0x40000
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01__SHIFT 0x12
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2_MASK 0x80000
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2__SHIFT 0x13
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3_MASK 0x100000
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3__SHIFT 0x14
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23_MASK 0x200000
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23__SHIFT 0x15
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0_MASK 0x1000000
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0__SHIFT 0x18
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1_MASK 0x2000000
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1__SHIFT 0x19
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE_MASK 0x4000000
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE__SHIFT 0x1a
+#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE_MASK 0x6
+#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT 0x1
+#define CC_GC_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE_MASK 0x8
+#define CC_GC_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE__SHIFT 0x3
+#define CC_GC_SHADER_RATE_CONFIG__HALF_LDS_MASK 0x10
+#define CC_GC_SHADER_RATE_CONFIG__HALF_LDS__SHIFT 0x4
+#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE_MASK 0x6
+#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT 0x1
+#define GC_USER_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE_MASK 0x8
+#define GC_USER_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE__SHIFT 0x3
+#define GC_USER_SHADER_RATE_CONFIG__HALF_LDS_MASK 0x10
+#define GC_USER_SHADER_RATE_CONFIG__HALF_LDS__SHIFT 0x4
+#define SQ_INTERRUPT_AUTO_MASK__MASK_MASK 0xffffff
+#define SQ_INTERRUPT_AUTO_MASK__MASK__SHIFT 0x0
+#define SQ_INTERRUPT_MSG_CTRL__STALL_MASK 0x1
+#define SQ_INTERRUPT_MSG_CTRL__STALL__SHIFT 0x0
+#define SQ_PERFCOUNTER_CTRL__PS_EN_MASK 0x1
+#define SQ_PERFCOUNTER_CTRL__PS_EN__SHIFT 0x0
+#define SQ_PERFCOUNTER_CTRL__VS_EN_MASK 0x2
+#define SQ_PERFCOUNTER_CTRL__VS_EN__SHIFT 0x1
+#define SQ_PERFCOUNTER_CTRL__GS_EN_MASK 0x4
+#define SQ_PERFCOUNTER_CTRL__GS_EN__SHIFT 0x2
+#define SQ_PERFCOUNTER_CTRL__ES_EN_MASK 0x8
+#define SQ_PERFCOUNTER_CTRL__ES_EN__SHIFT 0x3
+#define SQ_PERFCOUNTER_CTRL__HS_EN_MASK 0x10
+#define SQ_PERFCOUNTER_CTRL__HS_EN__SHIFT 0x4
+#define SQ_PERFCOUNTER_CTRL__LS_EN_MASK 0x20
+#define SQ_PERFCOUNTER_CTRL__LS_EN__SHIFT 0x5
+#define SQ_PERFCOUNTER_CTRL__CS_EN_MASK 0x40
+#define SQ_PERFCOUNTER_CTRL__CS_EN__SHIFT 0x6
+#define SQ_PERFCOUNTER_CTRL__CNTR_RATE_MASK 0x1f00
+#define SQ_PERFCOUNTER_CTRL__CNTR_RATE__SHIFT 0x8
+#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH_MASK 0x2000
+#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH__SHIFT 0xd
+#define SQ_PERFCOUNTER_MASK__SH0_MASK_MASK 0xffff
+#define SQ_PERFCOUNTER_MASK__SH0_MASK__SHIFT 0x0
+#define SQ_PERFCOUNTER_MASK__SH1_MASK_MASK 0xffff0000
+#define SQ_PERFCOUNTER_MASK__SH1_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN_MASK 0x1
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT 0x0
+#define CC_SQC_BANK_DISABLE__SQC0_BANK_DISABLE_MASK 0xf0000
+#define CC_SQC_BANK_DISABLE__SQC0_BANK_DISABLE__SHIFT 0x10
+#define CC_SQC_BANK_DISABLE__SQC1_BANK_DISABLE_MASK 0xf00000
+#define CC_SQC_BANK_DISABLE__SQC1_BANK_DISABLE__SHIFT 0x14
+#define CC_SQC_BANK_DISABLE__SQC2_BANK_DISABLE_MASK 0xf000000
+#define CC_SQC_BANK_DISABLE__SQC2_BANK_DISABLE__SHIFT 0x18
+#define CC_SQC_BANK_DISABLE__SQC3_BANK_DISABLE_MASK 0xf0000000
+#define CC_SQC_BANK_DISABLE__SQC3_BANK_DISABLE__SHIFT 0x1c
+#define USER_SQC_BANK_DISABLE__SQC0_BANK_DISABLE_MASK 0xf0000
+#define USER_SQC_BANK_DISABLE__SQC0_BANK_DISABLE__SHIFT 0x10
+#define USER_SQC_BANK_DISABLE__SQC1_BANK_DISABLE_MASK 0xf00000
+#define USER_SQC_BANK_DISABLE__SQC1_BANK_DISABLE__SHIFT 0x14
+#define USER_SQC_BANK_DISABLE__SQC2_BANK_DISABLE_MASK 0xf000000
+#define USER_SQC_BANK_DISABLE__SQC2_BANK_DISABLE__SHIFT 0x18
+#define USER_SQC_BANK_DISABLE__SQC3_BANK_DISABLE_MASK 0xf0000000
+#define USER_SQC_BANK_DISABLE__SQC3_BANK_DISABLE__SHIFT 0x1c
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE__SHIFT 0x1c
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL_MASK 0x1ff
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL__SHIFT 0x0
+#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK_MASK 0xf000
+#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK__SHIFT 0xc
+#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK_MASK 0xf0000
+#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK__SHIFT 0x10
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE_MASK 0xf00000
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE__SHIFT 0x14
+#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK_MASK 0xf000000
+#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK__SHIFT 0x18
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE_MASK 0xf0000000
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE__SHIFT 0x1c
+#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SQ_CLK_CTRL__PERFMON_OVERRIDE_MASK 0x20000000
+#define CGTT_SQ_CLK_CTRL__PERFMON_OVERRIDE__SHIFT 0x1d
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE_MASK 0x10000000
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE__SHIFT 0x1c
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE_MASK 0x20000000
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE__SHIFT 0x1d
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0xffff
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x0
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x10
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0xffff
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x0
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x10
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0xffff
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x0
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x10
+#define SQ_POWER_THROTTLE__MIN_POWER_MASK 0x3fff
+#define SQ_POWER_THROTTLE__MIN_POWER__SHIFT 0x0
+#define SQ_POWER_THROTTLE__MAX_POWER_MASK 0x3fff0000
+#define SQ_POWER_THROTTLE__MAX_POWER__SHIFT 0x10
+#define SQ_POWER_THROTTLE__PHASE_OFFSET_MASK 0xc0000000
+#define SQ_POWER_THROTTLE__PHASE_OFFSET__SHIFT 0x1e
+#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA_MASK 0x3fff
+#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA__SHIFT 0x0
+#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000
+#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10
+#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000
+#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b
+#define SQ_POWER_THROTTLE2__USE_REF_CLOCK_MASK 0x80000000
+#define SQ_POWER_THROTTLE2__USE_REF_CLOCK__SHIFT 0x1f
+#define SQ_TIME_HI__TIME_MASK 0xffffffff
+#define SQ_TIME_HI__TIME__SHIFT 0x0
+#define SQ_TIME_LO__TIME_MASK 0xffffffff
+#define SQ_TIME_LO__TIME__SHIFT 0x0
+#define SQ_THREAD_TRACE_BASE__ADDR_MASK 0xffffffff
+#define SQ_THREAD_TRACE_BASE__ADDR__SHIFT 0x0
+#define SQ_THREAD_TRACE_BASE2__ADDR_HI_MASK 0xf
+#define SQ_THREAD_TRACE_BASE2__ADDR_HI__SHIFT 0x0
+#define SQ_THREAD_TRACE_SIZE__SIZE_MASK 0x3fffff
+#define SQ_THREAD_TRACE_SIZE__SIZE__SHIFT 0x0
+#define SQ_THREAD_TRACE_MASK__CU_SEL_MASK 0x1f
+#define SQ_THREAD_TRACE_MASK__CU_SEL__SHIFT 0x0
+#define SQ_THREAD_TRACE_MASK__SH_SEL_MASK 0x20
+#define SQ_THREAD_TRACE_MASK__SH_SEL__SHIFT 0x5
+#define SQ_THREAD_TRACE_MASK__REG_STALL_EN_MASK 0x80
+#define SQ_THREAD_TRACE_MASK__REG_STALL_EN__SHIFT 0x7
+#define SQ_THREAD_TRACE_MASK__SIMD_EN_MASK 0xf00
+#define SQ_THREAD_TRACE_MASK__SIMD_EN__SHIFT 0x8
+#define SQ_THREAD_TRACE_MASK__VM_ID_MASK_MASK 0x3000
+#define SQ_THREAD_TRACE_MASK__VM_ID_MASK__SHIFT 0xc
+#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN_MASK 0x4000
+#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN__SHIFT 0xe
+#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN_MASK 0x8000
+#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN__SHIFT 0xf
+#define SQ_THREAD_TRACE_USERDATA_0__DATA_MASK 0xffffffff
+#define SQ_THREAD_TRACE_USERDATA_0__DATA__SHIFT 0x0
+#define SQ_THREAD_TRACE_USERDATA_1__DATA_MASK 0xffffffff
+#define SQ_THREAD_TRACE_USERDATA_1__DATA__SHIFT 0x0
+#define SQ_THREAD_TRACE_USERDATA_2__DATA_MASK 0xffffffff
+#define SQ_THREAD_TRACE_USERDATA_2__DATA__SHIFT 0x0
+#define SQ_THREAD_TRACE_USERDATA_3__DATA_MASK 0xffffffff
+#define SQ_THREAD_TRACE_USERDATA_3__DATA__SHIFT 0x0
+#define SQ_THREAD_TRACE_MODE__MASK_PS_MASK 0x7
+#define SQ_THREAD_TRACE_MODE__MASK_PS__SHIFT 0x0
+#define SQ_THREAD_TRACE_MODE__MASK_VS_MASK 0x38
+#define SQ_THREAD_TRACE_MODE__MASK_VS__SHIFT 0x3
+#define SQ_THREAD_TRACE_MODE__MASK_GS_MASK 0x1c0
+#define SQ_THREAD_TRACE_MODE__MASK_GS__SHIFT 0x6
+#define SQ_THREAD_TRACE_MODE__MASK_ES_MASK 0xe00
+#define SQ_THREAD_TRACE_MODE__MASK_ES__SHIFT 0x9
+#define SQ_THREAD_TRACE_MODE__MASK_HS_MASK 0x7000
+#define SQ_THREAD_TRACE_MODE__MASK_HS__SHIFT 0xc
+#define SQ_THREAD_TRACE_MODE__MASK_LS_MASK 0x38000
+#define SQ_THREAD_TRACE_MODE__MASK_LS__SHIFT 0xf
+#define SQ_THREAD_TRACE_MODE__MASK_CS_MASK 0x1c0000
+#define SQ_THREAD_TRACE_MODE__MASK_CS__SHIFT 0x12
+#define SQ_THREAD_TRACE_MODE__MODE_MASK 0x600000
+#define SQ_THREAD_TRACE_MODE__MODE__SHIFT 0x15
+#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE_MASK 0x1800000
+#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE__SHIFT 0x17
+#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN_MASK 0x2000000
+#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN__SHIFT 0x19
+#define SQ_THREAD_TRACE_MODE__PRIV_MASK 0x4000000
+#define SQ_THREAD_TRACE_MODE__PRIV__SHIFT 0x1a
+#define SQ_THREAD_TRACE_MODE__ISSUE_MASK_MASK 0x18000000
+#define SQ_THREAD_TRACE_MODE__ISSUE_MASK__SHIFT 0x1b
+#define SQ_THREAD_TRACE_MODE__TEST_MODE_MASK 0x20000000
+#define SQ_THREAD_TRACE_MODE__TEST_MODE__SHIFT 0x1d
+#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN_MASK 0x40000000
+#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN__SHIFT 0x1e
+#define SQ_THREAD_TRACE_MODE__WRAP_MASK 0x80000000
+#define SQ_THREAD_TRACE_MODE__WRAP__SHIFT 0x1f
+#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER_MASK 0x80000000
+#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER__SHIFT 0x1f
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK_MASK 0xffff
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK__SHIFT 0x0
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK_MASK 0xff0000
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK__SHIFT 0x10
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL_MASK 0x1000000
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL__SHIFT 0x18
+#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK_MASK 0xffffffff
+#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK__SHIFT 0x0
+#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK_MASK 0xffff
+#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK__SHIFT 0x0
+#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK_MASK 0xffff0000
+#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK__SHIFT 0x10
+#define SQ_THREAD_TRACE_WPTR__WPTR_MASK 0x3fffffff
+#define SQ_THREAD_TRACE_WPTR__WPTR__SHIFT 0x0
+#define SQ_THREAD_TRACE_WPTR__READ_OFFSET_MASK 0xc0000000
+#define SQ_THREAD_TRACE_WPTR__READ_OFFSET__SHIFT 0x1e
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK 0x3ff
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING__SHIFT 0x0
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK 0x3ff0000
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE__SHIFT 0x10
+#define SQ_THREAD_TRACE_STATUS__NEW_BUF_MASK 0x20000000
+#define SQ_THREAD_TRACE_STATUS__NEW_BUF__SHIFT 0x1d
+#define SQ_THREAD_TRACE_STATUS__BUSY_MASK 0x40000000
+#define SQ_THREAD_TRACE_STATUS__BUSY__SHIFT 0x1e
+#define SQ_THREAD_TRACE_STATUS__FULL_MASK 0x80000000
+#define SQ_THREAD_TRACE_STATUS__FULL__SHIFT 0x1f
+#define SQ_THREAD_TRACE_CNTR__CNTR_MASK 0xffffffff
+#define SQ_THREAD_TRACE_CNTR__CNTR__SHIFT 0x0
+#define SQ_THREAD_TRACE_HIWATER__HIWATER_MASK 0x7
+#define SQ_THREAD_TRACE_HIWATER__HIWATER__SHIFT 0x0
+#define SQ_LB_CTR_CTRL__START_MASK 0x1
+#define SQ_LB_CTR_CTRL__START__SHIFT 0x0
+#define SQ_LB_CTR_CTRL__LOAD_MASK 0x2
+#define SQ_LB_CTR_CTRL__LOAD__SHIFT 0x1
+#define SQ_LB_CTR_CTRL__CLEAR_MASK 0x4
+#define SQ_LB_CTR_CTRL__CLEAR__SHIFT 0x2
+#define SQ_LB_DATA_ALU_CYCLES__DATA_MASK 0xffffffff
+#define SQ_LB_DATA_ALU_CYCLES__DATA__SHIFT 0x0
+#define SQ_LB_DATA_TEX_CYCLES__DATA_MASK 0xffffffff
+#define SQ_LB_DATA_TEX_CYCLES__DATA__SHIFT 0x0
+#define SQ_LB_DATA_ALU_STALLS__DATA_MASK 0xffffffff
+#define SQ_LB_DATA_ALU_STALLS__DATA__SHIFT 0x0
+#define SQ_LB_DATA_TEX_STALLS__DATA_MASK 0xffffffff
+#define SQ_LB_DATA_TEX_STALLS__DATA__SHIFT 0x0
+#define SQC_EDC_CNT__INST_SEC_MASK 0xff
+#define SQC_EDC_CNT__INST_SEC__SHIFT 0x0
+#define SQC_EDC_CNT__INST_DED_MASK 0xff00
+#define SQC_EDC_CNT__INST_DED__SHIFT 0x8
+#define SQC_EDC_CNT__DATA_SEC_MASK 0xff0000
+#define SQC_EDC_CNT__DATA_SEC__SHIFT 0x10
+#define SQC_EDC_CNT__DATA_DED_MASK 0xff000000
+#define SQC_EDC_CNT__DATA_DED__SHIFT 0x18
+#define SQ_EDC_SEC_CNT__LDS_SEC_MASK 0xff
+#define SQ_EDC_SEC_CNT__LDS_SEC__SHIFT 0x0
+#define SQ_EDC_SEC_CNT__SGPR_SEC_MASK 0xff00
+#define SQ_EDC_SEC_CNT__SGPR_SEC__SHIFT 0x8
+#define SQ_EDC_SEC_CNT__VGPR_SEC_MASK 0xff0000
+#define SQ_EDC_SEC_CNT__VGPR_SEC__SHIFT 0x10
+#define SQ_EDC_DED_CNT__LDS_DED_MASK 0xff
+#define SQ_EDC_DED_CNT__LDS_DED__SHIFT 0x0
+#define SQ_EDC_DED_CNT__SGPR_DED_MASK 0xff00
+#define SQ_EDC_DED_CNT__SGPR_DED__SHIFT 0x8
+#define SQ_EDC_DED_CNT__VGPR_DED_MASK 0xff0000
+#define SQ_EDC_DED_CNT__VGPR_DED__SHIFT 0x10
+#define SQ_EDC_INFO__WAVE_ID_MASK 0xf
+#define SQ_EDC_INFO__WAVE_ID__SHIFT 0x0
+#define SQ_EDC_INFO__SIMD_ID_MASK 0x30
+#define SQ_EDC_INFO__SIMD_ID__SHIFT 0x4
+#define SQ_EDC_INFO__SOURCE_MASK 0x1c0
+#define SQ_EDC_INFO__SOURCE__SHIFT 0x6
+#define SQ_EDC_INFO__VM_ID_MASK 0x1e00
+#define SQ_EDC_INFO__VM_ID__SHIFT 0x9
+#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS_MASK 0xffffffff
+#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS__SHIFT 0x0
+#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI_MASK 0xffff
+#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT 0x0
+#define SQ_BUF_RSRC_WORD1__STRIDE_MASK 0x3fff0000
+#define SQ_BUF_RSRC_WORD1__STRIDE__SHIFT 0x10
+#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE_MASK 0x40000000
+#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE__SHIFT 0x1e
+#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE_MASK 0x80000000
+#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE__SHIFT 0x1f
+#define SQ_BUF_RSRC_WORD2__NUM_RECORDS_MASK 0xffffffff
+#define SQ_BUF_RSRC_WORD2__NUM_RECORDS__SHIFT 0x0
+#define SQ_BUF_RSRC_WORD3__DST_SEL_X_MASK 0x7
+#define SQ_BUF_RSRC_WORD3__DST_SEL_X__SHIFT 0x0
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Y_MASK 0x38
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Y__SHIFT 0x3
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Z_MASK 0x1c0
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Z__SHIFT 0x6
+#define SQ_BUF_RSRC_WORD3__DST_SEL_W_MASK 0xe00
+#define SQ_BUF_RSRC_WORD3__DST_SEL_W__SHIFT 0x9
+#define SQ_BUF_RSRC_WORD3__NUM_FORMAT_MASK 0x7000
+#define SQ_BUF_RSRC_WORD3__NUM_FORMAT__SHIFT 0xc
+#define SQ_BUF_RSRC_WORD3__DATA_FORMAT_MASK 0x78000
+#define SQ_BUF_RSRC_WORD3__DATA_FORMAT__SHIFT 0xf
+#define SQ_BUF_RSRC_WORD3__ELEMENT_SIZE_MASK 0x180000
+#define SQ_BUF_RSRC_WORD3__ELEMENT_SIZE__SHIFT 0x13
+#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE_MASK 0x600000
+#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE__SHIFT 0x15
+#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE_MASK 0x800000
+#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE__SHIFT 0x17
+#define SQ_BUF_RSRC_WORD3__ATC_MASK 0x1000000
+#define SQ_BUF_RSRC_WORD3__ATC__SHIFT 0x18
+#define SQ_BUF_RSRC_WORD3__HASH_ENABLE_MASK 0x2000000
+#define SQ_BUF_RSRC_WORD3__HASH_ENABLE__SHIFT 0x19
+#define SQ_BUF_RSRC_WORD3__HEAP_MASK 0x4000000
+#define SQ_BUF_RSRC_WORD3__HEAP__SHIFT 0x1a
+#define SQ_BUF_RSRC_WORD3__MTYPE_MASK 0x38000000
+#define SQ_BUF_RSRC_WORD3__MTYPE__SHIFT 0x1b
+#define SQ_BUF_RSRC_WORD3__TYPE_MASK 0xc0000000
+#define SQ_BUF_RSRC_WORD3__TYPE__SHIFT 0x1e
+#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS_MASK 0xffffffff
+#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS__SHIFT 0x0
+#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI_MASK 0xff
+#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT 0x0
+#define SQ_IMG_RSRC_WORD1__MIN_LOD_MASK 0xfff00
+#define SQ_IMG_RSRC_WORD1__MIN_LOD__SHIFT 0x8
+#define SQ_IMG_RSRC_WORD1__DATA_FORMAT_MASK 0x3f00000
+#define SQ_IMG_RSRC_WORD1__DATA_FORMAT__SHIFT 0x14
+#define SQ_IMG_RSRC_WORD1__NUM_FORMAT_MASK 0x3c000000
+#define SQ_IMG_RSRC_WORD1__NUM_FORMAT__SHIFT 0x1a
+#define SQ_IMG_RSRC_WORD1__MTYPE_MASK 0xc0000000
+#define SQ_IMG_RSRC_WORD1__MTYPE__SHIFT 0x1e
+#define SQ_IMG_RSRC_WORD2__WIDTH_MASK 0x3fff
+#define SQ_IMG_RSRC_WORD2__WIDTH__SHIFT 0x0
+#define SQ_IMG_RSRC_WORD2__HEIGHT_MASK 0xfffc000
+#define SQ_IMG_RSRC_WORD2__HEIGHT__SHIFT 0xe
+#define SQ_IMG_RSRC_WORD2__PERF_MOD_MASK 0x70000000
+#define SQ_IMG_RSRC_WORD2__PERF_MOD__SHIFT 0x1c
+#define SQ_IMG_RSRC_WORD2__INTERLACED_MASK 0x80000000
+#define SQ_IMG_RSRC_WORD2__INTERLACED__SHIFT 0x1f
+#define SQ_IMG_RSRC_WORD3__DST_SEL_X_MASK 0x7
+#define SQ_IMG_RSRC_WORD3__DST_SEL_X__SHIFT 0x0
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Y_MASK 0x38
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Y__SHIFT 0x3
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Z_MASK 0x1c0
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Z__SHIFT 0x6
+#define SQ_IMG_RSRC_WORD3__DST_SEL_W_MASK 0xe00
+#define SQ_IMG_RSRC_WORD3__DST_SEL_W__SHIFT 0x9
+#define SQ_IMG_RSRC_WORD3__BASE_LEVEL_MASK 0xf000
+#define SQ_IMG_RSRC_WORD3__BASE_LEVEL__SHIFT 0xc
+#define SQ_IMG_RSRC_WORD3__LAST_LEVEL_MASK 0xf0000
+#define SQ_IMG_RSRC_WORD3__LAST_LEVEL__SHIFT 0x10
+#define SQ_IMG_RSRC_WORD3__TILING_INDEX_MASK 0x1f00000
+#define SQ_IMG_RSRC_WORD3__TILING_INDEX__SHIFT 0x14
+#define SQ_IMG_RSRC_WORD3__POW2_PAD_MASK 0x2000000
+#define SQ_IMG_RSRC_WORD3__POW2_PAD__SHIFT 0x19
+#define SQ_IMG_RSRC_WORD3__MTYPE_MASK 0x4000000
+#define SQ_IMG_RSRC_WORD3__MTYPE__SHIFT 0x1a
+#define SQ_IMG_RSRC_WORD3__ATC_MASK 0x8000000
+#define SQ_IMG_RSRC_WORD3__ATC__SHIFT 0x1b
+#define SQ_IMG_RSRC_WORD3__TYPE_MASK 0xf0000000
+#define SQ_IMG_RSRC_WORD3__TYPE__SHIFT 0x1c
+#define SQ_IMG_RSRC_WORD4__DEPTH_MASK 0x1fff
+#define SQ_IMG_RSRC_WORD4__DEPTH__SHIFT 0x0
+#define SQ_IMG_RSRC_WORD4__PITCH_MASK 0x7ffe000
+#define SQ_IMG_RSRC_WORD4__PITCH__SHIFT 0xd
+#define SQ_IMG_RSRC_WORD5__BASE_ARRAY_MASK 0x1fff
+#define SQ_IMG_RSRC_WORD5__BASE_ARRAY__SHIFT 0x0
+#define SQ_IMG_RSRC_WORD5__LAST_ARRAY_MASK 0x3ffe000
+#define SQ_IMG_RSRC_WORD5__LAST_ARRAY__SHIFT 0xd
+#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN_MASK 0xfff
+#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN__SHIFT 0x0
+#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID_MASK 0xff000
+#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID__SHIFT 0xc
+#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN_MASK 0x100000
+#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN__SHIFT 0x14
+#define SQ_IMG_RSRC_WORD6__COMPRESSION_EN_MASK 0x200000
+#define SQ_IMG_RSRC_WORD6__COMPRESSION_EN__SHIFT 0x15
+#define SQ_IMG_RSRC_WORD6__ALPHA_IS_ON_MSB_MASK 0x400000
+#define SQ_IMG_RSRC_WORD6__ALPHA_IS_ON_MSB__SHIFT 0x16
+#define SQ_IMG_RSRC_WORD6__COLOR_TRANSFORM_MASK 0x800000
+#define SQ_IMG_RSRC_WORD6__COLOR_TRANSFORM__SHIFT 0x17
+#define SQ_IMG_RSRC_WORD6__LOST_ALPHA_BITS_MASK 0xf000000
+#define SQ_IMG_RSRC_WORD6__LOST_ALPHA_BITS__SHIFT 0x18
+#define SQ_IMG_RSRC_WORD6__LOST_COLOR_BITS_MASK 0xf0000000
+#define SQ_IMG_RSRC_WORD6__LOST_COLOR_BITS__SHIFT 0x1c
+#define SQ_IMG_RSRC_WORD7__META_DATA_ADDRESS_MASK 0xffffffff
+#define SQ_IMG_RSRC_WORD7__META_DATA_ADDRESS__SHIFT 0x0
+#define SQ_IMG_SAMP_WORD0__CLAMP_X_MASK 0x7
+#define SQ_IMG_SAMP_WORD0__CLAMP_X__SHIFT 0x0
+#define SQ_IMG_SAMP_WORD0__CLAMP_Y_MASK 0x38
+#define SQ_IMG_SAMP_WORD0__CLAMP_Y__SHIFT 0x3
+#define SQ_IMG_SAMP_WORD0__CLAMP_Z_MASK 0x1c0
+#define SQ_IMG_SAMP_WORD0__CLAMP_Z__SHIFT 0x6
+#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO_MASK 0xe00
+#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO__SHIFT 0x9
+#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC_MASK 0x7000
+#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC__SHIFT 0xc
+#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED_MASK 0x8000
+#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED__SHIFT 0xf
+#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD_MASK 0x70000
+#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD__SHIFT 0x10
+#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC_MASK 0x80000
+#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC__SHIFT 0x13
+#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA_MASK 0x100000
+#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA__SHIFT 0x14
+#define SQ_IMG_SAMP_WORD0__ANISO_BIAS_MASK 0x7e00000
+#define SQ_IMG_SAMP_WORD0__ANISO_BIAS__SHIFT 0x15
+#define SQ_IMG_SAMP_WORD0__TRUNC_COORD_MASK 0x8000000
+#define SQ_IMG_SAMP_WORD0__TRUNC_COORD__SHIFT 0x1b
+#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP_MASK 0x10000000
+#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP__SHIFT 0x1c
+#define SQ_IMG_SAMP_WORD0__FILTER_MODE_MASK 0x60000000
+#define SQ_IMG_SAMP_WORD0__FILTER_MODE__SHIFT 0x1d
+#define SQ_IMG_SAMP_WORD0__COMPAT_MODE_MASK 0x80000000
+#define SQ_IMG_SAMP_WORD0__COMPAT_MODE__SHIFT 0x1f
+#define SQ_IMG_SAMP_WORD1__MIN_LOD_MASK 0xfff
+#define SQ_IMG_SAMP_WORD1__MIN_LOD__SHIFT 0x0
+#define SQ_IMG_SAMP_WORD1__MAX_LOD_MASK 0xfff000
+#define SQ_IMG_SAMP_WORD1__MAX_LOD__SHIFT 0xc
+#define SQ_IMG_SAMP_WORD1__PERF_MIP_MASK 0xf000000
+#define SQ_IMG_SAMP_WORD1__PERF_MIP__SHIFT 0x18
+#define SQ_IMG_SAMP_WORD1__PERF_Z_MASK 0xf0000000
+#define SQ_IMG_SAMP_WORD1__PERF_Z__SHIFT 0x1c
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_MASK 0x3fff
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS__SHIFT 0x0
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC_MASK 0xfc000
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC__SHIFT 0xe
+#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER_MASK 0x300000
+#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER__SHIFT 0x14
+#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER_MASK 0xc00000
+#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER__SHIFT 0x16
+#define SQ_IMG_SAMP_WORD2__Z_FILTER_MASK 0x3000000
+#define SQ_IMG_SAMP_WORD2__Z_FILTER__SHIFT 0x18
+#define SQ_IMG_SAMP_WORD2__MIP_FILTER_MASK 0xc000000
+#define SQ_IMG_SAMP_WORD2__MIP_FILTER__SHIFT 0x1a
+#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP_MASK 0x10000000
+#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP__SHIFT 0x1c
+#define SQ_IMG_SAMP_WORD2__DISABLE_LSB_CEIL_MASK 0x20000000
+#define SQ_IMG_SAMP_WORD2__DISABLE_LSB_CEIL__SHIFT 0x1d
+#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX_MASK 0x40000000
+#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX__SHIFT 0x1e
+#define SQ_IMG_SAMP_WORD2__ANISO_OVERRIDE_MASK 0x80000000
+#define SQ_IMG_SAMP_WORD2__ANISO_OVERRIDE__SHIFT 0x1f
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR_MASK 0xfff
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR__SHIFT 0x0
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE_MASK 0xc0000000
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE__SHIFT 0x1e
+#define SQ_FLAT_SCRATCH_WORD0__SIZE_MASK 0x7ffff
+#define SQ_FLAT_SCRATCH_WORD0__SIZE__SHIFT 0x0
+#define SQ_FLAT_SCRATCH_WORD1__OFFSET_MASK 0xffffff
+#define SQ_FLAT_SCRATCH_WORD1__OFFSET__SHIFT 0x0
+#define SQ_M0_GPR_IDX_WORD__INDEX_MASK 0xff
+#define SQ_M0_GPR_IDX_WORD__INDEX__SHIFT 0x0
+#define SQ_M0_GPR_IDX_WORD__VSRC0_REL_MASK 0x1000
+#define SQ_M0_GPR_IDX_WORD__VSRC0_REL__SHIFT 0xc
+#define SQ_M0_GPR_IDX_WORD__VSRC1_REL_MASK 0x2000
+#define SQ_M0_GPR_IDX_WORD__VSRC1_REL__SHIFT 0xd
+#define SQ_M0_GPR_IDX_WORD__VSRC2_REL_MASK 0x4000
+#define SQ_M0_GPR_IDX_WORD__VSRC2_REL__SHIFT 0xe
+#define SQ_M0_GPR_IDX_WORD__VDST_REL_MASK 0x8000
+#define SQ_M0_GPR_IDX_WORD__VDST_REL__SHIFT 0xf
+#define SQ_IND_INDEX__WAVE_ID_MASK 0xf
+#define SQ_IND_INDEX__WAVE_ID__SHIFT 0x0
+#define SQ_IND_INDEX__SIMD_ID_MASK 0x30
+#define SQ_IND_INDEX__SIMD_ID__SHIFT 0x4
+#define SQ_IND_INDEX__THREAD_ID_MASK 0xfc0
+#define SQ_IND_INDEX__THREAD_ID__SHIFT 0x6
+#define SQ_IND_INDEX__AUTO_INCR_MASK 0x1000
+#define SQ_IND_INDEX__AUTO_INCR__SHIFT 0xc
+#define SQ_IND_INDEX__FORCE_READ_MASK 0x2000
+#define SQ_IND_INDEX__FORCE_READ__SHIFT 0xd
+#define SQ_IND_INDEX__READ_TIMEOUT_MASK 0x4000
+#define SQ_IND_INDEX__READ_TIMEOUT__SHIFT 0xe
+#define SQ_IND_INDEX__UNINDEXED_MASK 0x8000
+#define SQ_IND_INDEX__UNINDEXED__SHIFT 0xf
+#define SQ_IND_INDEX__INDEX_MASK 0xffff0000
+#define SQ_IND_INDEX__INDEX__SHIFT 0x10
+#define SQ_CMD__CMD_MASK 0x7
+#define SQ_CMD__CMD__SHIFT 0x0
+#define SQ_CMD__MODE_MASK 0x70
+#define SQ_CMD__MODE__SHIFT 0x4
+#define SQ_CMD__CHECK_VMID_MASK 0x80
+#define SQ_CMD__CHECK_VMID__SHIFT 0x7
+#define SQ_CMD__DATA_MASK 0x700
+#define SQ_CMD__DATA__SHIFT 0x8
+#define SQ_CMD__WAVE_ID_MASK 0xf0000
+#define SQ_CMD__WAVE_ID__SHIFT 0x10
+#define SQ_CMD__SIMD_ID_MASK 0x300000
+#define SQ_CMD__SIMD_ID__SHIFT 0x14
+#define SQ_CMD__QUEUE_ID_MASK 0x7000000
+#define SQ_CMD__QUEUE_ID__SHIFT 0x18
+#define SQ_CMD__VM_ID_MASK 0xf0000000
+#define SQ_CMD__VM_ID__SHIFT 0x1c
+#define SQ_IND_DATA__DATA_MASK 0xffffffff
+#define SQ_IND_DATA__DATA__SHIFT 0x0
+#define SQ_REG_TIMESTAMP__TIMESTAMP_MASK 0xff
+#define SQ_REG_TIMESTAMP__TIMESTAMP__SHIFT 0x0
+#define SQ_CMD_TIMESTAMP__TIMESTAMP_MASK 0xff
+#define SQ_CMD_TIMESTAMP__TIMESTAMP__SHIFT 0x0
+#define SQ_HV_VMID_CTRL__DEFAULT_VMID_MASK 0xf
+#define SQ_HV_VMID_CTRL__DEFAULT_VMID__SHIFT 0x0
+#define SQ_HV_VMID_CTRL__ALLOWED_VMID_MASK_MASK 0xffff0
+#define SQ_HV_VMID_CTRL__ALLOWED_VMID_MASK__SHIFT 0x4
+#define SQ_WAVE_INST_DW0__INST_DW0_MASK 0xffffffff
+#define SQ_WAVE_INST_DW0__INST_DW0__SHIFT 0x0
+#define SQ_WAVE_INST_DW1__INST_DW1_MASK 0xffffffff
+#define SQ_WAVE_INST_DW1__INST_DW1__SHIFT 0x0
+#define SQ_WAVE_PC_LO__PC_LO_MASK 0xffffffff
+#define SQ_WAVE_PC_LO__PC_LO__SHIFT 0x0
+#define SQ_WAVE_PC_HI__PC_HI_MASK 0xffff
+#define SQ_WAVE_PC_HI__PC_HI__SHIFT 0x0
+#define SQ_WAVE_IB_DBG0__IBUF_ST_MASK 0x7
+#define SQ_WAVE_IB_DBG0__IBUF_ST__SHIFT 0x0
+#define SQ_WAVE_IB_DBG0__PC_INVALID_MASK 0x8
+#define SQ_WAVE_IB_DBG0__PC_INVALID__SHIFT 0x3
+#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW_MASK 0x10
+#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW__SHIFT 0x4
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_MASK 0xe0
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT__SHIFT 0x5
+#define SQ_WAVE_IB_DBG0__IBUF_RPTR_MASK 0x300
+#define SQ_WAVE_IB_DBG0__IBUF_RPTR__SHIFT 0x8
+#define SQ_WAVE_IB_DBG0__IBUF_WPTR_MASK 0xc00
+#define SQ_WAVE_IB_DBG0__IBUF_WPTR__SHIFT 0xa
+#define SQ_WAVE_IB_DBG0__INST_STR_ST_MASK 0xf0000
+#define SQ_WAVE_IB_DBG0__INST_STR_ST__SHIFT 0x10
+#define SQ_WAVE_IB_DBG0__MISC_CNT_MASK 0xf00000
+#define SQ_WAVE_IB_DBG0__MISC_CNT__SHIFT 0x14
+#define SQ_WAVE_IB_DBG0__ECC_ST_MASK 0x3000000
+#define SQ_WAVE_IB_DBG0__ECC_ST__SHIFT 0x18
+#define SQ_WAVE_IB_DBG0__IS_HYB_MASK 0x4000000
+#define SQ_WAVE_IB_DBG0__IS_HYB__SHIFT 0x1a
+#define SQ_WAVE_IB_DBG0__HYB_CNT_MASK 0x18000000
+#define SQ_WAVE_IB_DBG0__HYB_CNT__SHIFT 0x1b
+#define SQ_WAVE_IB_DBG0__KILL_MASK 0x20000000
+#define SQ_WAVE_IB_DBG0__KILL__SHIFT 0x1d
+#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH_MASK 0x40000000
+#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH__SHIFT 0x1e
+#define SQ_WAVE_IB_DBG1__IXNACK_MASK 0x1
+#define SQ_WAVE_IB_DBG1__IXNACK__SHIFT 0x0
+#define SQ_WAVE_IB_DBG1__XNACK_MASK 0x2
+#define SQ_WAVE_IB_DBG1__XNACK__SHIFT 0x1
+#define SQ_WAVE_IB_DBG1__TA_NEED_RESET_MASK 0x4
+#define SQ_WAVE_IB_DBG1__TA_NEED_RESET__SHIFT 0x2
+#define SQ_WAVE_IB_DBG1__XCNT_MASK 0xf0
+#define SQ_WAVE_IB_DBG1__XCNT__SHIFT 0x4
+#define SQ_WAVE_IB_DBG1__QCNT_MASK 0xf00
+#define SQ_WAVE_IB_DBG1__QCNT__SHIFT 0x8
+#define SQ_WAVE_EXEC_LO__EXEC_LO_MASK 0xffffffff
+#define SQ_WAVE_EXEC_LO__EXEC_LO__SHIFT 0x0
+#define SQ_WAVE_EXEC_HI__EXEC_HI_MASK 0xffffffff
+#define SQ_WAVE_EXEC_HI__EXEC_HI__SHIFT 0x0
+#define SQ_WAVE_STATUS__SCC_MASK 0x1
+#define SQ_WAVE_STATUS__SCC__SHIFT 0x0
+#define SQ_WAVE_STATUS__SPI_PRIO_MASK 0x6
+#define SQ_WAVE_STATUS__SPI_PRIO__SHIFT 0x1
+#define SQ_WAVE_STATUS__USER_PRIO_MASK 0x18
+#define SQ_WAVE_STATUS__USER_PRIO__SHIFT 0x3
+#define SQ_WAVE_STATUS__PRIV_MASK 0x20
+#define SQ_WAVE_STATUS__PRIV__SHIFT 0x5
+#define SQ_WAVE_STATUS__TRAP_EN_MASK 0x40
+#define SQ_WAVE_STATUS__TRAP_EN__SHIFT 0x6
+#define SQ_WAVE_STATUS__TTRACE_EN_MASK 0x80
+#define SQ_WAVE_STATUS__TTRACE_EN__SHIFT 0x7
+#define SQ_WAVE_STATUS__EXPORT_RDY_MASK 0x100
+#define SQ_WAVE_STATUS__EXPORT_RDY__SHIFT 0x8
+#define SQ_WAVE_STATUS__EXECZ_MASK 0x200
+#define SQ_WAVE_STATUS__EXECZ__SHIFT 0x9
+#define SQ_WAVE_STATUS__VCCZ_MASK 0x400
+#define SQ_WAVE_STATUS__VCCZ__SHIFT 0xa
+#define SQ_WAVE_STATUS__IN_TG_MASK 0x800
+#define SQ_WAVE_STATUS__IN_TG__SHIFT 0xb
+#define SQ_WAVE_STATUS__IN_BARRIER_MASK 0x1000
+#define SQ_WAVE_STATUS__IN_BARRIER__SHIFT 0xc
+#define SQ_WAVE_STATUS__HALT_MASK 0x2000
+#define SQ_WAVE_STATUS__HALT__SHIFT 0xd
+#define SQ_WAVE_STATUS__TRAP_MASK 0x4000
+#define SQ_WAVE_STATUS__TRAP__SHIFT 0xe
+#define SQ_WAVE_STATUS__TTRACE_CU_EN_MASK 0x8000
+#define SQ_WAVE_STATUS__TTRACE_CU_EN__SHIFT 0xf
+#define SQ_WAVE_STATUS__VALID_MASK 0x10000
+#define SQ_WAVE_STATUS__VALID__SHIFT 0x10
+#define SQ_WAVE_STATUS__ECC_ERR_MASK 0x20000
+#define SQ_WAVE_STATUS__ECC_ERR__SHIFT 0x11
+#define SQ_WAVE_STATUS__SKIP_EXPORT_MASK 0x40000
+#define SQ_WAVE_STATUS__SKIP_EXPORT__SHIFT 0x12
+#define SQ_WAVE_STATUS__PERF_EN_MASK 0x80000
+#define SQ_WAVE_STATUS__PERF_EN__SHIFT 0x13
+#define SQ_WAVE_STATUS__COND_DBG_USER_MASK 0x100000
+#define SQ_WAVE_STATUS__COND_DBG_USER__SHIFT 0x14
+#define SQ_WAVE_STATUS__COND_DBG_SYS_MASK 0x200000
+#define SQ_WAVE_STATUS__COND_DBG_SYS__SHIFT 0x15
+#define SQ_WAVE_STATUS__ALLOW_REPLAY_MASK 0x400000
+#define SQ_WAVE_STATUS__ALLOW_REPLAY__SHIFT 0x16
+#define SQ_WAVE_STATUS__INST_ATC_MASK 0x800000
+#define SQ_WAVE_STATUS__INST_ATC__SHIFT 0x17
+#define SQ_WAVE_STATUS__MUST_EXPORT_MASK 0x8000000
+#define SQ_WAVE_STATUS__MUST_EXPORT__SHIFT 0x1b
+#define SQ_WAVE_MODE__FP_ROUND_MASK 0xf
+#define SQ_WAVE_MODE__FP_ROUND__SHIFT 0x0
+#define SQ_WAVE_MODE__FP_DENORM_MASK 0xf0
+#define SQ_WAVE_MODE__FP_DENORM__SHIFT 0x4
+#define SQ_WAVE_MODE__DX10_CLAMP_MASK 0x100
+#define SQ_WAVE_MODE__DX10_CLAMP__SHIFT 0x8
+#define SQ_WAVE_MODE__IEEE_MASK 0x200
+#define SQ_WAVE_MODE__IEEE__SHIFT 0x9
+#define SQ_WAVE_MODE__LOD_CLAMPED_MASK 0x400
+#define SQ_WAVE_MODE__LOD_CLAMPED__SHIFT 0xa
+#define SQ_WAVE_MODE__DEBUG_EN_MASK 0x800
+#define SQ_WAVE_MODE__DEBUG_EN__SHIFT 0xb
+#define SQ_WAVE_MODE__EXCP_EN_MASK 0x1ff000
+#define SQ_WAVE_MODE__EXCP_EN__SHIFT 0xc
+#define SQ_WAVE_MODE__GPR_IDX_EN_MASK 0x8000000
+#define SQ_WAVE_MODE__GPR_IDX_EN__SHIFT 0x1b
+#define SQ_WAVE_MODE__VSKIP_MASK 0x10000000
+#define SQ_WAVE_MODE__VSKIP__SHIFT 0x1c
+#define SQ_WAVE_MODE__CSP_MASK 0xe0000000
+#define SQ_WAVE_MODE__CSP__SHIFT 0x1d
+#define SQ_WAVE_TRAPSTS__EXCP_MASK 0x1ff
+#define SQ_WAVE_TRAPSTS__EXCP__SHIFT 0x0
+#define SQ_WAVE_TRAPSTS__SAVECTX_MASK 0x400
+#define SQ_WAVE_TRAPSTS__SAVECTX__SHIFT 0xa
+#define SQ_WAVE_TRAPSTS__EXCP_CYCLE_MASK 0x3f0000
+#define SQ_WAVE_TRAPSTS__EXCP_CYCLE__SHIFT 0x10
+#define SQ_WAVE_TRAPSTS__DP_RATE_MASK 0xe0000000
+#define SQ_WAVE_TRAPSTS__DP_RATE__SHIFT 0x1d
+#define SQ_WAVE_HW_ID__WAVE_ID_MASK 0xf
+#define SQ_WAVE_HW_ID__WAVE_ID__SHIFT 0x0
+#define SQ_WAVE_HW_ID__SIMD_ID_MASK 0x30
+#define SQ_WAVE_HW_ID__SIMD_ID__SHIFT 0x4
+#define SQ_WAVE_HW_ID__PIPE_ID_MASK 0xc0
+#define SQ_WAVE_HW_ID__PIPE_ID__SHIFT 0x6
+#define SQ_WAVE_HW_ID__CU_ID_MASK 0xf00
+#define SQ_WAVE_HW_ID__CU_ID__SHIFT 0x8
+#define SQ_WAVE_HW_ID__SH_ID_MASK 0x1000
+#define SQ_WAVE_HW_ID__SH_ID__SHIFT 0xc
+#define SQ_WAVE_HW_ID__SE_ID_MASK 0x6000
+#define SQ_WAVE_HW_ID__SE_ID__SHIFT 0xd
+#define SQ_WAVE_HW_ID__TG_ID_MASK 0xf0000
+#define SQ_WAVE_HW_ID__TG_ID__SHIFT 0x10
+#define SQ_WAVE_HW_ID__VM_ID_MASK 0xf00000
+#define SQ_WAVE_HW_ID__VM_ID__SHIFT 0x14
+#define SQ_WAVE_HW_ID__QUEUE_ID_MASK 0x7000000
+#define SQ_WAVE_HW_ID__QUEUE_ID__SHIFT 0x18
+#define SQ_WAVE_HW_ID__STATE_ID_MASK 0x38000000
+#define SQ_WAVE_HW_ID__STATE_ID__SHIFT 0x1b
+#define SQ_WAVE_HW_ID__ME_ID_MASK 0xc0000000
+#define SQ_WAVE_HW_ID__ME_ID__SHIFT 0x1e
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE_MASK 0x3f
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE__SHIFT 0x0
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE_MASK 0x3f00
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE__SHIFT 0x8
+#define SQ_WAVE_GPR_ALLOC__SGPR_BASE_MASK 0x3f0000
+#define SQ_WAVE_GPR_ALLOC__SGPR_BASE__SHIFT 0x10
+#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE_MASK 0xf000000
+#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE__SHIFT 0x18
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE_MASK 0xff
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE__SHIFT 0x0
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE_MASK 0x1ff000
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE__SHIFT 0xc
+#define SQ_WAVE_IB_STS__VM_CNT_MASK 0xf
+#define SQ_WAVE_IB_STS__VM_CNT__SHIFT 0x0
+#define SQ_WAVE_IB_STS__EXP_CNT_MASK 0x70
+#define SQ_WAVE_IB_STS__EXP_CNT__SHIFT 0x4
+#define SQ_WAVE_IB_STS__LGKM_CNT_MASK 0xf00
+#define SQ_WAVE_IB_STS__LGKM_CNT__SHIFT 0x8
+#define SQ_WAVE_IB_STS__VALU_CNT_MASK 0x7000
+#define SQ_WAVE_IB_STS__VALU_CNT__SHIFT 0xc
+#define SQ_WAVE_IB_STS__FIRST_REPLAY_MASK 0x8000
+#define SQ_WAVE_IB_STS__FIRST_REPLAY__SHIFT 0xf
+#define SQ_WAVE_IB_STS__RCNT_MASK 0xf0000
+#define SQ_WAVE_IB_STS__RCNT__SHIFT 0x10
+#define SQ_WAVE_M0__M0_MASK 0xffffffff
+#define SQ_WAVE_M0__M0__SHIFT 0x0
+#define SQ_WAVE_TBA_LO__ADDR_LO_MASK 0xffffffff
+#define SQ_WAVE_TBA_LO__ADDR_LO__SHIFT 0x0
+#define SQ_WAVE_TBA_HI__ADDR_HI_MASK 0xff
+#define SQ_WAVE_TBA_HI__ADDR_HI__SHIFT 0x0
+#define SQ_WAVE_TMA_LO__ADDR_LO_MASK 0xffffffff
+#define SQ_WAVE_TMA_LO__ADDR_LO__SHIFT 0x0
+#define SQ_WAVE_TMA_HI__ADDR_HI_MASK 0xff
+#define SQ_WAVE_TMA_HI__ADDR_HI__SHIFT 0x0
+#define SQ_WAVE_TTMP0__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP0__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP1__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP1__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP2__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP2__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP3__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP3__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP4__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP4__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP5__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP5__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP6__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP6__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP7__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP7__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP8__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP8__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP9__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP9__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP10__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP10__DATA__SHIFT 0x0
+#define SQ_WAVE_TTMP11__DATA_MASK 0xffffffff
+#define SQ_WAVE_TTMP11__DATA__SHIFT 0x0
+#define SQ_DEBUG_STS_GLOBAL__BUSY_MASK 0x1
+#define SQ_DEBUG_STS_GLOBAL__BUSY__SHIFT 0x0
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY_MASK 0x2
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY__SHIFT 0x1
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0_MASK 0xfff0
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0__SHIFT 0x4
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1_MASK 0xfff0000
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1__SHIFT 0x10
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0_MASK 0xff
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0__SHIFT 0x0
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1_MASK 0xff00
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1__SHIFT 0x8
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED_MASK 0xff0000
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED__SHIFT 0x10
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST_MASK 0xff000000
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST__SHIFT 0x18
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD_MASK 0xf
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD__SHIFT 0x0
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG_MASK 0x3f0
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG__SHIFT 0x4
+#define SQ_DEBUG_STS_LOCAL__BUSY_MASK 0x1
+#define SQ_DEBUG_STS_LOCAL__BUSY__SHIFT 0x0
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL_MASK 0x3f0
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL__SHIFT 0x4
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED_MASK 0xff
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED__SHIFT 0x0
+#define SH_MEM_BASES__PRIVATE_BASE_MASK 0xffff
+#define SH_MEM_BASES__PRIVATE_BASE__SHIFT 0x0
+#define SH_MEM_BASES__SHARED_BASE_MASK 0xffff0000
+#define SH_MEM_BASES__SHARED_BASE__SHIFT 0x10
+#define SH_MEM_APE1_BASE__BASE_MASK 0xffffffff
+#define SH_MEM_APE1_BASE__BASE__SHIFT 0x0
+#define SH_MEM_APE1_LIMIT__LIMIT_MASK 0xffffffff
+#define SH_MEM_APE1_LIMIT__LIMIT__SHIFT 0x0
+#define SH_MEM_CONFIG__ADDRESS_MODE_MASK 0x3
+#define SH_MEM_CONFIG__ADDRESS_MODE__SHIFT 0x0
+#define SH_MEM_CONFIG__PRIVATE_ATC_MASK 0x4
+#define SH_MEM_CONFIG__PRIVATE_ATC__SHIFT 0x2
+#define SH_MEM_CONFIG__ALIGNMENT_MODE_MASK 0x18
+#define SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT 0x3
+#define SH_MEM_CONFIG__DEFAULT_MTYPE_MASK 0xe0
+#define SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT 0x5
+#define SH_MEM_CONFIG__APE1_MTYPE_MASK 0x700
+#define SH_MEM_CONFIG__APE1_MTYPE__SHIFT 0x8
+#define SH_MEM_CONFIG__APE1_ATC_MASK 0x800
+#define SH_MEM_CONFIG__APE1_ATC__SHIFT 0xb
+#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID_MASK 0x1e0
+#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID_MASK 0x600
+#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID__SHIFT 0x9
+#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE_MASK 0xf800
+#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE__SHIFT 0xb
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID_MASK 0x1e0
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID_MASK 0x600
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID__SHIFT 0x9
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO_MASK 0xffff0000
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO__SHIFT 0x10
+#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI_MASK 0xffffff
+#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID_MASK 0x20
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID_MASK 0x3c0
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID__SHIFT 0x6
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID_MASK 0x3c00
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID__SHIFT 0xa
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID_MASK 0xc000
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID__SHIFT 0xe
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO_MASK 0xffff0000
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO__SHIFT 0x10
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI_MASK 0xffff
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO_MASK 0xffff0000
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO__SHIFT 0x10
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI_MASK 0xffffffff
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID_MASK 0x20
+#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID_MASK 0x3c0
+#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID__SHIFT 0x6
+#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID_MASK 0x3c00
+#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID__SHIFT 0xa
+#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID_MASK 0xc000
+#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID__SHIFT 0xe
+#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA_MASK 0xff0
+#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_MISC__SH_ID_MASK 0x1000
+#define SQ_THREAD_TRACE_WORD_MISC__SH_ID__SHIFT 0xc
+#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE_MASK 0xe000
+#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE__SHIFT 0xd
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID_MASK 0x20
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID_MASK 0x3c0
+#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID__SHIFT 0x6
+#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID_MASK 0x3c00
+#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID__SHIFT 0xa
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID_MASK 0xc000
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID__SHIFT 0xe
+#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER_MASK 0x1f0000
+#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER__SHIFT 0x10
+#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED_MASK 0x200000
+#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED__SHIFT 0x15
+#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT_MASK 0x1fc00000
+#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT__SHIFT 0x16
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID_MASK 0xe0000000
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID__SHIFT 0x1d
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID_MASK 0x60
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID_MASK 0x180
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID__SHIFT 0x7
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV_MASK 0x200
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV__SHIFT 0x9
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE_MASK 0x1c00
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE__SHIFT 0xa
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV_MASK 0x4000
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV__SHIFT 0xe
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP_MASK 0x8000
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP__SHIFT 0xf
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR_MASK 0xffff0000
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR__SHIFT 0x10
+#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA_MASK 0xffffffff
+#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID_MASK 0x60
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID_MASK 0x180
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID__SHIFT 0x7
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR_MASK 0xfe00
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR__SHIFT 0x9
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO_MASK 0xffff0000
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO__SHIFT 0x10
+#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI_MASK 0xffff
+#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID_MASK 0x20
+#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_EVENT__STAGE_MASK 0x1c0
+#define SQ_THREAD_TRACE_WORD_EVENT__STAGE__SHIFT 0x6
+#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE_MASK 0xfc00
+#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE__SHIFT 0xa
+#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID_MASK 0x60
+#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST0_MASK 0x300
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST0__SHIFT 0x8
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST1_MASK 0xc00
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST1__SHIFT 0xa
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST2_MASK 0x3000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST2__SHIFT 0xc
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST3_MASK 0xc000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST3__SHIFT 0xe
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST4_MASK 0x30000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST4__SHIFT 0x10
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST5_MASK 0xc0000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST5__SHIFT 0x12
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST6_MASK 0x300000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST6__SHIFT 0x14
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST7_MASK 0xc00000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST7__SHIFT 0x16
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST8_MASK 0x3000000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST8__SHIFT 0x18
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST9_MASK 0xc000000
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST9__SHIFT 0x1a
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE_MASK 0xf
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA_MASK 0x10
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA__SHIFT 0x4
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID_MASK 0x20
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID__SHIFT 0x5
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID_MASK 0x3c0
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID__SHIFT 0x6
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK_MASK 0xc00
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK__SHIFT 0xa
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0_MASK 0x1fff000
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0__SHIFT 0xc
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO_MASK 0xfe000000
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO__SHIFT 0x19
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI_MASK 0x3f
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI__SHIFT 0x0
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2_MASK 0x7ffc0
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2__SHIFT 0x6
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3_MASK 0xfff80000
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3__SHIFT 0x13
+#define SQ_WREXEC_EXEC_LO__ADDR_LO_MASK 0xffffffff
+#define SQ_WREXEC_EXEC_LO__ADDR_LO__SHIFT 0x0
+#define SQ_WREXEC_EXEC_HI__ADDR_HI_MASK 0xffff
+#define SQ_WREXEC_EXEC_HI__ADDR_HI__SHIFT 0x0
+#define SQ_WREXEC_EXEC_HI__FIRST_WAVE_MASK 0x4000000
+#define SQ_WREXEC_EXEC_HI__FIRST_WAVE__SHIFT 0x1a
+#define SQ_WREXEC_EXEC_HI__ATC_MASK 0x8000000
+#define SQ_WREXEC_EXEC_HI__ATC__SHIFT 0x1b
+#define SQ_WREXEC_EXEC_HI__MTYPE_MASK 0x70000000
+#define SQ_WREXEC_EXEC_HI__MTYPE__SHIFT 0x1c
+#define SQ_WREXEC_EXEC_HI__MSB_MASK 0x80000000
+#define SQ_WREXEC_EXEC_HI__MSB__SHIFT 0x1f
+#define SQC_GATCL1_CNTL__RESERVED_MASK 0x3ffff
+#define SQC_GATCL1_CNTL__RESERVED__SHIFT 0x0
+#define SQC_GATCL1_CNTL__DCACHE_INVALIDATE_ALL_VMID_MASK 0x40000
+#define SQC_GATCL1_CNTL__DCACHE_INVALIDATE_ALL_VMID__SHIFT 0x12
+#define SQC_GATCL1_CNTL__DCACHE_FORCE_MISS_MASK 0x80000
+#define SQC_GATCL1_CNTL__DCACHE_FORCE_MISS__SHIFT 0x13
+#define SQC_GATCL1_CNTL__DCACHE_FORCE_IN_ORDER_MASK 0x100000
+#define SQC_GATCL1_CNTL__DCACHE_FORCE_IN_ORDER__SHIFT 0x14
+#define SQC_GATCL1_CNTL__DCACHE_REDUCE_FIFO_DEPTH_BY_2_MASK 0x600000
+#define SQC_GATCL1_CNTL__DCACHE_REDUCE_FIFO_DEPTH_BY_2__SHIFT 0x15
+#define SQC_GATCL1_CNTL__DCACHE_REDUCE_CACHE_SIZE_BY_2_MASK 0x1800000
+#define SQC_GATCL1_CNTL__DCACHE_REDUCE_CACHE_SIZE_BY_2__SHIFT 0x17
+#define SQC_GATCL1_CNTL__ICACHE_INVALIDATE_ALL_VMID_MASK 0x2000000
+#define SQC_GATCL1_CNTL__ICACHE_INVALIDATE_ALL_VMID__SHIFT 0x19
+#define SQC_GATCL1_CNTL__ICACHE_FORCE_MISS_MASK 0x4000000
+#define SQC_GATCL1_CNTL__ICACHE_FORCE_MISS__SHIFT 0x1a
+#define SQC_GATCL1_CNTL__ICACHE_FORCE_IN_ORDER_MASK 0x8000000
+#define SQC_GATCL1_CNTL__ICACHE_FORCE_IN_ORDER__SHIFT 0x1b
+#define SQC_GATCL1_CNTL__ICACHE_REDUCE_FIFO_DEPTH_BY_2_MASK 0x30000000
+#define SQC_GATCL1_CNTL__ICACHE_REDUCE_FIFO_DEPTH_BY_2__SHIFT 0x1c
+#define SQC_GATCL1_CNTL__ICACHE_REDUCE_CACHE_SIZE_BY_2_MASK 0xc0000000
+#define SQC_GATCL1_CNTL__ICACHE_REDUCE_CACHE_SIZE_BY_2__SHIFT 0x1e
+#define SQC_ATC_EDC_GATCL1_CNT__ICACHE_DATA_SEC_MASK 0xff
+#define SQC_ATC_EDC_GATCL1_CNT__ICACHE_DATA_SEC__SHIFT 0x0
+#define SQC_ATC_EDC_GATCL1_CNT__DCACHE_DATA_SEC_MASK 0xff0000
+#define SQC_ATC_EDC_GATCL1_CNT__DCACHE_DATA_SEC__SHIFT 0x10
+#define SQ_INTERRUPT_WORD_CMN__SE_ID_MASK 0x3000000
+#define SQ_INTERRUPT_WORD_CMN__SE_ID__SHIFT 0x18
+#define SQ_INTERRUPT_WORD_CMN__ENCODING_MASK 0xc000000
+#define SQ_INTERRUPT_WORD_CMN__ENCODING__SHIFT 0x1a
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_MASK 0x1
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE__SHIFT 0x0
+#define SQ_INTERRUPT_WORD_AUTO__WLT_MASK 0x2
+#define SQ_INTERRUPT_WORD_AUTO__WLT__SHIFT 0x1
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_BUF_FULL_MASK 0x4
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_BUF_FULL__SHIFT 0x2
+#define SQ_INTERRUPT_WORD_AUTO__REG_TIMESTAMP_MASK 0x8
+#define SQ_INTERRUPT_WORD_AUTO__REG_TIMESTAMP__SHIFT 0x3
+#define SQ_INTERRUPT_WORD_AUTO__CMD_TIMESTAMP_MASK 0x10
+#define SQ_INTERRUPT_WORD_AUTO__CMD_TIMESTAMP__SHIFT 0x4
+#define SQ_INTERRUPT_WORD_AUTO__HOST_CMD_OVERFLOW_MASK 0x20
+#define SQ_INTERRUPT_WORD_AUTO__HOST_CMD_OVERFLOW__SHIFT 0x5
+#define SQ_INTERRUPT_WORD_AUTO__HOST_REG_OVERFLOW_MASK 0x40
+#define SQ_INTERRUPT_WORD_AUTO__HOST_REG_OVERFLOW__SHIFT 0x6
+#define SQ_INTERRUPT_WORD_AUTO__IMMED_OVERFLOW_MASK 0x80
+#define SQ_INTERRUPT_WORD_AUTO__IMMED_OVERFLOW__SHIFT 0x7
+#define SQ_INTERRUPT_WORD_AUTO__SE_ID_MASK 0x3000000
+#define SQ_INTERRUPT_WORD_AUTO__SE_ID__SHIFT 0x18
+#define SQ_INTERRUPT_WORD_AUTO__ENCODING_MASK 0xc000000
+#define SQ_INTERRUPT_WORD_AUTO__ENCODING__SHIFT 0x1a
+#define SQ_INTERRUPT_WORD_WAVE__DATA_MASK 0xff
+#define SQ_INTERRUPT_WORD_WAVE__DATA__SHIFT 0x0
+#define SQ_INTERRUPT_WORD_WAVE__SH_ID_MASK 0x100
+#define SQ_INTERRUPT_WORD_WAVE__SH_ID__SHIFT 0x8
+#define SQ_INTERRUPT_WORD_WAVE__PRIV_MASK 0x200
+#define SQ_INTERRUPT_WORD_WAVE__PRIV__SHIFT 0x9
+#define SQ_INTERRUPT_WORD_WAVE__VM_ID_MASK 0x3c00
+#define SQ_INTERRUPT_WORD_WAVE__VM_ID__SHIFT 0xa
+#define SQ_INTERRUPT_WORD_WAVE__WAVE_ID_MASK 0x3c000
+#define SQ_INTERRUPT_WORD_WAVE__WAVE_ID__SHIFT 0xe
+#define SQ_INTERRUPT_WORD_WAVE__SIMD_ID_MASK 0xc0000
+#define SQ_INTERRUPT_WORD_WAVE__SIMD_ID__SHIFT 0x12
+#define SQ_INTERRUPT_WORD_WAVE__CU_ID_MASK 0xf00000
+#define SQ_INTERRUPT_WORD_WAVE__CU_ID__SHIFT 0x14
+#define SQ_INTERRUPT_WORD_WAVE__SE_ID_MASK 0x3000000
+#define SQ_INTERRUPT_WORD_WAVE__SE_ID__SHIFT 0x18
+#define SQ_INTERRUPT_WORD_WAVE__ENCODING_MASK 0xc000000
+#define SQ_INTERRUPT_WORD_WAVE__ENCODING__SHIFT 0x1a
+#define SQ_SOP2__SSRC0_MASK 0xff
+#define SQ_SOP2__SSRC0__SHIFT 0x0
+#define SQ_SOP2__SSRC1_MASK 0xff00
+#define SQ_SOP2__SSRC1__SHIFT 0x8
+#define SQ_SOP2__SDST_MASK 0x7f0000
+#define SQ_SOP2__SDST__SHIFT 0x10
+#define SQ_SOP2__OP_MASK 0x3f800000
+#define SQ_SOP2__OP__SHIFT 0x17
+#define SQ_SOP2__ENCODING_MASK 0xc0000000
+#define SQ_SOP2__ENCODING__SHIFT 0x1e
+#define SQ_VOP1__SRC0_MASK 0x1ff
+#define SQ_VOP1__SRC0__SHIFT 0x0
+#define SQ_VOP1__OP_MASK 0x1fe00
+#define SQ_VOP1__OP__SHIFT 0x9
+#define SQ_VOP1__VDST_MASK 0x1fe0000
+#define SQ_VOP1__VDST__SHIFT 0x11
+#define SQ_VOP1__ENCODING_MASK 0xfe000000
+#define SQ_VOP1__ENCODING__SHIFT 0x19
+#define SQ_MTBUF_1__VADDR_MASK 0xff
+#define SQ_MTBUF_1__VADDR__SHIFT 0x0
+#define SQ_MTBUF_1__VDATA_MASK 0xff00
+#define SQ_MTBUF_1__VDATA__SHIFT 0x8
+#define SQ_MTBUF_1__SRSRC_MASK 0x1f0000
+#define SQ_MTBUF_1__SRSRC__SHIFT 0x10
+#define SQ_MTBUF_1__SLC_MASK 0x400000
+#define SQ_MTBUF_1__SLC__SHIFT 0x16
+#define SQ_MTBUF_1__TFE_MASK 0x800000
+#define SQ_MTBUF_1__TFE__SHIFT 0x17
+#define SQ_MTBUF_1__SOFFSET_MASK 0xff000000
+#define SQ_MTBUF_1__SOFFSET__SHIFT 0x18
+#define SQ_EXP_1__VSRC0_MASK 0xff
+#define SQ_EXP_1__VSRC0__SHIFT 0x0
+#define SQ_EXP_1__VSRC1_MASK 0xff00
+#define SQ_EXP_1__VSRC1__SHIFT 0x8
+#define SQ_EXP_1__VSRC2_MASK 0xff0000
+#define SQ_EXP_1__VSRC2__SHIFT 0x10
+#define SQ_EXP_1__VSRC3_MASK 0xff000000
+#define SQ_EXP_1__VSRC3__SHIFT 0x18
+#define SQ_MUBUF_1__VADDR_MASK 0xff
+#define SQ_MUBUF_1__VADDR__SHIFT 0x0
+#define SQ_MUBUF_1__VDATA_MASK 0xff00
+#define SQ_MUBUF_1__VDATA__SHIFT 0x8
+#define SQ_MUBUF_1__SRSRC_MASK 0x1f0000
+#define SQ_MUBUF_1__SRSRC__SHIFT 0x10
+#define SQ_MUBUF_1__TFE_MASK 0x800000
+#define SQ_MUBUF_1__TFE__SHIFT 0x17
+#define SQ_MUBUF_1__SOFFSET_MASK 0xff000000
+#define SQ_MUBUF_1__SOFFSET__SHIFT 0x18
+#define SQ_SMEM_1__OFFSET_MASK 0xfffff
+#define SQ_SMEM_1__OFFSET__SHIFT 0x0
+#define SQ_INST__ENCODING_MASK 0xffffffff
+#define SQ_INST__ENCODING__SHIFT 0x0
+#define SQ_EXP_0__EN_MASK 0xf
+#define SQ_EXP_0__EN__SHIFT 0x0
+#define SQ_EXP_0__TGT_MASK 0x3f0
+#define SQ_EXP_0__TGT__SHIFT 0x4
+#define SQ_EXP_0__COMPR_MASK 0x400
+#define SQ_EXP_0__COMPR__SHIFT 0xa
+#define SQ_EXP_0__DONE_MASK 0x800
+#define SQ_EXP_0__DONE__SHIFT 0xb
+#define SQ_EXP_0__VM_MASK 0x1000
+#define SQ_EXP_0__VM__SHIFT 0xc
+#define SQ_EXP_0__ENCODING_MASK 0xfc000000
+#define SQ_EXP_0__ENCODING__SHIFT 0x1a
+#define SQ_MUBUF_0__OFFSET_MASK 0xfff
+#define SQ_MUBUF_0__OFFSET__SHIFT 0x0
+#define SQ_MUBUF_0__OFFEN_MASK 0x1000
+#define SQ_MUBUF_0__OFFEN__SHIFT 0xc
+#define SQ_MUBUF_0__IDXEN_MASK 0x2000
+#define SQ_MUBUF_0__IDXEN__SHIFT 0xd
+#define SQ_MUBUF_0__GLC_MASK 0x4000
+#define SQ_MUBUF_0__GLC__SHIFT 0xe
+#define SQ_MUBUF_0__LDS_MASK 0x10000
+#define SQ_MUBUF_0__LDS__SHIFT 0x10
+#define SQ_MUBUF_0__SLC_MASK 0x20000
+#define SQ_MUBUF_0__SLC__SHIFT 0x11
+#define SQ_MUBUF_0__OP_MASK 0x1fc0000
+#define SQ_MUBUF_0__OP__SHIFT 0x12
+#define SQ_MUBUF_0__ENCODING_MASK 0xfc000000
+#define SQ_MUBUF_0__ENCODING__SHIFT 0x1a
+#define SQ_VOP_SDWA__SRC0_MASK 0xff
+#define SQ_VOP_SDWA__SRC0__SHIFT 0x0
+#define SQ_VOP_SDWA__DST_SEL_MASK 0x700
+#define SQ_VOP_SDWA__DST_SEL__SHIFT 0x8
+#define SQ_VOP_SDWA__DST_UNUSED_MASK 0x1800
+#define SQ_VOP_SDWA__DST_UNUSED__SHIFT 0xb
+#define SQ_VOP_SDWA__CLAMP_MASK 0x2000
+#define SQ_VOP_SDWA__CLAMP__SHIFT 0xd
+#define SQ_VOP_SDWA__SRC0_SEL_MASK 0x70000
+#define SQ_VOP_SDWA__SRC0_SEL__SHIFT 0x10
+#define SQ_VOP_SDWA__SRC0_SEXT_MASK 0x80000
+#define SQ_VOP_SDWA__SRC0_SEXT__SHIFT 0x13
+#define SQ_VOP_SDWA__SRC0_NEG_MASK 0x100000
+#define SQ_VOP_SDWA__SRC0_NEG__SHIFT 0x14
+#define SQ_VOP_SDWA__SRC0_ABS_MASK 0x200000
+#define SQ_VOP_SDWA__SRC0_ABS__SHIFT 0x15
+#define SQ_VOP_SDWA__SRC1_SEL_MASK 0x7000000
+#define SQ_VOP_SDWA__SRC1_SEL__SHIFT 0x18
+#define SQ_VOP_SDWA__SRC1_SEXT_MASK 0x8000000
+#define SQ_VOP_SDWA__SRC1_SEXT__SHIFT 0x1b
+#define SQ_VOP_SDWA__SRC1_NEG_MASK 0x10000000
+#define SQ_VOP_SDWA__SRC1_NEG__SHIFT 0x1c
+#define SQ_VOP_SDWA__SRC1_ABS_MASK 0x20000000
+#define SQ_VOP_SDWA__SRC1_ABS__SHIFT 0x1d
+#define SQ_VOP3_0__VDST_MASK 0xff
+#define SQ_VOP3_0__VDST__SHIFT 0x0
+#define SQ_VOP3_0__ABS_MASK 0x700
+#define SQ_VOP3_0__ABS__SHIFT 0x8
+#define SQ_VOP3_0__CLAMP_MASK 0x8000
+#define SQ_VOP3_0__CLAMP__SHIFT 0xf
+#define SQ_VOP3_0__OP_MASK 0x3ff0000
+#define SQ_VOP3_0__OP__SHIFT 0x10
+#define SQ_VOP3_0__ENCODING_MASK 0xfc000000
+#define SQ_VOP3_0__ENCODING__SHIFT 0x1a
+#define SQ_VOP2__SRC0_MASK 0x1ff
+#define SQ_VOP2__SRC0__SHIFT 0x0
+#define SQ_VOP2__VSRC1_MASK 0x1fe00
+#define SQ_VOP2__VSRC1__SHIFT 0x9
+#define SQ_VOP2__VDST_MASK 0x1fe0000
+#define SQ_VOP2__VDST__SHIFT 0x11
+#define SQ_VOP2__OP_MASK 0x7e000000
+#define SQ_VOP2__OP__SHIFT 0x19
+#define SQ_VOP2__ENCODING_MASK 0x80000000
+#define SQ_VOP2__ENCODING__SHIFT 0x1f
+#define SQ_MTBUF_0__OFFSET_MASK 0xfff
+#define SQ_MTBUF_0__OFFSET__SHIFT 0x0
+#define SQ_MTBUF_0__OFFEN_MASK 0x1000
+#define SQ_MTBUF_0__OFFEN__SHIFT 0xc
+#define SQ_MTBUF_0__IDXEN_MASK 0x2000
+#define SQ_MTBUF_0__IDXEN__SHIFT 0xd
+#define SQ_MTBUF_0__GLC_MASK 0x4000
+#define SQ_MTBUF_0__GLC__SHIFT 0xe
+#define SQ_MTBUF_0__OP_MASK 0x78000
+#define SQ_MTBUF_0__OP__SHIFT 0xf
+#define SQ_MTBUF_0__DFMT_MASK 0x780000
+#define SQ_MTBUF_0__DFMT__SHIFT 0x13
+#define SQ_MTBUF_0__NFMT_MASK 0x3800000
+#define SQ_MTBUF_0__NFMT__SHIFT 0x17
+#define SQ_MTBUF_0__ENCODING_MASK 0xfc000000
+#define SQ_MTBUF_0__ENCODING__SHIFT 0x1a
+#define SQ_SOPP__SIMM16_MASK 0xffff
+#define SQ_SOPP__SIMM16__SHIFT 0x0
+#define SQ_SOPP__OP_MASK 0x7f0000
+#define SQ_SOPP__OP__SHIFT 0x10
+#define SQ_SOPP__ENCODING_MASK 0xff800000
+#define SQ_SOPP__ENCODING__SHIFT 0x17
+#define SQ_FLAT_0__GLC_MASK 0x10000
+#define SQ_FLAT_0__GLC__SHIFT 0x10
+#define SQ_FLAT_0__SLC_MASK 0x20000
+#define SQ_FLAT_0__SLC__SHIFT 0x11
+#define SQ_FLAT_0__OP_MASK 0x1fc0000
+#define SQ_FLAT_0__OP__SHIFT 0x12
+#define SQ_FLAT_0__ENCODING_MASK 0xfc000000
+#define SQ_FLAT_0__ENCODING__SHIFT 0x1a
+#define SQ_VOP3_0_SDST_ENC__VDST_MASK 0xff
+#define SQ_VOP3_0_SDST_ENC__VDST__SHIFT 0x0
+#define SQ_VOP3_0_SDST_ENC__SDST_MASK 0x7f00
+#define SQ_VOP3_0_SDST_ENC__SDST__SHIFT 0x8
+#define SQ_VOP3_0_SDST_ENC__CLAMP_MASK 0x8000
+#define SQ_VOP3_0_SDST_ENC__CLAMP__SHIFT 0xf
+#define SQ_VOP3_0_SDST_ENC__OP_MASK 0x3ff0000
+#define SQ_VOP3_0_SDST_ENC__OP__SHIFT 0x10
+#define SQ_VOP3_0_SDST_ENC__ENCODING_MASK 0xfc000000
+#define SQ_VOP3_0_SDST_ENC__ENCODING__SHIFT 0x1a
+#define SQ_MIMG_1__VADDR_MASK 0xff
+#define SQ_MIMG_1__VADDR__SHIFT 0x0
+#define SQ_MIMG_1__VDATA_MASK 0xff00
+#define SQ_MIMG_1__VDATA__SHIFT 0x8
+#define SQ_MIMG_1__SRSRC_MASK 0x1f0000
+#define SQ_MIMG_1__SRSRC__SHIFT 0x10
+#define SQ_MIMG_1__SSAMP_MASK 0x3e00000
+#define SQ_MIMG_1__SSAMP__SHIFT 0x15
+#define SQ_MIMG_1__D16_MASK 0x80000000
+#define SQ_MIMG_1__D16__SHIFT 0x1f
+#define SQ_SOP1__SSRC0_MASK 0xff
+#define SQ_SOP1__SSRC0__SHIFT 0x0
+#define SQ_SOP1__OP_MASK 0xff00
+#define SQ_SOP1__OP__SHIFT 0x8
+#define SQ_SOP1__SDST_MASK 0x7f0000
+#define SQ_SOP1__SDST__SHIFT 0x10
+#define SQ_SOP1__ENCODING_MASK 0xff800000
+#define SQ_SOP1__ENCODING__SHIFT 0x17
+#define SQ_SOPC__SSRC0_MASK 0xff
+#define SQ_SOPC__SSRC0__SHIFT 0x0
+#define SQ_SOPC__SSRC1_MASK 0xff00
+#define SQ_SOPC__SSRC1__SHIFT 0x8
+#define SQ_SOPC__OP_MASK 0x7f0000
+#define SQ_SOPC__OP__SHIFT 0x10
+#define SQ_SOPC__ENCODING_MASK 0xff800000
+#define SQ_SOPC__ENCODING__SHIFT 0x17
+#define SQ_FLAT_1__ADDR_MASK 0xff
+#define SQ_FLAT_1__ADDR__SHIFT 0x0
+#define SQ_FLAT_1__DATA_MASK 0xff00
+#define SQ_FLAT_1__DATA__SHIFT 0x8
+#define SQ_FLAT_1__TFE_MASK 0x800000
+#define SQ_FLAT_1__TFE__SHIFT 0x17
+#define SQ_FLAT_1__VDST_MASK 0xff000000
+#define SQ_FLAT_1__VDST__SHIFT 0x18
+#define SQ_DS_1__ADDR_MASK 0xff
+#define SQ_DS_1__ADDR__SHIFT 0x0
+#define SQ_DS_1__DATA0_MASK 0xff00
+#define SQ_DS_1__DATA0__SHIFT 0x8
+#define SQ_DS_1__DATA1_MASK 0xff0000
+#define SQ_DS_1__DATA1__SHIFT 0x10
+#define SQ_DS_1__VDST_MASK 0xff000000
+#define SQ_DS_1__VDST__SHIFT 0x18
+#define SQ_VOP3_1__SRC0_MASK 0x1ff
+#define SQ_VOP3_1__SRC0__SHIFT 0x0
+#define SQ_VOP3_1__SRC1_MASK 0x3fe00
+#define SQ_VOP3_1__SRC1__SHIFT 0x9
+#define SQ_VOP3_1__SRC2_MASK 0x7fc0000
+#define SQ_VOP3_1__SRC2__SHIFT 0x12
+#define SQ_VOP3_1__OMOD_MASK 0x18000000
+#define SQ_VOP3_1__OMOD__SHIFT 0x1b
+#define SQ_VOP3_1__NEG_MASK 0xe0000000
+#define SQ_VOP3_1__NEG__SHIFT 0x1d
+#define SQ_SMEM_0__SBASE_MASK 0x3f
+#define SQ_SMEM_0__SBASE__SHIFT 0x0
+#define SQ_SMEM_0__SDATA_MASK 0x1fc0
+#define SQ_SMEM_0__SDATA__SHIFT 0x6
+#define SQ_SMEM_0__GLC_MASK 0x10000
+#define SQ_SMEM_0__GLC__SHIFT 0x10
+#define SQ_SMEM_0__IMM_MASK 0x20000
+#define SQ_SMEM_0__IMM__SHIFT 0x11
+#define SQ_SMEM_0__OP_MASK 0x3fc0000
+#define SQ_SMEM_0__OP__SHIFT 0x12
+#define SQ_SMEM_0__ENCODING_MASK 0xfc000000
+#define SQ_SMEM_0__ENCODING__SHIFT 0x1a
+#define SQ_MIMG_0__DMASK_MASK 0xf00
+#define SQ_MIMG_0__DMASK__SHIFT 0x8
+#define SQ_MIMG_0__UNORM_MASK 0x1000
+#define SQ_MIMG_0__UNORM__SHIFT 0xc
+#define SQ_MIMG_0__GLC_MASK 0x2000
+#define SQ_MIMG_0__GLC__SHIFT 0xd
+#define SQ_MIMG_0__DA_MASK 0x4000
+#define SQ_MIMG_0__DA__SHIFT 0xe
+#define SQ_MIMG_0__R128_MASK 0x8000
+#define SQ_MIMG_0__R128__SHIFT 0xf
+#define SQ_MIMG_0__TFE_MASK 0x10000
+#define SQ_MIMG_0__TFE__SHIFT 0x10
+#define SQ_MIMG_0__LWE_MASK 0x20000
+#define SQ_MIMG_0__LWE__SHIFT 0x11
+#define SQ_MIMG_0__OP_MASK 0x1fc0000
+#define SQ_MIMG_0__OP__SHIFT 0x12
+#define SQ_MIMG_0__SLC_MASK 0x2000000
+#define SQ_MIMG_0__SLC__SHIFT 0x19
+#define SQ_MIMG_0__ENCODING_MASK 0xfc000000
+#define SQ_MIMG_0__ENCODING__SHIFT 0x1a
+#define SQ_SOPK__SIMM16_MASK 0xffff
+#define SQ_SOPK__SIMM16__SHIFT 0x0
+#define SQ_SOPK__SDST_MASK 0x7f0000
+#define SQ_SOPK__SDST__SHIFT 0x10
+#define SQ_SOPK__OP_MASK 0xf800000
+#define SQ_SOPK__OP__SHIFT 0x17
+#define SQ_SOPK__ENCODING_MASK 0xf0000000
+#define SQ_SOPK__ENCODING__SHIFT 0x1c
+#define SQ_DS_0__OFFSET0_MASK 0xff
+#define SQ_DS_0__OFFSET0__SHIFT 0x0
+#define SQ_DS_0__OFFSET1_MASK 0xff00
+#define SQ_DS_0__OFFSET1__SHIFT 0x8
+#define SQ_DS_0__GDS_MASK 0x10000
+#define SQ_DS_0__GDS__SHIFT 0x10
+#define SQ_DS_0__OP_MASK 0x1fe0000
+#define SQ_DS_0__OP__SHIFT 0x11
+#define SQ_DS_0__ENCODING_MASK 0xfc000000
+#define SQ_DS_0__ENCODING__SHIFT 0x1a
+#define SQ_VOP_DPP__SRC0_MASK 0xff
+#define SQ_VOP_DPP__SRC0__SHIFT 0x0
+#define SQ_VOP_DPP__DPP_CTRL_MASK 0x1ff00
+#define SQ_VOP_DPP__DPP_CTRL__SHIFT 0x8
+#define SQ_VOP_DPP__BOUND_CTRL_MASK 0x80000
+#define SQ_VOP_DPP__BOUND_CTRL__SHIFT 0x13
+#define SQ_VOP_DPP__SRC0_NEG_MASK 0x100000
+#define SQ_VOP_DPP__SRC0_NEG__SHIFT 0x14
+#define SQ_VOP_DPP__SRC0_ABS_MASK 0x200000
+#define SQ_VOP_DPP__SRC0_ABS__SHIFT 0x15
+#define SQ_VOP_DPP__SRC1_NEG_MASK 0x400000
+#define SQ_VOP_DPP__SRC1_NEG__SHIFT 0x16
+#define SQ_VOP_DPP__SRC1_ABS_MASK 0x800000
+#define SQ_VOP_DPP__SRC1_ABS__SHIFT 0x17
+#define SQ_VOP_DPP__BANK_MASK_MASK 0xf000000
+#define SQ_VOP_DPP__BANK_MASK__SHIFT 0x18
+#define SQ_VOP_DPP__ROW_MASK_MASK 0xf0000000
+#define SQ_VOP_DPP__ROW_MASK__SHIFT 0x1c
+#define SQ_VOPC__SRC0_MASK 0x1ff
+#define SQ_VOPC__SRC0__SHIFT 0x0
+#define SQ_VOPC__VSRC1_MASK 0x1fe00
+#define SQ_VOPC__VSRC1__SHIFT 0x9
+#define SQ_VOPC__OP_MASK 0x1fe0000
+#define SQ_VOPC__OP__SHIFT 0x11
+#define SQ_VOPC__ENCODING_MASK 0xfe000000
+#define SQ_VOPC__ENCODING__SHIFT 0x19
+#define SQ_VINTRP__VSRC_MASK 0xff
+#define SQ_VINTRP__VSRC__SHIFT 0x0
+#define SQ_VINTRP__ATTRCHAN_MASK 0x300
+#define SQ_VINTRP__ATTRCHAN__SHIFT 0x8
+#define SQ_VINTRP__ATTR_MASK 0xfc00
+#define SQ_VINTRP__ATTR__SHIFT 0xa
+#define SQ_VINTRP__OP_MASK 0x30000
+#define SQ_VINTRP__OP__SHIFT 0x10
+#define SQ_VINTRP__VDST_MASK 0x3fc0000
+#define SQ_VINTRP__VDST__SHIFT 0x12
+#define SQ_VINTRP__ENCODING_MASK 0xfc000000
+#define SQ_VINTRP__ENCODING__SHIFT 0x1a
+#define CGTT_SX_CLK_CTRL0__ON_DELAY_MASK 0xf
+#define CGTT_SX_CLK_CTRL0__ON_DELAY__SHIFT 0x0
+#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SX_CLK_CTRL0__RESERVED_MASK 0xfff000
+#define CGTT_SX_CLK_CTRL0__RESERVED__SHIFT 0xc
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x1f
+#define CGTT_SX_CLK_CTRL1__ON_DELAY_MASK 0xf
+#define CGTT_SX_CLK_CTRL1__ON_DELAY__SHIFT 0x0
+#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SX_CLK_CTRL1__RESERVED_MASK 0xfff000
+#define CGTT_SX_CLK_CTRL1__RESERVED__SHIFT 0xc
+#define CGTT_SX_CLK_CTRL1__DBG_EN_MASK 0x1000000
+#define CGTT_SX_CLK_CTRL1__DBG_EN__SHIFT 0x18
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0__SHIFT 0x1f
+#define CGTT_SX_CLK_CTRL2__ON_DELAY_MASK 0xf
+#define CGTT_SX_CLK_CTRL2__ON_DELAY__SHIFT 0x0
+#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SX_CLK_CTRL2__RESERVED_MASK 0xfff000
+#define CGTT_SX_CLK_CTRL2__RESERVED__SHIFT 0xc
+#define CGTT_SX_CLK_CTRL2__DBG_EN_MASK 0x1000000
+#define CGTT_SX_CLK_CTRL2__DBG_EN__SHIFT 0x18
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0__SHIFT 0x1f
+#define CGTT_SX_CLK_CTRL3__ON_DELAY_MASK 0xf
+#define CGTT_SX_CLK_CTRL3__ON_DELAY__SHIFT 0x0
+#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SX_CLK_CTRL3__RESERVED_MASK 0xfff000
+#define CGTT_SX_CLK_CTRL3__RESERVED__SHIFT 0xc
+#define CGTT_SX_CLK_CTRL3__DBG_EN_MASK 0x1000000
+#define CGTT_SX_CLK_CTRL3__DBG_EN__SHIFT 0x18
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0__SHIFT 0x1f
+#define CGTT_SX_CLK_CTRL4__ON_DELAY_MASK 0xf
+#define CGTT_SX_CLK_CTRL4__ON_DELAY__SHIFT 0x0
+#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_SX_CLK_CTRL4__RESERVED_MASK 0xfff000
+#define CGTT_SX_CLK_CTRL4__RESERVED__SHIFT 0xc
+#define CGTT_SX_CLK_CTRL4__DBG_EN_MASK 0x1000000
+#define CGTT_SX_CLK_CTRL4__DBG_EN__SHIFT 0x18
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0__SHIFT 0x1f
+#define SX_DEBUG_BUSY__POS_FREE_OR_VALIDS_MASK 0x1
+#define SX_DEBUG_BUSY__POS_FREE_OR_VALIDS__SHIFT 0x0
+#define SX_DEBUG_BUSY__POS_REQUESTER_BUSY_MASK 0x2
+#define SX_DEBUG_BUSY__POS_REQUESTER_BUSY__SHIFT 0x1
+#define SX_DEBUG_BUSY__PA_SX_BUSY_MASK 0x4
+#define SX_DEBUG_BUSY__PA_SX_BUSY__SHIFT 0x2
+#define SX_DEBUG_BUSY__POS_SCBD_BUSY_MASK 0x8
+#define SX_DEBUG_BUSY__POS_SCBD_BUSY__SHIFT 0x3
+#define SX_DEBUG_BUSY__POS_BANK3VAL3_BUSY_MASK 0x10
+#define SX_DEBUG_BUSY__POS_BANK3VAL3_BUSY__SHIFT 0x4
+#define SX_DEBUG_BUSY__POS_BANK3VAL2_BUSY_MASK 0x20
+#define SX_DEBUG_BUSY__POS_BANK3VAL2_BUSY__SHIFT 0x5
+#define SX_DEBUG_BUSY__POS_BANK3VAL1_BUSY_MASK 0x40
+#define SX_DEBUG_BUSY__POS_BANK3VAL1_BUSY__SHIFT 0x6
+#define SX_DEBUG_BUSY__POS_BANK3VAL0_BUSY_MASK 0x80
+#define SX_DEBUG_BUSY__POS_BANK3VAL0_BUSY__SHIFT 0x7
+#define SX_DEBUG_BUSY__POS_BANK2VAL3_BUSY_MASK 0x100
+#define SX_DEBUG_BUSY__POS_BANK2VAL3_BUSY__SHIFT 0x8
+#define SX_DEBUG_BUSY__POS_BANK2VAL2_BUSY_MASK 0x200
+#define SX_DEBUG_BUSY__POS_BANK2VAL2_BUSY__SHIFT 0x9
+#define SX_DEBUG_BUSY__POS_BANK2VAL1_BUSY_MASK 0x400
+#define SX_DEBUG_BUSY__POS_BANK2VAL1_BUSY__SHIFT 0xa
+#define SX_DEBUG_BUSY__POS_BANK2VAL0_BUSY_MASK 0x800
+#define SX_DEBUG_BUSY__POS_BANK2VAL0_BUSY__SHIFT 0xb
+#define SX_DEBUG_BUSY__POS_BANK1VAL3_BUSY_MASK 0x1000
+#define SX_DEBUG_BUSY__POS_BANK1VAL3_BUSY__SHIFT 0xc
+#define SX_DEBUG_BUSY__POS_BANK1VAL2_BUSY_MASK 0x2000
+#define SX_DEBUG_BUSY__POS_BANK1VAL2_BUSY__SHIFT 0xd
+#define SX_DEBUG_BUSY__POS_BANK1VAL1_BUSY_MASK 0x4000
+#define SX_DEBUG_BUSY__POS_BANK1VAL1_BUSY__SHIFT 0xe
+#define SX_DEBUG_BUSY__POS_BANK1VAL0_BUSY_MASK 0x8000
+#define SX_DEBUG_BUSY__POS_BANK1VAL0_BUSY__SHIFT 0xf
+#define SX_DEBUG_BUSY__POS_BANK0VAL3_BUSY_MASK 0x10000
+#define SX_DEBUG_BUSY__POS_BANK0VAL3_BUSY__SHIFT 0x10
+#define SX_DEBUG_BUSY__POS_BANK0VAL2_BUSY_MASK 0x20000
+#define SX_DEBUG_BUSY__POS_BANK0VAL2_BUSY__SHIFT 0x11
+#define SX_DEBUG_BUSY__POS_BANK0VAL1_BUSY_MASK 0x40000
+#define SX_DEBUG_BUSY__POS_BANK0VAL1_BUSY__SHIFT 0x12
+#define SX_DEBUG_BUSY__POS_BANK0VAL0_BUSY_MASK 0x80000
+#define SX_DEBUG_BUSY__POS_BANK0VAL0_BUSY__SHIFT 0x13
+#define SX_DEBUG_BUSY__POS_INMUX_VALID_MASK 0x100000
+#define SX_DEBUG_BUSY__POS_INMUX_VALID__SHIFT 0x14
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ3_MASK 0x200000
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ3__SHIFT 0x15
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ2_MASK 0x400000
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ2__SHIFT 0x16
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ1_MASK 0x800000
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ1__SHIFT 0x17
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ3_MASK 0x1000000
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ3__SHIFT 0x18
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ2_MASK 0x2000000
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ2__SHIFT 0x19
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ1_MASK 0x4000000
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ1__SHIFT 0x1a
+#define SX_DEBUG_BUSY__PCCMD_VALID_MASK 0x8000000
+#define SX_DEBUG_BUSY__PCCMD_VALID__SHIFT 0x1b
+#define SX_DEBUG_BUSY__VDATA1_VALID_MASK 0x10000000
+#define SX_DEBUG_BUSY__VDATA1_VALID__SHIFT 0x1c
+#define SX_DEBUG_BUSY__VDATA0_VALID_MASK 0x20000000
+#define SX_DEBUG_BUSY__VDATA0_VALID__SHIFT 0x1d
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL_MASK 0x40000000
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL__SHIFT 0x1e
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL_MASK 0x80000000
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL__SHIFT 0x1f
+#define SX_DEBUG_BUSY_2__COL_SCBD_BUSY_MASK 0x1
+#define SX_DEBUG_BUSY_2__COL_SCBD_BUSY__SHIFT 0x0
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0_MASK 0x2
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0__SHIFT 0x1
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE_MASK 0x4
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE__SHIFT 0x2
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY_MASK 0x8
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY__SHIFT 0x3
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0_MASK 0x10
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0__SHIFT 0x4
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE_MASK 0x20
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE__SHIFT 0x5
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY_MASK 0x40
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY__SHIFT 0x6
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0_MASK 0x80
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0__SHIFT 0x7
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE_MASK 0x100
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE__SHIFT 0x8
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY_MASK 0x200
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY__SHIFT 0x9
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0_MASK 0x400
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0__SHIFT 0xa
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE_MASK 0x800
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE__SHIFT 0xb
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY_MASK 0x1000
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY__SHIFT 0xc
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY_MASK 0x2000
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY__SHIFT 0xd
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY_MASK 0x4000
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY__SHIFT 0xe
+#define SX_DEBUG_BUSY_2__COL_DBIF3_READ_VALID_MASK 0x8000
+#define SX_DEBUG_BUSY_2__COL_DBIF3_READ_VALID__SHIFT 0xf
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY_MASK 0x10000
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY__SHIFT 0x10
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY_MASK 0x20000
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY__SHIFT 0x11
+#define SX_DEBUG_BUSY_2__COL_DBIF2_READ_VALID_MASK 0x40000
+#define SX_DEBUG_BUSY_2__COL_DBIF2_READ_VALID__SHIFT 0x12
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY_MASK 0x80000
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY__SHIFT 0x13
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY_MASK 0x100000
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY__SHIFT 0x14
+#define SX_DEBUG_BUSY_2__COL_DBIF1_READ_VALID_MASK 0x200000
+#define SX_DEBUG_BUSY_2__COL_DBIF1_READ_VALID__SHIFT 0x15
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY_MASK 0x400000
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY__SHIFT 0x16
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY_MASK 0x800000
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY__SHIFT 0x17
+#define SX_DEBUG_BUSY_2__COL_DBIF0_READ_VALID_MASK 0x1000000
+#define SX_DEBUG_BUSY_2__COL_DBIF0_READ_VALID__SHIFT 0x18
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY_MASK 0x2000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY__SHIFT 0x19
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY_MASK 0x4000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY__SHIFT 0x1a
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY_MASK 0x8000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY__SHIFT 0x1b
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY_MASK 0x10000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY__SHIFT 0x1c
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY_MASK 0x20000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY__SHIFT 0x1d
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY_MASK 0x40000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY__SHIFT 0x1e
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY_MASK 0x80000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY__SHIFT 0x1f
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY_MASK 0x1
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY__SHIFT 0x0
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY_MASK 0x2
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY__SHIFT 0x1
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY_MASK 0x4
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY__SHIFT 0x2
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY_MASK 0x8
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY__SHIFT 0x3
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY_MASK 0x10
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY__SHIFT 0x4
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY_MASK 0x20
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY__SHIFT 0x5
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY_MASK 0x40
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY__SHIFT 0x6
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY_MASK 0x80
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY__SHIFT 0x7
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY_MASK 0x100
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY__SHIFT 0x8
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY_MASK 0x200
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY__SHIFT 0x9
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY_MASK 0x400
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY__SHIFT 0xa
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY_MASK 0x800
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY__SHIFT 0xb
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY_MASK 0x1000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY__SHIFT 0xc
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY_MASK 0x2000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY__SHIFT 0xd
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY_MASK 0x4000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY__SHIFT 0xe
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY_MASK 0x8000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY__SHIFT 0xf
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY_MASK 0x10000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY__SHIFT 0x10
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY_MASK 0x20000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY__SHIFT 0x11
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY_MASK 0x40000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY__SHIFT 0x12
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY_MASK 0x80000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY__SHIFT 0x13
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY_MASK 0x100000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY__SHIFT 0x14
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY_MASK 0x200000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY__SHIFT 0x15
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY_MASK 0x400000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY__SHIFT 0x16
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY_MASK 0x800000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY__SHIFT 0x17
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY_MASK 0x1000000
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY__SHIFT 0x18
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY_MASK 0x2000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY__SHIFT 0x19
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY_MASK 0x4000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY__SHIFT 0x1a
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY_MASK 0x8000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY__SHIFT 0x1b
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY_MASK 0x10000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY__SHIFT 0x1c
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY_MASK 0x20000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY__SHIFT 0x1d
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY_MASK 0x40000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY__SHIFT 0x1e
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY_MASK 0x80000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY__SHIFT 0x1f
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY_MASK 0x1
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY__SHIFT 0x0
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY_MASK 0x2
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY__SHIFT 0x1
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY_MASK 0x4
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY__SHIFT 0x2
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY_MASK 0x8
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY__SHIFT 0x3
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY_MASK 0x10
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY__SHIFT 0x4
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY_MASK 0x20
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY__SHIFT 0x5
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY_MASK 0x40
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY__SHIFT 0x6
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY_MASK 0x80
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY__SHIFT 0x7
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY_MASK 0x100
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY__SHIFT 0x8
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY_MASK 0x200
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY__SHIFT 0x9
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY_MASK 0x400
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY__SHIFT 0xa
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY_MASK 0x800
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY__SHIFT 0xb
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY_MASK 0x1000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY__SHIFT 0xc
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY_MASK 0x2000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY__SHIFT 0xd
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY_MASK 0x4000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY__SHIFT 0xe
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY_MASK 0x8000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY__SHIFT 0xf
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY_MASK 0x10000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY__SHIFT 0x10
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY_MASK 0x20000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY__SHIFT 0x11
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY_MASK 0x40000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY__SHIFT 0x12
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY_MASK 0x80000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY__SHIFT 0x13
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY_MASK 0x100000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY__SHIFT 0x14
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY_MASK 0x200000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY__SHIFT 0x15
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY_MASK 0x400000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY__SHIFT 0x16
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY_MASK 0x800000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY__SHIFT 0x17
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY_MASK 0x1000000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY__SHIFT 0x18
+#define SX_DEBUG_BUSY_4__RESERVED_MASK 0xfe000000
+#define SX_DEBUG_BUSY_4__RESERVED__SHIFT 0x19
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT_MASK 0x7f
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT__SHIFT 0x0
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST_MASK 0x100
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT 0x8
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS_MASK 0x200
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS__SHIFT 0x9
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK 0x400
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT 0xa
+#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT_MASK 0x800
+#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT__SHIFT 0xb
+#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT_MASK 0x1000
+#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT__SHIFT 0xc
+#define SX_DEBUG_1__DEBUG_DATA_MASK 0xffffe000
+#define SX_DEBUG_1__DEBUG_DATA__SHIFT 0xd
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK 0x3ff
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x0
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK 0xffc00
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0xa
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2_MASK 0x3ff
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x0
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3_MASK 0xffc00
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0xa
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define SX_PS_DOWNCONVERT__MRT0_MASK 0xf
+#define SX_PS_DOWNCONVERT__MRT0__SHIFT 0x0
+#define SX_PS_DOWNCONVERT__MRT1_MASK 0xf0
+#define SX_PS_DOWNCONVERT__MRT1__SHIFT 0x4
+#define SX_PS_DOWNCONVERT__MRT2_MASK 0xf00
+#define SX_PS_DOWNCONVERT__MRT2__SHIFT 0x8
+#define SX_PS_DOWNCONVERT__MRT3_MASK 0xf000
+#define SX_PS_DOWNCONVERT__MRT3__SHIFT 0xc
+#define SX_PS_DOWNCONVERT__MRT4_MASK 0xf0000
+#define SX_PS_DOWNCONVERT__MRT4__SHIFT 0x10
+#define SX_PS_DOWNCONVERT__MRT5_MASK 0xf00000
+#define SX_PS_DOWNCONVERT__MRT5__SHIFT 0x14
+#define SX_PS_DOWNCONVERT__MRT6_MASK 0xf000000
+#define SX_PS_DOWNCONVERT__MRT6__SHIFT 0x18
+#define SX_PS_DOWNCONVERT__MRT7_MASK 0xf0000000
+#define SX_PS_DOWNCONVERT__MRT7__SHIFT 0x1c
+#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON_MASK 0xf
+#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON__SHIFT 0x0
+#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON_MASK 0xf0
+#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON__SHIFT 0x4
+#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON_MASK 0xf00
+#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON__SHIFT 0x8
+#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON_MASK 0xf000
+#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON__SHIFT 0xc
+#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON_MASK 0xf0000
+#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON__SHIFT 0x10
+#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON_MASK 0xf00000
+#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON__SHIFT 0x14
+#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON_MASK 0xf000000
+#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON__SHIFT 0x18
+#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON_MASK 0xf0000000
+#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON__SHIFT 0x1c
+#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE_MASK 0x1
+#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE__SHIFT 0x0
+#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE_MASK 0x2
+#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE__SHIFT 0x1
+#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE_MASK 0x10
+#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE__SHIFT 0x4
+#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE_MASK 0x20
+#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE__SHIFT 0x5
+#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE_MASK 0x100
+#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE__SHIFT 0x8
+#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE_MASK 0x200
+#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE__SHIFT 0x9
+#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE_MASK 0x1000
+#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE__SHIFT 0xc
+#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE_MASK 0x2000
+#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE__SHIFT 0xd
+#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE_MASK 0x10000
+#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE__SHIFT 0x10
+#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE_MASK 0x20000
+#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE__SHIFT 0x11
+#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE_MASK 0x100000
+#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE__SHIFT 0x14
+#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE_MASK 0x200000
+#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE__SHIFT 0x15
+#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE_MASK 0x1000000
+#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE__SHIFT 0x18
+#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE_MASK 0x2000000
+#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE__SHIFT 0x19
+#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE_MASK 0x10000000
+#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE__SHIFT 0x1c
+#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE_MASK 0x20000000
+#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE__SHIFT 0x1d
+#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE_MASK 0x80000000
+#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE__SHIFT 0x1f
+#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT_MASK 0x7
+#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT__SHIFT 0x0
+#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT_MASK 0x70
+#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT__SHIFT 0x4
+#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN_MASK 0x700
+#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN__SHIFT 0x8
+#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT_MASK 0x70000
+#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT__SHIFT 0x10
+#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT_MASK 0x700000
+#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT__SHIFT 0x14
+#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN_MASK 0x7000000
+#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN__SHIFT 0x18
+#define TCC_CTRL__CACHE_SIZE_MASK 0x3
+#define TCC_CTRL__CACHE_SIZE__SHIFT 0x0
+#define TCC_CTRL__RATE_MASK 0xc
+#define TCC_CTRL__RATE__SHIFT 0x2
+#define TCC_CTRL__WRITEBACK_MARGIN_MASK 0xf0
+#define TCC_CTRL__WRITEBACK_MARGIN__SHIFT 0x4
+#define TCC_CTRL__METADATA_LATENCY_FIFO_SIZE_MASK 0xf00
+#define TCC_CTRL__METADATA_LATENCY_FIFO_SIZE__SHIFT 0x8
+#define TCC_CTRL__SRC_FIFO_SIZE_MASK 0xf000
+#define TCC_CTRL__SRC_FIFO_SIZE__SHIFT 0xc
+#define TCC_CTRL__LATENCY_FIFO_SIZE_MASK 0xf0000
+#define TCC_CTRL__LATENCY_FIFO_SIZE__SHIFT 0x10
+#define TCC_CTRL__WB_OR_INV_ALL_VMIDS_MASK 0x100000
+#define TCC_CTRL__WB_OR_INV_ALL_VMIDS__SHIFT 0x14
+#define TCC_CTRL__MDC_SIZE_MASK 0x3000000
+#define TCC_CTRL__MDC_SIZE__SHIFT 0x18
+#define TCC_CTRL__MDC_SECTOR_SIZE_MASK 0xc000000
+#define TCC_CTRL__MDC_SECTOR_SIZE__SHIFT 0x1a
+#define TCC_CTRL__MDC_SIDEBAND_FIFO_SIZE_MASK 0xf0000000
+#define TCC_CTRL__MDC_SIDEBAND_FIFO_SIZE__SHIFT 0x1c
+#define TCC_EDC_CNT__SEC_COUNT_MASK 0xff
+#define TCC_EDC_CNT__SEC_COUNT__SHIFT 0x0
+#define TCC_EDC_CNT__DED_COUNT_MASK 0xff0000
+#define TCC_EDC_CNT__DED_COUNT__SHIFT 0x10
+#define TCC_REDUNDANCY__MC_SEL0_MASK 0x1
+#define TCC_REDUNDANCY__MC_SEL0__SHIFT 0x0
+#define TCC_REDUNDANCY__MC_SEL1_MASK 0x2
+#define TCC_REDUNDANCY__MC_SEL1__SHIFT 0x1
+#define TCC_EXE_DISABLE__EXE_DISABLE_MASK 0x2
+#define TCC_EXE_DISABLE__EXE_DISABLE__SHIFT 0x1
+#define TCC_DSM_CNTL__CACHE_RAM_IRRITATOR_DATA_SEL_MASK 0x3
+#define TCC_DSM_CNTL__CACHE_RAM_IRRITATOR_DATA_SEL__SHIFT 0x0
+#define TCC_DSM_CNTL__CACHE_RAM_IRRITATOR_SINGLE_WRITE_MASK 0x4
+#define TCC_DSM_CNTL__CACHE_RAM_IRRITATOR_SINGLE_WRITE__SHIFT 0x2
+#define TCC_CGTT_SCLK_CTRL__ON_DELAY_MASK 0xf
+#define TCC_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0
+#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define TCA_CGTT_SCLK_CTRL__ON_DELAY_MASK 0xf
+#define TCA_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0
+#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf000000
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x18
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf0000000
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x1c
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf000000
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x18
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf0000000
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x1c
+#define TCC_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff
+#define TCC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCC_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCC_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff
+#define TCC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCC_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCA_CTRL__HOLE_TIMEOUT_MASK 0xf
+#define TCA_CTRL__HOLE_TIMEOUT__SHIFT 0x0
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf000000
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x18
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf0000000
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x1c
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf000000
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x18
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf0000000
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x1c
+#define TCA_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff
+#define TCA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCA_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCA_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff
+#define TCA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCA_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TA_BC_BASE_ADDR__ADDRESS_MASK 0xffffffff
+#define TA_BC_BASE_ADDR__ADDRESS__SHIFT 0x0
+#define TA_BC_BASE_ADDR_HI__ADDRESS_MASK 0xff
+#define TA_BC_BASE_ADDR_HI__ADDRESS__SHIFT 0x0
+#define TD_CNTL__SYNC_PHASE_SH_MASK 0x3
+#define TD_CNTL__SYNC_PHASE_SH__SHIFT 0x0
+#define TD_CNTL__SYNC_PHASE_VC_SMX_MASK 0x30
+#define TD_CNTL__SYNC_PHASE_VC_SMX__SHIFT 0x4
+#define TD_CNTL__PAD_STALL_EN_MASK 0x100
+#define TD_CNTL__PAD_STALL_EN__SHIFT 0x8
+#define TD_CNTL__EXTEND_LDS_STALL_MASK 0x600
+#define TD_CNTL__EXTEND_LDS_STALL__SHIFT 0x9
+#define TD_CNTL__LDS_STALL_PHASE_ADJUST_MASK 0x1800
+#define TD_CNTL__LDS_STALL_PHASE_ADJUST__SHIFT 0xb
+#define TD_CNTL__PRECISION_COMPATIBILITY_MASK 0x8000
+#define TD_CNTL__PRECISION_COMPATIBILITY__SHIFT 0xf
+#define TD_CNTL__GATHER4_FLOAT_MODE_MASK 0x10000
+#define TD_CNTL__GATHER4_FLOAT_MODE__SHIFT 0x10
+#define TD_CNTL__LD_FLOAT_MODE_MASK 0x40000
+#define TD_CNTL__LD_FLOAT_MODE__SHIFT 0x12
+#define TD_CNTL__GATHER4_DX9_MODE_MASK 0x80000
+#define TD_CNTL__GATHER4_DX9_MODE__SHIFT 0x13
+#define TD_CNTL__DISABLE_POWER_THROTTLE_MASK 0x100000
+#define TD_CNTL__DISABLE_POWER_THROTTLE__SHIFT 0x14
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO_MASK 0x200000
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO__SHIFT 0x15
+#define TD_CNTL__DISABLE_D16_PACKING_MASK 0x400000
+#define TD_CNTL__DISABLE_D16_PACKING__SHIFT 0x16
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT_MASK 0x800000
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT__SHIFT 0x17
+#define TD_STATUS__BUSY_MASK 0x80000000
+#define TD_STATUS__BUSY__SHIFT 0x1f
+#define TD_DEBUG_INDEX__INDEX_MASK 0x1f
+#define TD_DEBUG_INDEX__INDEX__SHIFT 0x0
+#define TD_DEBUG_DATA__DATA_MASK 0xffffffff
+#define TD_DEBUG_DATA__DATA__SHIFT 0x0
+#define TD_DSM_CNTL__FORCE_SEDB_0_MASK 0x1
+#define TD_DSM_CNTL__FORCE_SEDB_0__SHIFT 0x0
+#define TD_DSM_CNTL__FORCE_SEDB_1_MASK 0x2
+#define TD_DSM_CNTL__FORCE_SEDB_1__SHIFT 0x1
+#define TD_DSM_CNTL__EN_SINGLE_WR_SEDB_MASK 0x4
+#define TD_DSM_CNTL__EN_SINGLE_WR_SEDB__SHIFT 0x2
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0xff
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x3fc00
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x3fc00
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0xff
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x3fc00
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TD_SCRATCH__SCRATCH_MASK 0xffffffff
+#define TD_SCRATCH__SCRATCH__SHIFT 0x0
+#define TA_CNTL__FX_XNACK_CREDIT_MASK 0x7f
+#define TA_CNTL__FX_XNACK_CREDIT__SHIFT 0x0
+#define TA_CNTL__SQ_XNACK_CREDIT_MASK 0x1e00
+#define TA_CNTL__SQ_XNACK_CREDIT__SHIFT 0x9
+#define TA_CNTL__TC_DATA_CREDIT_MASK 0xe000
+#define TA_CNTL__TC_DATA_CREDIT__SHIFT 0xd
+#define TA_CNTL__ALIGNER_CREDIT_MASK 0x1f0000
+#define TA_CNTL__ALIGNER_CREDIT__SHIFT 0x10
+#define TA_CNTL__TD_FIFO_CREDIT_MASK 0xffc00000
+#define TA_CNTL__TD_FIFO_CREDIT__SHIFT 0x16
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N_MASK 0x1
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N__SHIFT 0x0
+#define TA_CNTL_AUX__RESERVED_MASK 0xe
+#define TA_CNTL_AUX__RESERVED__SHIFT 0x1
+#define TA_CNTL_AUX__D16_PACK_DISABLE_MASK 0x10
+#define TA_CNTL_AUX__D16_PACK_DISABLE__SHIFT 0x4
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE_MASK 0x10000
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE__SHIFT 0x10
+#define TA_CNTL_AUX__ANISO_RATIO_LUT_MASK 0x20000
+#define TA_CNTL_AUX__ANISO_RATIO_LUT__SHIFT 0x11
+#define TA_CNTL_AUX__ANISO_TAP_MASK 0x40000
+#define TA_CNTL_AUX__ANISO_TAP__SHIFT 0x12
+#define TA_CNTL_AUX__ANISO_MIP_ADJ_MODE_MASK 0x80000
+#define TA_CNTL_AUX__ANISO_MIP_ADJ_MODE__SHIFT 0x13
+#define TA_RESERVED_010C__Unused_MASK 0xffffffff
+#define TA_RESERVED_010C__Unused__SHIFT 0x0
+#define TA_CS_BC_BASE_ADDR__ADDRESS_MASK 0xffffffff
+#define TA_CS_BC_BASE_ADDR__ADDRESS__SHIFT 0x0
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS_MASK 0xff
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS__SHIFT 0x0
+#define TA_STATUS__FG_PFIFO_EMPTYB_MASK 0x1000
+#define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT 0xc
+#define TA_STATUS__FG_LFIFO_EMPTYB_MASK 0x2000
+#define TA_STATUS__FG_LFIFO_EMPTYB__SHIFT 0xd
+#define TA_STATUS__FG_SFIFO_EMPTYB_MASK 0x4000
+#define TA_STATUS__FG_SFIFO_EMPTYB__SHIFT 0xe
+#define TA_STATUS__FL_PFIFO_EMPTYB_MASK 0x10000
+#define TA_STATUS__FL_PFIFO_EMPTYB__SHIFT 0x10
+#define TA_STATUS__FL_LFIFO_EMPTYB_MASK 0x20000
+#define TA_STATUS__FL_LFIFO_EMPTYB__SHIFT 0x11
+#define TA_STATUS__FL_SFIFO_EMPTYB_MASK 0x40000
+#define TA_STATUS__FL_SFIFO_EMPTYB__SHIFT 0x12
+#define TA_STATUS__FA_PFIFO_EMPTYB_MASK 0x100000
+#define TA_STATUS__FA_PFIFO_EMPTYB__SHIFT 0x14
+#define TA_STATUS__FA_LFIFO_EMPTYB_MASK 0x200000
+#define TA_STATUS__FA_LFIFO_EMPTYB__SHIFT 0x15
+#define TA_STATUS__FA_SFIFO_EMPTYB_MASK 0x400000
+#define TA_STATUS__FA_SFIFO_EMPTYB__SHIFT 0x16
+#define TA_STATUS__IN_BUSY_MASK 0x1000000
+#define TA_STATUS__IN_BUSY__SHIFT 0x18
+#define TA_STATUS__FG_BUSY_MASK 0x2000000
+#define TA_STATUS__FG_BUSY__SHIFT 0x19
+#define TA_STATUS__LA_BUSY_MASK 0x4000000
+#define TA_STATUS__LA_BUSY__SHIFT 0x1a
+#define TA_STATUS__FL_BUSY_MASK 0x8000000
+#define TA_STATUS__FL_BUSY__SHIFT 0x1b
+#define TA_STATUS__TA_BUSY_MASK 0x10000000
+#define TA_STATUS__TA_BUSY__SHIFT 0x1c
+#define TA_STATUS__FA_BUSY_MASK 0x20000000
+#define TA_STATUS__FA_BUSY__SHIFT 0x1d
+#define TA_STATUS__AL_BUSY_MASK 0x40000000
+#define TA_STATUS__AL_BUSY__SHIFT 0x1e
+#define TA_STATUS__BUSY_MASK 0x80000000
+#define TA_STATUS__BUSY__SHIFT 0x1f
+#define TA_DEBUG_INDEX__INDEX_MASK 0x1f
+#define TA_DEBUG_INDEX__INDEX__SHIFT 0x0
+#define TA_DEBUG_DATA__DATA_MASK 0xffffffff
+#define TA_DEBUG_DATA__DATA__SHIFT 0x0
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0xff
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x3fc00
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x3fc00
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0xff
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x3fc00
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TA_SCRATCH__SCRATCH_MASK 0xffffffff
+#define TA_SCRATCH__SCRATCH__SHIFT 0x0
+#define SH_HIDDEN_PRIVATE_BASE_VMID__ADDRESS_MASK 0xffffffff
+#define SH_HIDDEN_PRIVATE_BASE_VMID__ADDRESS__SHIFT 0x0
+#define SH_STATIC_MEM_CONFIG__SWIZZLE_ENABLE_MASK 0x1
+#define SH_STATIC_MEM_CONFIG__SWIZZLE_ENABLE__SHIFT 0x0
+#define SH_STATIC_MEM_CONFIG__ELEMENT_SIZE_MASK 0x6
+#define SH_STATIC_MEM_CONFIG__ELEMENT_SIZE__SHIFT 0x1
+#define SH_STATIC_MEM_CONFIG__INDEX_STRIDE_MASK 0x18
+#define SH_STATIC_MEM_CONFIG__INDEX_STRIDE__SHIFT 0x3
+#define SH_STATIC_MEM_CONFIG__PRIVATE_MTYPE_MASK 0xe0
+#define SH_STATIC_MEM_CONFIG__PRIVATE_MTYPE__SHIFT 0x5
+#define SH_STATIC_MEM_CONFIG__READ_ONLY_CNTL_MASK 0xff00
+#define SH_STATIC_MEM_CONFIG__READ_ONLY_CNTL__SHIFT 0x8
+#define TCP_INVALIDATE__START_MASK 0x1
+#define TCP_INVALIDATE__START__SHIFT 0x0
+#define TCP_STATUS__TCP_BUSY_MASK 0x1
+#define TCP_STATUS__TCP_BUSY__SHIFT 0x0
+#define TCP_STATUS__INPUT_BUSY_MASK 0x2
+#define TCP_STATUS__INPUT_BUSY__SHIFT 0x1
+#define TCP_STATUS__ADRS_BUSY_MASK 0x4
+#define TCP_STATUS__ADRS_BUSY__SHIFT 0x2
+#define TCP_STATUS__TAGRAMS_BUSY_MASK 0x8
+#define TCP_STATUS__TAGRAMS_BUSY__SHIFT 0x3
+#define TCP_STATUS__CNTRL_BUSY_MASK 0x10
+#define TCP_STATUS__CNTRL_BUSY__SHIFT 0x4
+#define TCP_STATUS__LFIFO_BUSY_MASK 0x20
+#define TCP_STATUS__LFIFO_BUSY__SHIFT 0x5
+#define TCP_STATUS__READ_BUSY_MASK 0x40
+#define TCP_STATUS__READ_BUSY__SHIFT 0x6
+#define TCP_STATUS__FORMAT_BUSY_MASK 0x80
+#define TCP_STATUS__FORMAT_BUSY__SHIFT 0x7
+#define TCP_CNTL__FORCE_HIT_MASK 0x1
+#define TCP_CNTL__FORCE_HIT__SHIFT 0x0
+#define TCP_CNTL__FORCE_MISS_MASK 0x2
+#define TCP_CNTL__FORCE_MISS__SHIFT 0x1
+#define TCP_CNTL__L1_SIZE_MASK 0xc
+#define TCP_CNTL__L1_SIZE__SHIFT 0x2
+#define TCP_CNTL__FLAT_BUF_HASH_ENABLE_MASK 0x10
+#define TCP_CNTL__FLAT_BUF_HASH_ENABLE__SHIFT 0x4
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE_MASK 0x20
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE__SHIFT 0x5
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT_MASK 0x1f8000
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT__SHIFT 0xf
+#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT_MASK 0xfc00000
+#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT__SHIFT 0x16
+#define TCP_CNTL__DISABLE_Z_MAP_MASK 0x10000000
+#define TCP_CNTL__DISABLE_Z_MAP__SHIFT 0x1c
+#define TCP_CNTL__INV_ALL_VMIDS_MASK 0x20000000
+#define TCP_CNTL__INV_ALL_VMIDS__SHIFT 0x1d
+#define TCP_CHAN_STEER_LO__CHAN0_MASK 0xf
+#define TCP_CHAN_STEER_LO__CHAN0__SHIFT 0x0
+#define TCP_CHAN_STEER_LO__CHAN1_MASK 0xf0
+#define TCP_CHAN_STEER_LO__CHAN1__SHIFT 0x4
+#define TCP_CHAN_STEER_LO__CHAN2_MASK 0xf00
+#define TCP_CHAN_STEER_LO__CHAN2__SHIFT 0x8
+#define TCP_CHAN_STEER_LO__CHAN3_MASK 0xf000
+#define TCP_CHAN_STEER_LO__CHAN3__SHIFT 0xc
+#define TCP_CHAN_STEER_LO__CHAN4_MASK 0xf0000
+#define TCP_CHAN_STEER_LO__CHAN4__SHIFT 0x10
+#define TCP_CHAN_STEER_LO__CHAN5_MASK 0xf00000
+#define TCP_CHAN_STEER_LO__CHAN5__SHIFT 0x14
+#define TCP_CHAN_STEER_LO__CHAN6_MASK 0xf000000
+#define TCP_CHAN_STEER_LO__CHAN6__SHIFT 0x18
+#define TCP_CHAN_STEER_LO__CHAN7_MASK 0xf0000000
+#define TCP_CHAN_STEER_LO__CHAN7__SHIFT 0x1c
+#define TCP_CHAN_STEER_HI__CHAN8_MASK 0xf
+#define TCP_CHAN_STEER_HI__CHAN8__SHIFT 0x0
+#define TCP_CHAN_STEER_HI__CHAN9_MASK 0xf0
+#define TCP_CHAN_STEER_HI__CHAN9__SHIFT 0x4
+#define TCP_CHAN_STEER_HI__CHANA_MASK 0xf00
+#define TCP_CHAN_STEER_HI__CHANA__SHIFT 0x8
+#define TCP_CHAN_STEER_HI__CHANB_MASK 0xf000
+#define TCP_CHAN_STEER_HI__CHANB__SHIFT 0xc
+#define TCP_CHAN_STEER_HI__CHANC_MASK 0xf0000
+#define TCP_CHAN_STEER_HI__CHANC__SHIFT 0x10
+#define TCP_CHAN_STEER_HI__CHAND_MASK 0xf00000
+#define TCP_CHAN_STEER_HI__CHAND__SHIFT 0x14
+#define TCP_CHAN_STEER_HI__CHANE_MASK 0xf000000
+#define TCP_CHAN_STEER_HI__CHANE__SHIFT 0x18
+#define TCP_CHAN_STEER_HI__CHANF_MASK 0xf0000000
+#define TCP_CHAN_STEER_HI__CHANF__SHIFT 0x1c
+#define TCP_ADDR_CONFIG__NUM_TCC_BANKS_MASK 0xf
+#define TCP_ADDR_CONFIG__NUM_TCC_BANKS__SHIFT 0x0
+#define TCP_ADDR_CONFIG__NUM_BANKS_MASK 0x30
+#define TCP_ADDR_CONFIG__NUM_BANKS__SHIFT 0x4
+#define TCP_ADDR_CONFIG__COLHI_WIDTH_MASK 0x1c0
+#define TCP_ADDR_CONFIG__COLHI_WIDTH__SHIFT 0x6
+#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI_MASK 0x200
+#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI__SHIFT 0x9
+#define TCP_CREDIT__LFIFO_CREDIT_MASK 0x3ff
+#define TCP_CREDIT__LFIFO_CREDIT__SHIFT 0x0
+#define TCP_CREDIT__REQ_FIFO_CREDIT_MASK 0x7f0000
+#define TCP_CREDIT__REQ_FIFO_CREDIT__SHIFT 0x10
+#define TCP_CREDIT__TD_CREDIT_MASK 0xe0000000
+#define TCP_CREDIT__TD_CREDIT__SHIFT 0x1d
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf000000
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x18
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS_MASK 0x7
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS__SHIFT 0x0
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS_MASK 0x700
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS__SHIFT 0x8
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT_MASK 0x70000
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT__SHIFT 0x10
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT_MASK 0x7000000
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT__SHIFT 0x18
+#define TCP_EDC_CNT__SEC_COUNT_MASK 0xff
+#define TCP_EDC_CNT__SEC_COUNT__SHIFT 0x0
+#define TCP_EDC_CNT__LFIFO_SED_COUNT_MASK 0xff00
+#define TCP_EDC_CNT__LFIFO_SED_COUNT__SHIFT 0x8
+#define TCP_EDC_CNT__DED_COUNT_MASK 0xff0000
+#define TCP_EDC_CNT__DED_COUNT__SHIFT 0x10
+#define TCP_EDC_CNT__UNUSED_MASK 0xff000000
+#define TCP_EDC_CNT__UNUSED__SHIFT 0x18
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_0_MASK 0x3
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_0__SHIFT 0x0
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_1_MASK 0xc
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_1__SHIFT 0x2
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_2_MASK 0x30
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_2__SHIFT 0x4
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_3_MASK 0xc0
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_3__SHIFT 0x6
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_4_MASK 0x300
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_4__SHIFT 0x8
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_5_MASK 0xc00
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_5__SHIFT 0xa
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_6_MASK 0x3000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_6__SHIFT 0xc
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_7_MASK 0xc000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_7__SHIFT 0xe
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_8_MASK 0x30000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_8__SHIFT 0x10
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_9_MASK 0xc0000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_9__SHIFT 0x12
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_10_MASK 0x300000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_10__SHIFT 0x14
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_11_MASK 0xc00000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_11__SHIFT 0x16
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_12_MASK 0x3000000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_12__SHIFT 0x18
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_13_MASK 0xc000000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_13__SHIFT 0x1a
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_14_MASK 0x30000000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_14__SHIFT 0x1c
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_15_MASK 0xc0000000
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_15__SHIFT 0x1e
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_16_MASK 0x3
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_16__SHIFT 0x0
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_17_MASK 0xc
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_17__SHIFT 0x2
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_18_MASK 0x30
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_18__SHIFT 0x4
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_19_MASK 0xc0
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_19__SHIFT 0x6
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_20_MASK 0x300
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_20__SHIFT 0x8
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_21_MASK 0xc00
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_21__SHIFT 0xa
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_22_MASK 0x3000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_22__SHIFT 0xc
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_23_MASK 0xc000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_23__SHIFT 0xe
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_24_MASK 0x30000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_24__SHIFT 0x10
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_25_MASK 0xc0000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_25__SHIFT 0x12
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_26_MASK 0x300000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_26__SHIFT 0x14
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_27_MASK 0xc00000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_27__SHIFT 0x16
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_28_MASK 0x3000000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_28__SHIFT 0x18
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_29_MASK 0xc000000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_29__SHIFT 0x1a
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_30_MASK 0x30000000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_30__SHIFT 0x1c
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_31_MASK 0xc0000000
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_31__SHIFT 0x1e
+#define TC_CFG_L1_STORE_POLICY__POLICY_0_MASK 0x1
+#define TC_CFG_L1_STORE_POLICY__POLICY_0__SHIFT 0x0
+#define TC_CFG_L1_STORE_POLICY__POLICY_1_MASK 0x2
+#define TC_CFG_L1_STORE_POLICY__POLICY_1__SHIFT 0x1
+#define TC_CFG_L1_STORE_POLICY__POLICY_2_MASK 0x4
+#define TC_CFG_L1_STORE_POLICY__POLICY_2__SHIFT 0x2
+#define TC_CFG_L1_STORE_POLICY__POLICY_3_MASK 0x8
+#define TC_CFG_L1_STORE_POLICY__POLICY_3__SHIFT 0x3
+#define TC_CFG_L1_STORE_POLICY__POLICY_4_MASK 0x10
+#define TC_CFG_L1_STORE_POLICY__POLICY_4__SHIFT 0x4
+#define TC_CFG_L1_STORE_POLICY__POLICY_5_MASK 0x20
+#define TC_CFG_L1_STORE_POLICY__POLICY_5__SHIFT 0x5
+#define TC_CFG_L1_STORE_POLICY__POLICY_6_MASK 0x40
+#define TC_CFG_L1_STORE_POLICY__POLICY_6__SHIFT 0x6
+#define TC_CFG_L1_STORE_POLICY__POLICY_7_MASK 0x80
+#define TC_CFG_L1_STORE_POLICY__POLICY_7__SHIFT 0x7
+#define TC_CFG_L1_STORE_POLICY__POLICY_8_MASK 0x100
+#define TC_CFG_L1_STORE_POLICY__POLICY_8__SHIFT 0x8
+#define TC_CFG_L1_STORE_POLICY__POLICY_9_MASK 0x200
+#define TC_CFG_L1_STORE_POLICY__POLICY_9__SHIFT 0x9
+#define TC_CFG_L1_STORE_POLICY__POLICY_10_MASK 0x400
+#define TC_CFG_L1_STORE_POLICY__POLICY_10__SHIFT 0xa
+#define TC_CFG_L1_STORE_POLICY__POLICY_11_MASK 0x800
+#define TC_CFG_L1_STORE_POLICY__POLICY_11__SHIFT 0xb
+#define TC_CFG_L1_STORE_POLICY__POLICY_12_MASK 0x1000
+#define TC_CFG_L1_STORE_POLICY__POLICY_12__SHIFT 0xc
+#define TC_CFG_L1_STORE_POLICY__POLICY_13_MASK 0x2000
+#define TC_CFG_L1_STORE_POLICY__POLICY_13__SHIFT 0xd
+#define TC_CFG_L1_STORE_POLICY__POLICY_14_MASK 0x4000
+#define TC_CFG_L1_STORE_POLICY__POLICY_14__SHIFT 0xe
+#define TC_CFG_L1_STORE_POLICY__POLICY_15_MASK 0x8000
+#define TC_CFG_L1_STORE_POLICY__POLICY_15__SHIFT 0xf
+#define TC_CFG_L1_STORE_POLICY__POLICY_16_MASK 0x10000
+#define TC_CFG_L1_STORE_POLICY__POLICY_16__SHIFT 0x10
+#define TC_CFG_L1_STORE_POLICY__POLICY_17_MASK 0x20000
+#define TC_CFG_L1_STORE_POLICY__POLICY_17__SHIFT 0x11
+#define TC_CFG_L1_STORE_POLICY__POLICY_18_MASK 0x40000
+#define TC_CFG_L1_STORE_POLICY__POLICY_18__SHIFT 0x12
+#define TC_CFG_L1_STORE_POLICY__POLICY_19_MASK 0x80000
+#define TC_CFG_L1_STORE_POLICY__POLICY_19__SHIFT 0x13
+#define TC_CFG_L1_STORE_POLICY__POLICY_20_MASK 0x100000
+#define TC_CFG_L1_STORE_POLICY__POLICY_20__SHIFT 0x14
+#define TC_CFG_L1_STORE_POLICY__POLICY_21_MASK 0x200000
+#define TC_CFG_L1_STORE_POLICY__POLICY_21__SHIFT 0x15
+#define TC_CFG_L1_STORE_POLICY__POLICY_22_MASK 0x400000
+#define TC_CFG_L1_STORE_POLICY__POLICY_22__SHIFT 0x16
+#define TC_CFG_L1_STORE_POLICY__POLICY_23_MASK 0x800000
+#define TC_CFG_L1_STORE_POLICY__POLICY_23__SHIFT 0x17
+#define TC_CFG_L1_STORE_POLICY__POLICY_24_MASK 0x1000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_24__SHIFT 0x18
+#define TC_CFG_L1_STORE_POLICY__POLICY_25_MASK 0x2000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_25__SHIFT 0x19
+#define TC_CFG_L1_STORE_POLICY__POLICY_26_MASK 0x4000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_26__SHIFT 0x1a
+#define TC_CFG_L1_STORE_POLICY__POLICY_27_MASK 0x8000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_27__SHIFT 0x1b
+#define TC_CFG_L1_STORE_POLICY__POLICY_28_MASK 0x10000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_28__SHIFT 0x1c
+#define TC_CFG_L1_STORE_POLICY__POLICY_29_MASK 0x20000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_29__SHIFT 0x1d
+#define TC_CFG_L1_STORE_POLICY__POLICY_30_MASK 0x40000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_30__SHIFT 0x1e
+#define TC_CFG_L1_STORE_POLICY__POLICY_31_MASK 0x80000000
+#define TC_CFG_L1_STORE_POLICY__POLICY_31__SHIFT 0x1f
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_0_MASK 0x3
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_0__SHIFT 0x0
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_1_MASK 0xc
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_1__SHIFT 0x2
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_2_MASK 0x30
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_2__SHIFT 0x4
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_3_MASK 0xc0
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_3__SHIFT 0x6
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_4_MASK 0x300
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_4__SHIFT 0x8
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_5_MASK 0xc00
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_5__SHIFT 0xa
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_6_MASK 0x3000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_6__SHIFT 0xc
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_7_MASK 0xc000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_7__SHIFT 0xe
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_8_MASK 0x30000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_8__SHIFT 0x10
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_9_MASK 0xc0000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_9__SHIFT 0x12
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_10_MASK 0x300000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_10__SHIFT 0x14
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_11_MASK 0xc00000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_11__SHIFT 0x16
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_12_MASK 0x3000000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_12__SHIFT 0x18
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_13_MASK 0xc000000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_13__SHIFT 0x1a
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_14_MASK 0x30000000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_14__SHIFT 0x1c
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_15_MASK 0xc0000000
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_15__SHIFT 0x1e
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_16_MASK 0x3
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_16__SHIFT 0x0
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_17_MASK 0xc
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_17__SHIFT 0x2
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_18_MASK 0x30
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_18__SHIFT 0x4
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_19_MASK 0xc0
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_19__SHIFT 0x6
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_20_MASK 0x300
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_20__SHIFT 0x8
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_21_MASK 0xc00
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_21__SHIFT 0xa
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_22_MASK 0x3000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_22__SHIFT 0xc
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_23_MASK 0xc000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_23__SHIFT 0xe
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_24_MASK 0x30000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_24__SHIFT 0x10
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_25_MASK 0xc0000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_25__SHIFT 0x12
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_26_MASK 0x300000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_26__SHIFT 0x14
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_27_MASK 0xc00000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_27__SHIFT 0x16
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_28_MASK 0x3000000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_28__SHIFT 0x18
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_29_MASK 0xc000000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_29__SHIFT 0x1a
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_30_MASK 0x30000000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_30__SHIFT 0x1c
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_31_MASK 0xc0000000
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_31__SHIFT 0x1e
+#define TC_CFG_L2_STORE_POLICY0__POLICY_0_MASK 0x3
+#define TC_CFG_L2_STORE_POLICY0__POLICY_0__SHIFT 0x0
+#define TC_CFG_L2_STORE_POLICY0__POLICY_1_MASK 0xc
+#define TC_CFG_L2_STORE_POLICY0__POLICY_1__SHIFT 0x2
+#define TC_CFG_L2_STORE_POLICY0__POLICY_2_MASK 0x30
+#define TC_CFG_L2_STORE_POLICY0__POLICY_2__SHIFT 0x4
+#define TC_CFG_L2_STORE_POLICY0__POLICY_3_MASK 0xc0
+#define TC_CFG_L2_STORE_POLICY0__POLICY_3__SHIFT 0x6
+#define TC_CFG_L2_STORE_POLICY0__POLICY_4_MASK 0x300
+#define TC_CFG_L2_STORE_POLICY0__POLICY_4__SHIFT 0x8
+#define TC_CFG_L2_STORE_POLICY0__POLICY_5_MASK 0xc00
+#define TC_CFG_L2_STORE_POLICY0__POLICY_5__SHIFT 0xa
+#define TC_CFG_L2_STORE_POLICY0__POLICY_6_MASK 0x3000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_6__SHIFT 0xc
+#define TC_CFG_L2_STORE_POLICY0__POLICY_7_MASK 0xc000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_7__SHIFT 0xe
+#define TC_CFG_L2_STORE_POLICY0__POLICY_8_MASK 0x30000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_8__SHIFT 0x10
+#define TC_CFG_L2_STORE_POLICY0__POLICY_9_MASK 0xc0000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_9__SHIFT 0x12
+#define TC_CFG_L2_STORE_POLICY0__POLICY_10_MASK 0x300000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_10__SHIFT 0x14
+#define TC_CFG_L2_STORE_POLICY0__POLICY_11_MASK 0xc00000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_11__SHIFT 0x16
+#define TC_CFG_L2_STORE_POLICY0__POLICY_12_MASK 0x3000000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_12__SHIFT 0x18
+#define TC_CFG_L2_STORE_POLICY0__POLICY_13_MASK 0xc000000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_13__SHIFT 0x1a
+#define TC_CFG_L2_STORE_POLICY0__POLICY_14_MASK 0x30000000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_14__SHIFT 0x1c
+#define TC_CFG_L2_STORE_POLICY0__POLICY_15_MASK 0xc0000000
+#define TC_CFG_L2_STORE_POLICY0__POLICY_15__SHIFT 0x1e
+#define TC_CFG_L2_STORE_POLICY1__POLICY_16_MASK 0x3
+#define TC_CFG_L2_STORE_POLICY1__POLICY_16__SHIFT 0x0
+#define TC_CFG_L2_STORE_POLICY1__POLICY_17_MASK 0xc
+#define TC_CFG_L2_STORE_POLICY1__POLICY_17__SHIFT 0x2
+#define TC_CFG_L2_STORE_POLICY1__POLICY_18_MASK 0x30
+#define TC_CFG_L2_STORE_POLICY1__POLICY_18__SHIFT 0x4
+#define TC_CFG_L2_STORE_POLICY1__POLICY_19_MASK 0xc0
+#define TC_CFG_L2_STORE_POLICY1__POLICY_19__SHIFT 0x6
+#define TC_CFG_L2_STORE_POLICY1__POLICY_20_MASK 0x300
+#define TC_CFG_L2_STORE_POLICY1__POLICY_20__SHIFT 0x8
+#define TC_CFG_L2_STORE_POLICY1__POLICY_21_MASK 0xc00
+#define TC_CFG_L2_STORE_POLICY1__POLICY_21__SHIFT 0xa
+#define TC_CFG_L2_STORE_POLICY1__POLICY_22_MASK 0x3000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_22__SHIFT 0xc
+#define TC_CFG_L2_STORE_POLICY1__POLICY_23_MASK 0xc000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_23__SHIFT 0xe
+#define TC_CFG_L2_STORE_POLICY1__POLICY_24_MASK 0x30000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_24__SHIFT 0x10
+#define TC_CFG_L2_STORE_POLICY1__POLICY_25_MASK 0xc0000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_25__SHIFT 0x12
+#define TC_CFG_L2_STORE_POLICY1__POLICY_26_MASK 0x300000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_26__SHIFT 0x14
+#define TC_CFG_L2_STORE_POLICY1__POLICY_27_MASK 0xc00000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_27__SHIFT 0x16
+#define TC_CFG_L2_STORE_POLICY1__POLICY_28_MASK 0x3000000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_28__SHIFT 0x18
+#define TC_CFG_L2_STORE_POLICY1__POLICY_29_MASK 0xc000000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_29__SHIFT 0x1a
+#define TC_CFG_L2_STORE_POLICY1__POLICY_30_MASK 0x30000000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_30__SHIFT 0x1c
+#define TC_CFG_L2_STORE_POLICY1__POLICY_31_MASK 0xc0000000
+#define TC_CFG_L2_STORE_POLICY1__POLICY_31__SHIFT 0x1e
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0_MASK 0x3
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0__SHIFT 0x0
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1_MASK 0xc
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1__SHIFT 0x2
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2_MASK 0x30
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2__SHIFT 0x4
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3_MASK 0xc0
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3__SHIFT 0x6
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4_MASK 0x300
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4__SHIFT 0x8
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5_MASK 0xc00
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5__SHIFT 0xa
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6_MASK 0x3000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6__SHIFT 0xc
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7_MASK 0xc000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7__SHIFT 0xe
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8_MASK 0x30000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8__SHIFT 0x10
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9_MASK 0xc0000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9__SHIFT 0x12
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10_MASK 0x300000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10__SHIFT 0x14
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11_MASK 0xc00000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11__SHIFT 0x16
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12_MASK 0x3000000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12__SHIFT 0x18
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13_MASK 0xc000000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13__SHIFT 0x1a
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14_MASK 0x30000000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14__SHIFT 0x1c
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15_MASK 0xc0000000
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15__SHIFT 0x1e
+#define TC_CFG_L1_VOLATILE__VOL_MASK 0xf
+#define TC_CFG_L1_VOLATILE__VOL__SHIFT 0x0
+#define TC_CFG_L2_VOLATILE__VOL_MASK 0xf
+#define TC_CFG_L2_VOLATILE__VOL__SHIFT 0x0
+#define TCP_WATCH0_ADDR_H__ADDR_MASK 0xffff
+#define TCP_WATCH0_ADDR_H__ADDR__SHIFT 0x0
+#define TCP_WATCH1_ADDR_H__ADDR_MASK 0xffff
+#define TCP_WATCH1_ADDR_H__ADDR__SHIFT 0x0
+#define TCP_WATCH2_ADDR_H__ADDR_MASK 0xffff
+#define TCP_WATCH2_ADDR_H__ADDR__SHIFT 0x0
+#define TCP_WATCH3_ADDR_H__ADDR_MASK 0xffff
+#define TCP_WATCH3_ADDR_H__ADDR__SHIFT 0x0
+#define TCP_WATCH0_ADDR_L__ADDR_MASK 0xffffffc0
+#define TCP_WATCH0_ADDR_L__ADDR__SHIFT 0x6
+#define TCP_WATCH1_ADDR_L__ADDR_MASK 0xffffffc0
+#define TCP_WATCH1_ADDR_L__ADDR__SHIFT 0x6
+#define TCP_WATCH2_ADDR_L__ADDR_MASK 0xffffffc0
+#define TCP_WATCH2_ADDR_L__ADDR__SHIFT 0x6
+#define TCP_WATCH3_ADDR_L__ADDR_MASK 0xffffffc0
+#define TCP_WATCH3_ADDR_L__ADDR__SHIFT 0x6
+#define TCP_WATCH0_CNTL__MASK_MASK 0xffffff
+#define TCP_WATCH0_CNTL__MASK__SHIFT 0x0
+#define TCP_WATCH0_CNTL__VMID_MASK 0xf000000
+#define TCP_WATCH0_CNTL__VMID__SHIFT 0x18
+#define TCP_WATCH0_CNTL__ATC_MASK 0x10000000
+#define TCP_WATCH0_CNTL__ATC__SHIFT 0x1c
+#define TCP_WATCH0_CNTL__MODE_MASK 0x60000000
+#define TCP_WATCH0_CNTL__MODE__SHIFT 0x1d
+#define TCP_WATCH0_CNTL__VALID_MASK 0x80000000
+#define TCP_WATCH0_CNTL__VALID__SHIFT 0x1f
+#define TCP_WATCH1_CNTL__MASK_MASK 0xffffff
+#define TCP_WATCH1_CNTL__MASK__SHIFT 0x0
+#define TCP_WATCH1_CNTL__VMID_MASK 0xf000000
+#define TCP_WATCH1_CNTL__VMID__SHIFT 0x18
+#define TCP_WATCH1_CNTL__ATC_MASK 0x10000000
+#define TCP_WATCH1_CNTL__ATC__SHIFT 0x1c
+#define TCP_WATCH1_CNTL__MODE_MASK 0x60000000
+#define TCP_WATCH1_CNTL__MODE__SHIFT 0x1d
+#define TCP_WATCH1_CNTL__VALID_MASK 0x80000000
+#define TCP_WATCH1_CNTL__VALID__SHIFT 0x1f
+#define TCP_WATCH2_CNTL__MASK_MASK 0xffffff
+#define TCP_WATCH2_CNTL__MASK__SHIFT 0x0
+#define TCP_WATCH2_CNTL__VMID_MASK 0xf000000
+#define TCP_WATCH2_CNTL__VMID__SHIFT 0x18
+#define TCP_WATCH2_CNTL__ATC_MASK 0x10000000
+#define TCP_WATCH2_CNTL__ATC__SHIFT 0x1c
+#define TCP_WATCH2_CNTL__MODE_MASK 0x60000000
+#define TCP_WATCH2_CNTL__MODE__SHIFT 0x1d
+#define TCP_WATCH2_CNTL__VALID_MASK 0x80000000
+#define TCP_WATCH2_CNTL__VALID__SHIFT 0x1f
+#define TCP_WATCH3_CNTL__MASK_MASK 0xffffff
+#define TCP_WATCH3_CNTL__MASK__SHIFT 0x0
+#define TCP_WATCH3_CNTL__VMID_MASK 0xf000000
+#define TCP_WATCH3_CNTL__VMID__SHIFT 0x18
+#define TCP_WATCH3_CNTL__ATC_MASK 0x10000000
+#define TCP_WATCH3_CNTL__ATC__SHIFT 0x1c
+#define TCP_WATCH3_CNTL__MODE_MASK 0x60000000
+#define TCP_WATCH3_CNTL__MODE__SHIFT 0x1d
+#define TCP_WATCH3_CNTL__VALID_MASK 0x80000000
+#define TCP_WATCH3_CNTL__VALID__SHIFT 0x1f
+#define TCP_GATCL1_CNTL__INVALIDATE_ALL_VMID_MASK 0x2000000
+#define TCP_GATCL1_CNTL__INVALIDATE_ALL_VMID__SHIFT 0x19
+#define TCP_GATCL1_CNTL__FORCE_MISS_MASK 0x4000000
+#define TCP_GATCL1_CNTL__FORCE_MISS__SHIFT 0x1a
+#define TCP_GATCL1_CNTL__FORCE_IN_ORDER_MASK 0x8000000
+#define TCP_GATCL1_CNTL__FORCE_IN_ORDER__SHIFT 0x1b
+#define TCP_GATCL1_CNTL__REDUCE_FIFO_DEPTH_BY_2_MASK 0x30000000
+#define TCP_GATCL1_CNTL__REDUCE_FIFO_DEPTH_BY_2__SHIFT 0x1c
+#define TCP_GATCL1_CNTL__REDUCE_CACHE_SIZE_BY_2_MASK 0xc0000000
+#define TCP_GATCL1_CNTL__REDUCE_CACHE_SIZE_BY_2__SHIFT 0x1e
+#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC_MASK 0xff
+#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC__SHIFT 0x0
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A0_MASK 0x1
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A0__SHIFT 0x0
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A1_MASK 0x2
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A1__SHIFT 0x1
+#define TCP_GATCL1_DSM_CNTL__TCP_GATCL1_ENABLE_SINGLE_WRITE_A_MASK 0x4
+#define TCP_GATCL1_DSM_CNTL__TCP_GATCL1_ENABLE_SINGLE_WRITE_A__SHIFT 0x2
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_DATA_SEL_MASK 0x3
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_DATA_SEL__SHIFT 0x0
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_SINGLE_WRITE_MASK 0x4
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_SINGLE_WRITE__SHIFT 0x2
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_DATA_SEL_MASK 0x18
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_DATA_SEL__SHIFT 0x3
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_SINGLE_WRITE_MASK 0x20
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_SINGLE_WRITE__SHIFT 0x5
+#define TCP_CNTL2__LS_DISABLE_CLOCKS_MASK 0xff
+#define TCP_CNTL2__LS_DISABLE_CLOCKS__SHIFT 0x0
+#define TD_CGTT_CTRL__ON_DELAY_MASK 0xf
+#define TD_CGTT_CTRL__ON_DELAY__SHIFT 0x0
+#define TD_CGTT_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define TD_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define TA_CGTT_CTRL__ON_DELAY_MASK 0xf
+#define TA_CGTT_CTRL__ON_DELAY__SHIFT 0x0
+#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define CGTT_TCP_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_TCP_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_TCP_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_TCP_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define CGTT_TCI_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_TCI_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define TCI_STATUS__TCI_BUSY_MASK 0x1
+#define TCI_STATUS__TCI_BUSY__SHIFT 0x0
+#define TCI_CNTL_1__WBINVL1_NUM_CYCLES_MASK 0xffff
+#define TCI_CNTL_1__WBINVL1_NUM_CYCLES__SHIFT 0x0
+#define TCI_CNTL_1__REQ_FIFO_DEPTH_MASK 0xff0000
+#define TCI_CNTL_1__REQ_FIFO_DEPTH__SHIFT 0x10
+#define TCI_CNTL_1__WDATA_RAM_DEPTH_MASK 0xff000000
+#define TCI_CNTL_1__WDATA_RAM_DEPTH__SHIFT 0x18
+#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2_MASK 0x1
+#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2__SHIFT 0x0
+#define TCI_CNTL_2__TCA_MAX_CREDIT_MASK 0x1fe
+#define TCI_CNTL_2__TCA_MAX_CREDIT__SHIFT 0x1
+#define GDS_CONFIG__SH0_GPR_PHASE_SEL_MASK 0x6
+#define GDS_CONFIG__SH0_GPR_PHASE_SEL__SHIFT 0x1
+#define GDS_CONFIG__SH1_GPR_PHASE_SEL_MASK 0x18
+#define GDS_CONFIG__SH1_GPR_PHASE_SEL__SHIFT 0x3
+#define GDS_CONFIG__SH2_GPR_PHASE_SEL_MASK 0x60
+#define GDS_CONFIG__SH2_GPR_PHASE_SEL__SHIFT 0x5
+#define GDS_CONFIG__SH3_GPR_PHASE_SEL_MASK 0x180
+#define GDS_CONFIG__SH3_GPR_PHASE_SEL__SHIFT 0x7
+#define GDS_CNTL_STATUS__GDS_BUSY_MASK 0x1
+#define GDS_CNTL_STATUS__GDS_BUSY__SHIFT 0x0
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY_MASK 0x2
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY__SHIFT 0x1
+#define GDS_CNTL_STATUS__ORD_APP_BUSY_MASK 0x4
+#define GDS_CNTL_STATUS__ORD_APP_BUSY__SHIFT 0x2
+#define GDS_CNTL_STATUS__DS_BANK_CONFLICT_MASK 0x8
+#define GDS_CNTL_STATUS__DS_BANK_CONFLICT__SHIFT 0x3
+#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT_MASK 0x10
+#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT__SHIFT 0x4
+#define GDS_CNTL_STATUS__DS_WR_CLAMP_MASK 0x20
+#define GDS_CNTL_STATUS__DS_WR_CLAMP__SHIFT 0x5
+#define GDS_CNTL_STATUS__DS_RD_CLAMP_MASK 0x40
+#define GDS_CNTL_STATUS__DS_RD_CLAMP__SHIFT 0x6
+#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY_MASK 0x80
+#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY__SHIFT 0x7
+#define GDS_CNTL_STATUS__DS_BUSY_MASK 0x100
+#define GDS_CNTL_STATUS__DS_BUSY__SHIFT 0x8
+#define GDS_CNTL_STATUS__GWS_BUSY_MASK 0x200
+#define GDS_CNTL_STATUS__GWS_BUSY__SHIFT 0x9
+#define GDS_CNTL_STATUS__ORD_FIFO_BUSY_MASK 0x400
+#define GDS_CNTL_STATUS__ORD_FIFO_BUSY__SHIFT 0xa
+#define GDS_CNTL_STATUS__CREDIT_BUSY0_MASK 0x800
+#define GDS_CNTL_STATUS__CREDIT_BUSY0__SHIFT 0xb
+#define GDS_CNTL_STATUS__CREDIT_BUSY1_MASK 0x1000
+#define GDS_CNTL_STATUS__CREDIT_BUSY1__SHIFT 0xc
+#define GDS_CNTL_STATUS__CREDIT_BUSY2_MASK 0x2000
+#define GDS_CNTL_STATUS__CREDIT_BUSY2__SHIFT 0xd
+#define GDS_CNTL_STATUS__CREDIT_BUSY3_MASK 0x4000
+#define GDS_CNTL_STATUS__CREDIT_BUSY3__SHIFT 0xe
+#define GDS_ENHANCE2__MISC_MASK 0xffff
+#define GDS_ENHANCE2__MISC__SHIFT 0x0
+#define GDS_ENHANCE2__UNUSED_MASK 0xffff0000
+#define GDS_ENHANCE2__UNUSED__SHIFT 0x10
+#define GDS_PROTECTION_FAULT__WRITE_DIS_MASK 0x1
+#define GDS_PROTECTION_FAULT__WRITE_DIS__SHIFT 0x0
+#define GDS_PROTECTION_FAULT__FAULT_DETECTED_MASK 0x2
+#define GDS_PROTECTION_FAULT__FAULT_DETECTED__SHIFT 0x1
+#define GDS_PROTECTION_FAULT__GRBM_MASK 0x4
+#define GDS_PROTECTION_FAULT__GRBM__SHIFT 0x2
+#define GDS_PROTECTION_FAULT__SH_ID_MASK 0x38
+#define GDS_PROTECTION_FAULT__SH_ID__SHIFT 0x3
+#define GDS_PROTECTION_FAULT__CU_ID_MASK 0x3c0
+#define GDS_PROTECTION_FAULT__CU_ID__SHIFT 0x6
+#define GDS_PROTECTION_FAULT__SIMD_ID_MASK 0xc00
+#define GDS_PROTECTION_FAULT__SIMD_ID__SHIFT 0xa
+#define GDS_PROTECTION_FAULT__WAVE_ID_MASK 0xf000
+#define GDS_PROTECTION_FAULT__WAVE_ID__SHIFT 0xc
+#define GDS_PROTECTION_FAULT__ADDRESS_MASK 0xffff0000
+#define GDS_PROTECTION_FAULT__ADDRESS__SHIFT 0x10
+#define GDS_VM_PROTECTION_FAULT__WRITE_DIS_MASK 0x1
+#define GDS_VM_PROTECTION_FAULT__WRITE_DIS__SHIFT 0x0
+#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED_MASK 0x2
+#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED__SHIFT 0x1
+#define GDS_VM_PROTECTION_FAULT__GWS_MASK 0x4
+#define GDS_VM_PROTECTION_FAULT__GWS__SHIFT 0x2
+#define GDS_VM_PROTECTION_FAULT__OA_MASK 0x8
+#define GDS_VM_PROTECTION_FAULT__OA__SHIFT 0x3
+#define GDS_VM_PROTECTION_FAULT__GRBM_MASK 0x10
+#define GDS_VM_PROTECTION_FAULT__GRBM__SHIFT 0x4
+#define GDS_VM_PROTECTION_FAULT__VMID_MASK 0xf00
+#define GDS_VM_PROTECTION_FAULT__VMID__SHIFT 0x8
+#define GDS_VM_PROTECTION_FAULT__ADDRESS_MASK 0xffff0000
+#define GDS_VM_PROTECTION_FAULT__ADDRESS__SHIFT 0x10
+#define GDS_EDC_CNT__DED_MASK 0xff
+#define GDS_EDC_CNT__DED__SHIFT 0x0
+#define GDS_EDC_CNT__SED_MASK 0xff00
+#define GDS_EDC_CNT__SED__SHIFT 0x8
+#define GDS_EDC_CNT__SEC_MASK 0xff0000
+#define GDS_EDC_CNT__SEC__SHIFT 0x10
+#define GDS_EDC_GRBM_CNT__DED_MASK 0xff
+#define GDS_EDC_GRBM_CNT__DED__SHIFT 0x0
+#define GDS_EDC_GRBM_CNT__SEC_MASK 0xff0000
+#define GDS_EDC_GRBM_CNT__SEC__SHIFT 0x10
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED_MASK 0x1
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED__SHIFT 0x0
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED_MASK 0x2
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED__SHIFT 0x1
+#define GDS_EDC_OA_DED__ME0_CS_DED_MASK 0x4
+#define GDS_EDC_OA_DED__ME0_CS_DED__SHIFT 0x2
+#define GDS_EDC_OA_DED__UNUSED0_MASK 0x8
+#define GDS_EDC_OA_DED__UNUSED0__SHIFT 0x3
+#define GDS_EDC_OA_DED__ME1_PIPE0_DED_MASK 0x10
+#define GDS_EDC_OA_DED__ME1_PIPE0_DED__SHIFT 0x4
+#define GDS_EDC_OA_DED__ME1_PIPE1_DED_MASK 0x20
+#define GDS_EDC_OA_DED__ME1_PIPE1_DED__SHIFT 0x5
+#define GDS_EDC_OA_DED__ME1_PIPE2_DED_MASK 0x40
+#define GDS_EDC_OA_DED__ME1_PIPE2_DED__SHIFT 0x6
+#define GDS_EDC_OA_DED__ME1_PIPE3_DED_MASK 0x80
+#define GDS_EDC_OA_DED__ME1_PIPE3_DED__SHIFT 0x7
+#define GDS_EDC_OA_DED__ME2_PIPE0_DED_MASK 0x100
+#define GDS_EDC_OA_DED__ME2_PIPE0_DED__SHIFT 0x8
+#define GDS_EDC_OA_DED__ME2_PIPE1_DED_MASK 0x200
+#define GDS_EDC_OA_DED__ME2_PIPE1_DED__SHIFT 0x9
+#define GDS_EDC_OA_DED__ME2_PIPE2_DED_MASK 0x400
+#define GDS_EDC_OA_DED__ME2_PIPE2_DED__SHIFT 0xa
+#define GDS_EDC_OA_DED__ME2_PIPE3_DED_MASK 0x800
+#define GDS_EDC_OA_DED__ME2_PIPE3_DED__SHIFT 0xb
+#define GDS_EDC_OA_DED__UNUSED1_MASK 0xfffff000
+#define GDS_EDC_OA_DED__UNUSED1__SHIFT 0xc
+#define GDS_DEBUG_CNTL__GDS_DEBUG_INDX_MASK 0x1f
+#define GDS_DEBUG_CNTL__GDS_DEBUG_INDX__SHIFT 0x0
+#define GDS_DEBUG_CNTL__UNUSED_MASK 0xffffffe0
+#define GDS_DEBUG_CNTL__UNUSED__SHIFT 0x5
+#define GDS_DEBUG_DATA__DATA_MASK 0xffffffff
+#define GDS_DEBUG_DATA__DATA__SHIFT 0x0
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_A_0_MASK 0x1
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_A_0__SHIFT 0x0
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_A_1_MASK 0x2
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_A_1__SHIFT 0x1
+#define GDS_DSM_CNTL__GDS_ENABLE_SINGLE_WRITE_A_MASK 0x4
+#define GDS_DSM_CNTL__GDS_ENABLE_SINGLE_WRITE_A__SHIFT 0x2
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_B_0_MASK 0x8
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_B_0__SHIFT 0x3
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_B_1_MASK 0x10
+#define GDS_DSM_CNTL__SEL_DSM_GDS_IRRITATOR_DATA_B_1__SHIFT 0x4
+#define GDS_DSM_CNTL__GDS_ENABLE_SINGLE_WRITE_B_MASK 0x20
+#define GDS_DSM_CNTL__GDS_ENABLE_SINGLE_WRITE_B__SHIFT 0x5
+#define GDS_DSM_CNTL__UNUSED_MASK 0xffffffc0
+#define GDS_DSM_CNTL__UNUSED__SHIFT 0x6
+#define CGTT_GDS_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_GDS_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f
+#define GDS_RD_ADDR__READ_ADDR_MASK 0xffffffff
+#define GDS_RD_ADDR__READ_ADDR__SHIFT 0x0
+#define GDS_RD_DATA__READ_DATA_MASK 0xffffffff
+#define GDS_RD_DATA__READ_DATA__SHIFT 0x0
+#define GDS_RD_BURST_ADDR__BURST_ADDR_MASK 0xffffffff
+#define GDS_RD_BURST_ADDR__BURST_ADDR__SHIFT 0x0
+#define GDS_RD_BURST_COUNT__BURST_COUNT_MASK 0xffffffff
+#define GDS_RD_BURST_COUNT__BURST_COUNT__SHIFT 0x0
+#define GDS_RD_BURST_DATA__BURST_DATA_MASK 0xffffffff
+#define GDS_RD_BURST_DATA__BURST_DATA__SHIFT 0x0
+#define GDS_WR_ADDR__WRITE_ADDR_MASK 0xffffffff
+#define GDS_WR_ADDR__WRITE_ADDR__SHIFT 0x0
+#define GDS_WR_DATA__WRITE_DATA_MASK 0xffffffff
+#define GDS_WR_DATA__WRITE_DATA__SHIFT 0x0
+#define GDS_WR_BURST_ADDR__WRITE_ADDR_MASK 0xffffffff
+#define GDS_WR_BURST_ADDR__WRITE_ADDR__SHIFT 0x0
+#define GDS_WR_BURST_DATA__WRITE_DATA_MASK 0xffffffff
+#define GDS_WR_BURST_DATA__WRITE_DATA__SHIFT 0x0
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE_MASK 0xffffffff
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE__SHIFT 0x0
+#define GDS_ATOM_CNTL__AINC_MASK 0x3f
+#define GDS_ATOM_CNTL__AINC__SHIFT 0x0
+#define GDS_ATOM_CNTL__UNUSED1_MASK 0xc0
+#define GDS_ATOM_CNTL__UNUSED1__SHIFT 0x6
+#define GDS_ATOM_CNTL__DMODE_MASK 0x300
+#define GDS_ATOM_CNTL__DMODE__SHIFT 0x8
+#define GDS_ATOM_CNTL__UNUSED2_MASK 0xfffffc00
+#define GDS_ATOM_CNTL__UNUSED2__SHIFT 0xa
+#define GDS_ATOM_COMPLETE__COMPLETE_MASK 0x1
+#define GDS_ATOM_COMPLETE__COMPLETE__SHIFT 0x0
+#define GDS_ATOM_COMPLETE__UNUSED_MASK 0xfffffffe
+#define GDS_ATOM_COMPLETE__UNUSED__SHIFT 0x1
+#define GDS_ATOM_BASE__BASE_MASK 0xffff
+#define GDS_ATOM_BASE__BASE__SHIFT 0x0
+#define GDS_ATOM_BASE__UNUSED_MASK 0xffff0000
+#define GDS_ATOM_BASE__UNUSED__SHIFT 0x10
+#define GDS_ATOM_SIZE__SIZE_MASK 0xffff
+#define GDS_ATOM_SIZE__SIZE__SHIFT 0x0
+#define GDS_ATOM_SIZE__UNUSED_MASK 0xffff0000
+#define GDS_ATOM_SIZE__UNUSED__SHIFT 0x10
+#define GDS_ATOM_OFFSET0__OFFSET0_MASK 0xff
+#define GDS_ATOM_OFFSET0__OFFSET0__SHIFT 0x0
+#define GDS_ATOM_OFFSET0__UNUSED_MASK 0xffffff00
+#define GDS_ATOM_OFFSET0__UNUSED__SHIFT 0x8
+#define GDS_ATOM_OFFSET1__OFFSET1_MASK 0xff
+#define GDS_ATOM_OFFSET1__OFFSET1__SHIFT 0x0
+#define GDS_ATOM_OFFSET1__UNUSED_MASK 0xffffff00
+#define GDS_ATOM_OFFSET1__UNUSED__SHIFT 0x8
+#define GDS_ATOM_DST__DST_MASK 0xffffffff
+#define GDS_ATOM_DST__DST__SHIFT 0x0
+#define GDS_ATOM_OP__OP_MASK 0xff
+#define GDS_ATOM_OP__OP__SHIFT 0x0
+#define GDS_ATOM_OP__UNUSED_MASK 0xffffff00
+#define GDS_ATOM_OP__UNUSED__SHIFT 0x8
+#define GDS_ATOM_SRC0__DATA_MASK 0xffffffff
+#define GDS_ATOM_SRC0__DATA__SHIFT 0x0
+#define GDS_ATOM_SRC0_U__DATA_MASK 0xffffffff
+#define GDS_ATOM_SRC0_U__DATA__SHIFT 0x0
+#define GDS_ATOM_SRC1__DATA_MASK 0xffffffff
+#define GDS_ATOM_SRC1__DATA__SHIFT 0x0
+#define GDS_ATOM_SRC1_U__DATA_MASK 0xffffffff
+#define GDS_ATOM_SRC1_U__DATA__SHIFT 0x0
+#define GDS_ATOM_READ0__DATA_MASK 0xffffffff
+#define GDS_ATOM_READ0__DATA__SHIFT 0x0
+#define GDS_ATOM_READ0_U__DATA_MASK 0xffffffff
+#define GDS_ATOM_READ0_U__DATA__SHIFT 0x0
+#define GDS_ATOM_READ1__DATA_MASK 0xffffffff
+#define GDS_ATOM_READ1__DATA__SHIFT 0x0
+#define GDS_ATOM_READ1_U__DATA_MASK 0xffffffff
+#define GDS_ATOM_READ1_U__DATA__SHIFT 0x0
+#define GDS_GWS_RESOURCE_CNTL__INDEX_MASK 0x3f
+#define GDS_GWS_RESOURCE_CNTL__INDEX__SHIFT 0x0
+#define GDS_GWS_RESOURCE_CNTL__UNUSED_MASK 0xffffffc0
+#define GDS_GWS_RESOURCE_CNTL__UNUSED__SHIFT 0x6
+#define GDS_GWS_RESOURCE__FLAG_MASK 0x1
+#define GDS_GWS_RESOURCE__FLAG__SHIFT 0x0
+#define GDS_GWS_RESOURCE__COUNTER_MASK 0x1ffe
+#define GDS_GWS_RESOURCE__COUNTER__SHIFT 0x1
+#define GDS_GWS_RESOURCE__TYPE_MASK 0x2000
+#define GDS_GWS_RESOURCE__TYPE__SHIFT 0xd
+#define GDS_GWS_RESOURCE__DED_MASK 0x4000
+#define GDS_GWS_RESOURCE__DED__SHIFT 0xe
+#define GDS_GWS_RESOURCE__RELEASE_ALL_MASK 0x8000
+#define GDS_GWS_RESOURCE__RELEASE_ALL__SHIFT 0xf
+#define GDS_GWS_RESOURCE__HEAD_QUEUE_MASK 0xfff0000
+#define GDS_GWS_RESOURCE__HEAD_QUEUE__SHIFT 0x10
+#define GDS_GWS_RESOURCE__HEAD_VALID_MASK 0x10000000
+#define GDS_GWS_RESOURCE__HEAD_VALID__SHIFT 0x1c
+#define GDS_GWS_RESOURCE__HEAD_FLAG_MASK 0x20000000
+#define GDS_GWS_RESOURCE__HEAD_FLAG__SHIFT 0x1d
+#define GDS_GWS_RESOURCE__UNUSED1_MASK 0xc0000000
+#define GDS_GWS_RESOURCE__UNUSED1__SHIFT 0x1e
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT_MASK 0xffff
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT__SHIFT 0x0
+#define GDS_GWS_RESOURCE_CNT__UNUSED_MASK 0xffff0000
+#define GDS_GWS_RESOURCE_CNT__UNUSED__SHIFT 0x10
+#define GDS_OA_CNTL__INDEX_MASK 0xf
+#define GDS_OA_CNTL__INDEX__SHIFT 0x0
+#define GDS_OA_CNTL__UNUSED_MASK 0xfffffff0
+#define GDS_OA_CNTL__UNUSED__SHIFT 0x4
+#define GDS_OA_COUNTER__SPACE_AVAILABLE_MASK 0xffffffff
+#define GDS_OA_COUNTER__SPACE_AVAILABLE__SHIFT 0x0
+#define GDS_OA_ADDRESS__DS_ADDRESS_MASK 0xffff
+#define GDS_OA_ADDRESS__DS_ADDRESS__SHIFT 0x0
+#define GDS_OA_ADDRESS__CRAWLER_MASK 0xf0000
+#define GDS_OA_ADDRESS__CRAWLER__SHIFT 0x10
+#define GDS_OA_ADDRESS__CRAWLER_TYPE_MASK 0x300000
+#define GDS_OA_ADDRESS__CRAWLER_TYPE__SHIFT 0x14
+#define GDS_OA_ADDRESS__UNUSED_MASK 0x3fc00000
+#define GDS_OA_ADDRESS__UNUSED__SHIFT 0x16
+#define GDS_OA_ADDRESS__NO_ALLOC_MASK 0x40000000
+#define GDS_OA_ADDRESS__NO_ALLOC__SHIFT 0x1e
+#define GDS_OA_ADDRESS__ENABLE_MASK 0x80000000
+#define GDS_OA_ADDRESS__ENABLE__SHIFT 0x1f
+#define GDS_OA_INCDEC__VALUE_MASK 0x7fffffff
+#define GDS_OA_INCDEC__VALUE__SHIFT 0x0
+#define GDS_OA_INCDEC__INCDEC_MASK 0x80000000
+#define GDS_OA_INCDEC__INCDEC__SHIFT 0x1f
+#define GDS_OA_RING_SIZE__RING_SIZE_MASK 0xffffffff
+#define GDS_OA_RING_SIZE__RING_SIZE__SHIFT 0x0
+#define GDS_DEBUG_REG0__spare1_MASK 0x3f
+#define GDS_DEBUG_REG0__spare1__SHIFT 0x0
+#define GDS_DEBUG_REG0__write_buff_valid_MASK 0x40
+#define GDS_DEBUG_REG0__write_buff_valid__SHIFT 0x6
+#define GDS_DEBUG_REG0__wr_pixel_nxt_ptr_MASK 0xf80
+#define GDS_DEBUG_REG0__wr_pixel_nxt_ptr__SHIFT 0x7
+#define GDS_DEBUG_REG0__last_pixel_ptr_MASK 0x1000
+#define GDS_DEBUG_REG0__last_pixel_ptr__SHIFT 0xc
+#define GDS_DEBUG_REG0__cstate_MASK 0x1e000
+#define GDS_DEBUG_REG0__cstate__SHIFT 0xd
+#define GDS_DEBUG_REG0__buff_write_MASK 0x20000
+#define GDS_DEBUG_REG0__buff_write__SHIFT 0x11
+#define GDS_DEBUG_REG0__flush_request_MASK 0x40000
+#define GDS_DEBUG_REG0__flush_request__SHIFT 0x12
+#define GDS_DEBUG_REG0__wr_buffer_wr_complete_MASK 0x80000
+#define GDS_DEBUG_REG0__wr_buffer_wr_complete__SHIFT 0x13
+#define GDS_DEBUG_REG0__wbuf_fifo_empty_MASK 0x100000
+#define GDS_DEBUG_REG0__wbuf_fifo_empty__SHIFT 0x14
+#define GDS_DEBUG_REG0__wbuf_fifo_full_MASK 0x200000
+#define GDS_DEBUG_REG0__wbuf_fifo_full__SHIFT 0x15
+#define GDS_DEBUG_REG0__spare_MASK 0xffc00000
+#define GDS_DEBUG_REG0__spare__SHIFT 0x16
+#define GDS_DEBUG_REG1__tag_hit_MASK 0x1
+#define GDS_DEBUG_REG1__tag_hit__SHIFT 0x0
+#define GDS_DEBUG_REG1__tag_miss_MASK 0x2
+#define GDS_DEBUG_REG1__tag_miss__SHIFT 0x1
+#define GDS_DEBUG_REG1__pixel_addr_MASK 0x1fffc
+#define GDS_DEBUG_REG1__pixel_addr__SHIFT 0x2
+#define GDS_DEBUG_REG1__pixel_vld_MASK 0x20000
+#define GDS_DEBUG_REG1__pixel_vld__SHIFT 0x11
+#define GDS_DEBUG_REG1__data_ready_MASK 0x40000
+#define GDS_DEBUG_REG1__data_ready__SHIFT 0x12
+#define GDS_DEBUG_REG1__awaiting_data_MASK 0x80000
+#define GDS_DEBUG_REG1__awaiting_data__SHIFT 0x13
+#define GDS_DEBUG_REG1__addr_fifo_full_MASK 0x100000
+#define GDS_DEBUG_REG1__addr_fifo_full__SHIFT 0x14
+#define GDS_DEBUG_REG1__addr_fifo_empty_MASK 0x200000
+#define GDS_DEBUG_REG1__addr_fifo_empty__SHIFT 0x15
+#define GDS_DEBUG_REG1__buffer_loaded_MASK 0x400000
+#define GDS_DEBUG_REG1__buffer_loaded__SHIFT 0x16
+#define GDS_DEBUG_REG1__buffer_invalid_MASK 0x800000
+#define GDS_DEBUG_REG1__buffer_invalid__SHIFT 0x17
+#define GDS_DEBUG_REG1__spare_MASK 0xff000000
+#define GDS_DEBUG_REG1__spare__SHIFT 0x18
+#define GDS_DEBUG_REG2__ds_full_MASK 0x1
+#define GDS_DEBUG_REG2__ds_full__SHIFT 0x0
+#define GDS_DEBUG_REG2__ds_credit_avail_MASK 0x2
+#define GDS_DEBUG_REG2__ds_credit_avail__SHIFT 0x1
+#define GDS_DEBUG_REG2__ord_idx_free_MASK 0x4
+#define GDS_DEBUG_REG2__ord_idx_free__SHIFT 0x2
+#define GDS_DEBUG_REG2__cmd_write_MASK 0x8
+#define GDS_DEBUG_REG2__cmd_write__SHIFT 0x3
+#define GDS_DEBUG_REG2__app_sel_MASK 0xf0
+#define GDS_DEBUG_REG2__app_sel__SHIFT 0x4
+#define GDS_DEBUG_REG2__req_MASK 0x7fff00
+#define GDS_DEBUG_REG2__req__SHIFT 0x8
+#define GDS_DEBUG_REG2__spare_MASK 0xff800000
+#define GDS_DEBUG_REG2__spare__SHIFT 0x17
+#define GDS_DEBUG_REG3__pipe_num_busy_MASK 0x7ff
+#define GDS_DEBUG_REG3__pipe_num_busy__SHIFT 0x0
+#define GDS_DEBUG_REG3__pipe0_busy_num_MASK 0x7800
+#define GDS_DEBUG_REG3__pipe0_busy_num__SHIFT 0xb
+#define GDS_DEBUG_REG3__spare_MASK 0xffff8000
+#define GDS_DEBUG_REG3__spare__SHIFT 0xf
+#define GDS_DEBUG_REG4__gws_busy_MASK 0x1
+#define GDS_DEBUG_REG4__gws_busy__SHIFT 0x0
+#define GDS_DEBUG_REG4__gws_req_MASK 0x2
+#define GDS_DEBUG_REG4__gws_req__SHIFT 0x1
+#define GDS_DEBUG_REG4__gws_out_stall_MASK 0x4
+#define GDS_DEBUG_REG4__gws_out_stall__SHIFT 0x2
+#define GDS_DEBUG_REG4__cur_reso_MASK 0x1f8
+#define GDS_DEBUG_REG4__cur_reso__SHIFT 0x3
+#define GDS_DEBUG_REG4__cur_reso_head_valid_MASK 0x200
+#define GDS_DEBUG_REG4__cur_reso_head_valid__SHIFT 0x9
+#define GDS_DEBUG_REG4__cur_reso_head_dirty_MASK 0x400
+#define GDS_DEBUG_REG4__cur_reso_head_dirty__SHIFT 0xa
+#define GDS_DEBUG_REG4__cur_reso_head_flag_MASK 0x800
+#define GDS_DEBUG_REG4__cur_reso_head_flag__SHIFT 0xb
+#define GDS_DEBUG_REG4__cur_reso_fed_MASK 0x1000
+#define GDS_DEBUG_REG4__cur_reso_fed__SHIFT 0xc
+#define GDS_DEBUG_REG4__cur_reso_barrier_MASK 0x2000
+#define GDS_DEBUG_REG4__cur_reso_barrier__SHIFT 0xd
+#define GDS_DEBUG_REG4__cur_reso_flag_MASK 0x4000
+#define GDS_DEBUG_REG4__cur_reso_flag__SHIFT 0xe
+#define GDS_DEBUG_REG4__cur_reso_cnt_gt0_MASK 0x8000
+#define GDS_DEBUG_REG4__cur_reso_cnt_gt0__SHIFT 0xf
+#define GDS_DEBUG_REG4__credit_cnt_gt0_MASK 0x10000
+#define GDS_DEBUG_REG4__credit_cnt_gt0__SHIFT 0x10
+#define GDS_DEBUG_REG4__cmd_write_MASK 0x20000
+#define GDS_DEBUG_REG4__cmd_write__SHIFT 0x11
+#define GDS_DEBUG_REG4__grbm_gws_reso_wr_MASK 0x40000
+#define GDS_DEBUG_REG4__grbm_gws_reso_wr__SHIFT 0x12
+#define GDS_DEBUG_REG4__grbm_gws_reso_rd_MASK 0x80000
+#define GDS_DEBUG_REG4__grbm_gws_reso_rd__SHIFT 0x13
+#define GDS_DEBUG_REG4__ram_read_busy_MASK 0x100000
+#define GDS_DEBUG_REG4__ram_read_busy__SHIFT 0x14
+#define GDS_DEBUG_REG4__gws_bulkfree_MASK 0x200000
+#define GDS_DEBUG_REG4__gws_bulkfree__SHIFT 0x15
+#define GDS_DEBUG_REG4__ram_gws_re_MASK 0x400000
+#define GDS_DEBUG_REG4__ram_gws_re__SHIFT 0x16
+#define GDS_DEBUG_REG4__ram_gws_we_MASK 0x800000
+#define GDS_DEBUG_REG4__ram_gws_we__SHIFT 0x17
+#define GDS_DEBUG_REG4__spare_MASK 0xff000000
+#define GDS_DEBUG_REG4__spare__SHIFT 0x18
+#define GDS_DEBUG_REG5__write_dis_MASK 0x1
+#define GDS_DEBUG_REG5__write_dis__SHIFT 0x0
+#define GDS_DEBUG_REG5__dec_error_MASK 0x2
+#define GDS_DEBUG_REG5__dec_error__SHIFT 0x1
+#define GDS_DEBUG_REG5__alloc_opco_error_MASK 0x4
+#define GDS_DEBUG_REG5__alloc_opco_error__SHIFT 0x2
+#define GDS_DEBUG_REG5__dealloc_opco_error_MASK 0x8
+#define GDS_DEBUG_REG5__dealloc_opco_error__SHIFT 0x3
+#define GDS_DEBUG_REG5__wrap_opco_error_MASK 0x10
+#define GDS_DEBUG_REG5__wrap_opco_error__SHIFT 0x4
+#define GDS_DEBUG_REG5__spare_MASK 0xe0
+#define GDS_DEBUG_REG5__spare__SHIFT 0x5
+#define GDS_DEBUG_REG5__error_ds_address_MASK 0x3fff00
+#define GDS_DEBUG_REG5__error_ds_address__SHIFT 0x8
+#define GDS_DEBUG_REG5__spare1_MASK 0xffc00000
+#define GDS_DEBUG_REG5__spare1__SHIFT 0x16
+#define GDS_DEBUG_REG6__oa_busy_MASK 0x1
+#define GDS_DEBUG_REG6__oa_busy__SHIFT 0x0
+#define GDS_DEBUG_REG6__counters_enabled_MASK 0x1e
+#define GDS_DEBUG_REG6__counters_enabled__SHIFT 0x1
+#define GDS_DEBUG_REG6__counters_busy_MASK 0x1fffe0
+#define GDS_DEBUG_REG6__counters_busy__SHIFT 0x5
+#define GDS_DEBUG_REG6__spare_MASK 0xffe00000
+#define GDS_DEBUG_REG6__spare__SHIFT 0x15
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK 0x3ff
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x0
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK 0xffc00
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0xa
+#define GDS_VMID0_BASE__BASE_MASK 0xffff
+#define GDS_VMID0_BASE__BASE__SHIFT 0x0
+#define GDS_VMID1_BASE__BASE_MASK 0xffff
+#define GDS_VMID1_BASE__BASE__SHIFT 0x0
+#define GDS_VMID2_BASE__BASE_MASK 0xffff
+#define GDS_VMID2_BASE__BASE__SHIFT 0x0
+#define GDS_VMID3_BASE__BASE_MASK 0xffff
+#define GDS_VMID3_BASE__BASE__SHIFT 0x0
+#define GDS_VMID4_BASE__BASE_MASK 0xffff
+#define GDS_VMID4_BASE__BASE__SHIFT 0x0
+#define GDS_VMID5_BASE__BASE_MASK 0xffff
+#define GDS_VMID5_BASE__BASE__SHIFT 0x0
+#define GDS_VMID6_BASE__BASE_MASK 0xffff
+#define GDS_VMID6_BASE__BASE__SHIFT 0x0
+#define GDS_VMID7_BASE__BASE_MASK 0xffff
+#define GDS_VMID7_BASE__BASE__SHIFT 0x0
+#define GDS_VMID8_BASE__BASE_MASK 0xffff
+#define GDS_VMID8_BASE__BASE__SHIFT 0x0
+#define GDS_VMID9_BASE__BASE_MASK 0xffff
+#define GDS_VMID9_BASE__BASE__SHIFT 0x0
+#define GDS_VMID10_BASE__BASE_MASK 0xffff
+#define GDS_VMID10_BASE__BASE__SHIFT 0x0
+#define GDS_VMID11_BASE__BASE_MASK 0xffff
+#define GDS_VMID11_BASE__BASE__SHIFT 0x0
+#define GDS_VMID12_BASE__BASE_MASK 0xffff
+#define GDS_VMID12_BASE__BASE__SHIFT 0x0
+#define GDS_VMID13_BASE__BASE_MASK 0xffff
+#define GDS_VMID13_BASE__BASE__SHIFT 0x0
+#define GDS_VMID14_BASE__BASE_MASK 0xffff
+#define GDS_VMID14_BASE__BASE__SHIFT 0x0
+#define GDS_VMID15_BASE__BASE_MASK 0xffff
+#define GDS_VMID15_BASE__BASE__SHIFT 0x0
+#define GDS_VMID0_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID0_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID1_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID1_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID2_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID2_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID3_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID3_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID4_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID4_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID5_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID5_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID6_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID6_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID7_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID7_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID8_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID8_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID9_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID9_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID10_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID10_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID11_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID11_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID12_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID12_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID13_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID13_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID14_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID14_SIZE__SIZE__SHIFT 0x0
+#define GDS_VMID15_SIZE__SIZE_MASK 0x1ffff
+#define GDS_VMID15_SIZE__SIZE__SHIFT 0x0
+#define GDS_GWS_VMID0__BASE_MASK 0x3f
+#define GDS_GWS_VMID0__BASE__SHIFT 0x0
+#define GDS_GWS_VMID0__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID0__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID1__BASE_MASK 0x3f
+#define GDS_GWS_VMID1__BASE__SHIFT 0x0
+#define GDS_GWS_VMID1__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID1__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID2__BASE_MASK 0x3f
+#define GDS_GWS_VMID2__BASE__SHIFT 0x0
+#define GDS_GWS_VMID2__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID2__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID3__BASE_MASK 0x3f
+#define GDS_GWS_VMID3__BASE__SHIFT 0x0
+#define GDS_GWS_VMID3__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID3__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID4__BASE_MASK 0x3f
+#define GDS_GWS_VMID4__BASE__SHIFT 0x0
+#define GDS_GWS_VMID4__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID4__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID5__BASE_MASK 0x3f
+#define GDS_GWS_VMID5__BASE__SHIFT 0x0
+#define GDS_GWS_VMID5__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID5__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID6__BASE_MASK 0x3f
+#define GDS_GWS_VMID6__BASE__SHIFT 0x0
+#define GDS_GWS_VMID6__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID6__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID7__BASE_MASK 0x3f
+#define GDS_GWS_VMID7__BASE__SHIFT 0x0
+#define GDS_GWS_VMID7__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID7__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID8__BASE_MASK 0x3f
+#define GDS_GWS_VMID8__BASE__SHIFT 0x0
+#define GDS_GWS_VMID8__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID8__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID9__BASE_MASK 0x3f
+#define GDS_GWS_VMID9__BASE__SHIFT 0x0
+#define GDS_GWS_VMID9__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID9__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID10__BASE_MASK 0x3f
+#define GDS_GWS_VMID10__BASE__SHIFT 0x0
+#define GDS_GWS_VMID10__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID10__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID11__BASE_MASK 0x3f
+#define GDS_GWS_VMID11__BASE__SHIFT 0x0
+#define GDS_GWS_VMID11__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID11__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID12__BASE_MASK 0x3f
+#define GDS_GWS_VMID12__BASE__SHIFT 0x0
+#define GDS_GWS_VMID12__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID12__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID13__BASE_MASK 0x3f
+#define GDS_GWS_VMID13__BASE__SHIFT 0x0
+#define GDS_GWS_VMID13__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID13__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID14__BASE_MASK 0x3f
+#define GDS_GWS_VMID14__BASE__SHIFT 0x0
+#define GDS_GWS_VMID14__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID14__SIZE__SHIFT 0x10
+#define GDS_GWS_VMID15__BASE_MASK 0x3f
+#define GDS_GWS_VMID15__BASE__SHIFT 0x0
+#define GDS_GWS_VMID15__SIZE_MASK 0x7f0000
+#define GDS_GWS_VMID15__SIZE__SHIFT 0x10
+#define GDS_OA_VMID0__MASK_MASK 0xffff
+#define GDS_OA_VMID0__MASK__SHIFT 0x0
+#define GDS_OA_VMID0__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID0__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID1__MASK_MASK 0xffff
+#define GDS_OA_VMID1__MASK__SHIFT 0x0
+#define GDS_OA_VMID1__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID1__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID2__MASK_MASK 0xffff
+#define GDS_OA_VMID2__MASK__SHIFT 0x0
+#define GDS_OA_VMID2__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID2__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID3__MASK_MASK 0xffff
+#define GDS_OA_VMID3__MASK__SHIFT 0x0
+#define GDS_OA_VMID3__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID3__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID4__MASK_MASK 0xffff
+#define GDS_OA_VMID4__MASK__SHIFT 0x0
+#define GDS_OA_VMID4__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID4__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID5__MASK_MASK 0xffff
+#define GDS_OA_VMID5__MASK__SHIFT 0x0
+#define GDS_OA_VMID5__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID5__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID6__MASK_MASK 0xffff
+#define GDS_OA_VMID6__MASK__SHIFT 0x0
+#define GDS_OA_VMID6__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID6__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID7__MASK_MASK 0xffff
+#define GDS_OA_VMID7__MASK__SHIFT 0x0
+#define GDS_OA_VMID7__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID7__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID8__MASK_MASK 0xffff
+#define GDS_OA_VMID8__MASK__SHIFT 0x0
+#define GDS_OA_VMID8__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID8__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID9__MASK_MASK 0xffff
+#define GDS_OA_VMID9__MASK__SHIFT 0x0
+#define GDS_OA_VMID9__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID9__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID10__MASK_MASK 0xffff
+#define GDS_OA_VMID10__MASK__SHIFT 0x0
+#define GDS_OA_VMID10__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID10__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID11__MASK_MASK 0xffff
+#define GDS_OA_VMID11__MASK__SHIFT 0x0
+#define GDS_OA_VMID11__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID11__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID12__MASK_MASK 0xffff
+#define GDS_OA_VMID12__MASK__SHIFT 0x0
+#define GDS_OA_VMID12__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID12__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID13__MASK_MASK 0xffff
+#define GDS_OA_VMID13__MASK__SHIFT 0x0
+#define GDS_OA_VMID13__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID13__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID14__MASK_MASK 0xffff
+#define GDS_OA_VMID14__MASK__SHIFT 0x0
+#define GDS_OA_VMID14__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID14__UNUSED__SHIFT 0x10
+#define GDS_OA_VMID15__MASK_MASK 0xffff
+#define GDS_OA_VMID15__MASK__SHIFT 0x0
+#define GDS_OA_VMID15__UNUSED_MASK 0xffff0000
+#define GDS_OA_VMID15__UNUSED__SHIFT 0x10
+#define GDS_GWS_RESET0__RESOURCE0_RESET_MASK 0x1
+#define GDS_GWS_RESET0__RESOURCE0_RESET__SHIFT 0x0
+#define GDS_GWS_RESET0__RESOURCE1_RESET_MASK 0x2
+#define GDS_GWS_RESET0__RESOURCE1_RESET__SHIFT 0x1
+#define GDS_GWS_RESET0__RESOURCE2_RESET_MASK 0x4
+#define GDS_GWS_RESET0__RESOURCE2_RESET__SHIFT 0x2
+#define GDS_GWS_RESET0__RESOURCE3_RESET_MASK 0x8
+#define GDS_GWS_RESET0__RESOURCE3_RESET__SHIFT 0x3
+#define GDS_GWS_RESET0__RESOURCE4_RESET_MASK 0x10
+#define GDS_GWS_RESET0__RESOURCE4_RESET__SHIFT 0x4
+#define GDS_GWS_RESET0__RESOURCE5_RESET_MASK 0x20
+#define GDS_GWS_RESET0__RESOURCE5_RESET__SHIFT 0x5
+#define GDS_GWS_RESET0__RESOURCE6_RESET_MASK 0x40
+#define GDS_GWS_RESET0__RESOURCE6_RESET__SHIFT 0x6
+#define GDS_GWS_RESET0__RESOURCE7_RESET_MASK 0x80
+#define GDS_GWS_RESET0__RESOURCE7_RESET__SHIFT 0x7
+#define GDS_GWS_RESET0__RESOURCE8_RESET_MASK 0x100
+#define GDS_GWS_RESET0__RESOURCE8_RESET__SHIFT 0x8
+#define GDS_GWS_RESET0__RESOURCE9_RESET_MASK 0x200
+#define GDS_GWS_RESET0__RESOURCE9_RESET__SHIFT 0x9
+#define GDS_GWS_RESET0__RESOURCE10_RESET_MASK 0x400
+#define GDS_GWS_RESET0__RESOURCE10_RESET__SHIFT 0xa
+#define GDS_GWS_RESET0__RESOURCE11_RESET_MASK 0x800
+#define GDS_GWS_RESET0__RESOURCE11_RESET__SHIFT 0xb
+#define GDS_GWS_RESET0__RESOURCE12_RESET_MASK 0x1000
+#define GDS_GWS_RESET0__RESOURCE12_RESET__SHIFT 0xc
+#define GDS_GWS_RESET0__RESOURCE13_RESET_MASK 0x2000
+#define GDS_GWS_RESET0__RESOURCE13_RESET__SHIFT 0xd
+#define GDS_GWS_RESET0__RESOURCE14_RESET_MASK 0x4000
+#define GDS_GWS_RESET0__RESOURCE14_RESET__SHIFT 0xe
+#define GDS_GWS_RESET0__RESOURCE15_RESET_MASK 0x8000
+#define GDS_GWS_RESET0__RESOURCE15_RESET__SHIFT 0xf
+#define GDS_GWS_RESET0__RESOURCE16_RESET_MASK 0x10000
+#define GDS_GWS_RESET0__RESOURCE16_RESET__SHIFT 0x10
+#define GDS_GWS_RESET0__RESOURCE17_RESET_MASK 0x20000
+#define GDS_GWS_RESET0__RESOURCE17_RESET__SHIFT 0x11
+#define GDS_GWS_RESET0__RESOURCE18_RESET_MASK 0x40000
+#define GDS_GWS_RESET0__RESOURCE18_RESET__SHIFT 0x12
+#define GDS_GWS_RESET0__RESOURCE19_RESET_MASK 0x80000
+#define GDS_GWS_RESET0__RESOURCE19_RESET__SHIFT 0x13
+#define GDS_GWS_RESET0__RESOURCE20_RESET_MASK 0x100000
+#define GDS_GWS_RESET0__RESOURCE20_RESET__SHIFT 0x14
+#define GDS_GWS_RESET0__RESOURCE21_RESET_MASK 0x200000
+#define GDS_GWS_RESET0__RESOURCE21_RESET__SHIFT 0x15
+#define GDS_GWS_RESET0__RESOURCE22_RESET_MASK 0x400000
+#define GDS_GWS_RESET0__RESOURCE22_RESET__SHIFT 0x16
+#define GDS_GWS_RESET0__RESOURCE23_RESET_MASK 0x800000
+#define GDS_GWS_RESET0__RESOURCE23_RESET__SHIFT 0x17
+#define GDS_GWS_RESET0__RESOURCE24_RESET_MASK 0x1000000
+#define GDS_GWS_RESET0__RESOURCE24_RESET__SHIFT 0x18
+#define GDS_GWS_RESET0__RESOURCE25_RESET_MASK 0x2000000
+#define GDS_GWS_RESET0__RESOURCE25_RESET__SHIFT 0x19
+#define GDS_GWS_RESET0__RESOURCE26_RESET_MASK 0x4000000
+#define GDS_GWS_RESET0__RESOURCE26_RESET__SHIFT 0x1a
+#define GDS_GWS_RESET0__RESOURCE27_RESET_MASK 0x8000000
+#define GDS_GWS_RESET0__RESOURCE27_RESET__SHIFT 0x1b
+#define GDS_GWS_RESET0__RESOURCE28_RESET_MASK 0x10000000
+#define GDS_GWS_RESET0__RESOURCE28_RESET__SHIFT 0x1c
+#define GDS_GWS_RESET0__RESOURCE29_RESET_MASK 0x20000000
+#define GDS_GWS_RESET0__RESOURCE29_RESET__SHIFT 0x1d
+#define GDS_GWS_RESET0__RESOURCE30_RESET_MASK 0x40000000
+#define GDS_GWS_RESET0__RESOURCE30_RESET__SHIFT 0x1e
+#define GDS_GWS_RESET0__RESOURCE31_RESET_MASK 0x80000000
+#define GDS_GWS_RESET0__RESOURCE31_RESET__SHIFT 0x1f
+#define GDS_GWS_RESET1__RESOURCE32_RESET_MASK 0x1
+#define GDS_GWS_RESET1__RESOURCE32_RESET__SHIFT 0x0
+#define GDS_GWS_RESET1__RESOURCE33_RESET_MASK 0x2
+#define GDS_GWS_RESET1__RESOURCE33_RESET__SHIFT 0x1
+#define GDS_GWS_RESET1__RESOURCE34_RESET_MASK 0x4
+#define GDS_GWS_RESET1__RESOURCE34_RESET__SHIFT 0x2
+#define GDS_GWS_RESET1__RESOURCE35_RESET_MASK 0x8
+#define GDS_GWS_RESET1__RESOURCE35_RESET__SHIFT 0x3
+#define GDS_GWS_RESET1__RESOURCE36_RESET_MASK 0x10
+#define GDS_GWS_RESET1__RESOURCE36_RESET__SHIFT 0x4
+#define GDS_GWS_RESET1__RESOURCE37_RESET_MASK 0x20
+#define GDS_GWS_RESET1__RESOURCE37_RESET__SHIFT 0x5
+#define GDS_GWS_RESET1__RESOURCE38_RESET_MASK 0x40
+#define GDS_GWS_RESET1__RESOURCE38_RESET__SHIFT 0x6
+#define GDS_GWS_RESET1__RESOURCE39_RESET_MASK 0x80
+#define GDS_GWS_RESET1__RESOURCE39_RESET__SHIFT 0x7
+#define GDS_GWS_RESET1__RESOURCE40_RESET_MASK 0x100
+#define GDS_GWS_RESET1__RESOURCE40_RESET__SHIFT 0x8
+#define GDS_GWS_RESET1__RESOURCE41_RESET_MASK 0x200
+#define GDS_GWS_RESET1__RESOURCE41_RESET__SHIFT 0x9
+#define GDS_GWS_RESET1__RESOURCE42_RESET_MASK 0x400
+#define GDS_GWS_RESET1__RESOURCE42_RESET__SHIFT 0xa
+#define GDS_GWS_RESET1__RESOURCE43_RESET_MASK 0x800
+#define GDS_GWS_RESET1__RESOURCE43_RESET__SHIFT 0xb
+#define GDS_GWS_RESET1__RESOURCE44_RESET_MASK 0x1000
+#define GDS_GWS_RESET1__RESOURCE44_RESET__SHIFT 0xc
+#define GDS_GWS_RESET1__RESOURCE45_RESET_MASK 0x2000
+#define GDS_GWS_RESET1__RESOURCE45_RESET__SHIFT 0xd
+#define GDS_GWS_RESET1__RESOURCE46_RESET_MASK 0x4000
+#define GDS_GWS_RESET1__RESOURCE46_RESET__SHIFT 0xe
+#define GDS_GWS_RESET1__RESOURCE47_RESET_MASK 0x8000
+#define GDS_GWS_RESET1__RESOURCE47_RESET__SHIFT 0xf
+#define GDS_GWS_RESET1__RESOURCE48_RESET_MASK 0x10000
+#define GDS_GWS_RESET1__RESOURCE48_RESET__SHIFT 0x10
+#define GDS_GWS_RESET1__RESOURCE49_RESET_MASK 0x20000
+#define GDS_GWS_RESET1__RESOURCE49_RESET__SHIFT 0x11
+#define GDS_GWS_RESET1__RESOURCE50_RESET_MASK 0x40000
+#define GDS_GWS_RESET1__RESOURCE50_RESET__SHIFT 0x12
+#define GDS_GWS_RESET1__RESOURCE51_RESET_MASK 0x80000
+#define GDS_GWS_RESET1__RESOURCE51_RESET__SHIFT 0x13
+#define GDS_GWS_RESET1__RESOURCE52_RESET_MASK 0x100000
+#define GDS_GWS_RESET1__RESOURCE52_RESET__SHIFT 0x14
+#define GDS_GWS_RESET1__RESOURCE53_RESET_MASK 0x200000
+#define GDS_GWS_RESET1__RESOURCE53_RESET__SHIFT 0x15
+#define GDS_GWS_RESET1__RESOURCE54_RESET_MASK 0x400000
+#define GDS_GWS_RESET1__RESOURCE54_RESET__SHIFT 0x16
+#define GDS_GWS_RESET1__RESOURCE55_RESET_MASK 0x800000
+#define GDS_GWS_RESET1__RESOURCE55_RESET__SHIFT 0x17
+#define GDS_GWS_RESET1__RESOURCE56_RESET_MASK 0x1000000
+#define GDS_GWS_RESET1__RESOURCE56_RESET__SHIFT 0x18
+#define GDS_GWS_RESET1__RESOURCE57_RESET_MASK 0x2000000
+#define GDS_GWS_RESET1__RESOURCE57_RESET__SHIFT 0x19
+#define GDS_GWS_RESET1__RESOURCE58_RESET_MASK 0x4000000
+#define GDS_GWS_RESET1__RESOURCE58_RESET__SHIFT 0x1a
+#define GDS_GWS_RESET1__RESOURCE59_RESET_MASK 0x8000000
+#define GDS_GWS_RESET1__RESOURCE59_RESET__SHIFT 0x1b
+#define GDS_GWS_RESET1__RESOURCE60_RESET_MASK 0x10000000
+#define GDS_GWS_RESET1__RESOURCE60_RESET__SHIFT 0x1c
+#define GDS_GWS_RESET1__RESOURCE61_RESET_MASK 0x20000000
+#define GDS_GWS_RESET1__RESOURCE61_RESET__SHIFT 0x1d
+#define GDS_GWS_RESET1__RESOURCE62_RESET_MASK 0x40000000
+#define GDS_GWS_RESET1__RESOURCE62_RESET__SHIFT 0x1e
+#define GDS_GWS_RESET1__RESOURCE63_RESET_MASK 0x80000000
+#define GDS_GWS_RESET1__RESOURCE63_RESET__SHIFT 0x1f
+#define GDS_GWS_RESOURCE_RESET__RESET_MASK 0x1
+#define GDS_GWS_RESOURCE_RESET__RESET__SHIFT 0x0
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID_MASK 0xff00
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID__SHIFT 0x8
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID_MASK 0xfff
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT 0x0
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET_MASK 0x1
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET__SHIFT 0x0
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET_MASK 0x2
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET__SHIFT 0x1
+#define GDS_OA_RESET_MASK__ME0_CS_RESET_MASK 0x4
+#define GDS_OA_RESET_MASK__ME0_CS_RESET__SHIFT 0x2
+#define GDS_OA_RESET_MASK__UNUSED0_MASK 0x8
+#define GDS_OA_RESET_MASK__UNUSED0__SHIFT 0x3
+#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET_MASK 0x10
+#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET__SHIFT 0x4
+#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET_MASK 0x20
+#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET__SHIFT 0x5
+#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET_MASK 0x40
+#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET__SHIFT 0x6
+#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET_MASK 0x80
+#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET__SHIFT 0x7
+#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET_MASK 0x100
+#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET__SHIFT 0x8
+#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET_MASK 0x200
+#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET__SHIFT 0x9
+#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET_MASK 0x400
+#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET__SHIFT 0xa
+#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET_MASK 0x800
+#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET__SHIFT 0xb
+#define GDS_OA_RESET_MASK__UNUSED1_MASK 0xfffff000
+#define GDS_OA_RESET_MASK__UNUSED1__SHIFT 0xc
+#define GDS_OA_RESET__RESET_MASK 0x1
+#define GDS_OA_RESET__RESET__SHIFT 0x0
+#define GDS_OA_RESET__PIPE_ID_MASK 0xff00
+#define GDS_OA_RESET__PIPE_ID__SHIFT 0x8
+#define GDS_ENHANCE__MISC_MASK 0xffff
+#define GDS_ENHANCE__MISC__SHIFT 0x0
+#define GDS_ENHANCE__AUTO_INC_INDEX_MASK 0x10000
+#define GDS_ENHANCE__AUTO_INC_INDEX__SHIFT 0x10
+#define GDS_ENHANCE__CGPG_RESTORE_MASK 0x20000
+#define GDS_ENHANCE__CGPG_RESTORE__SHIFT 0x11
+#define GDS_ENHANCE__UNUSED_MASK 0xfffc0000
+#define GDS_ENHANCE__UNUSED__SHIFT 0x12
+#define GDS_OA_CGPG_RESTORE__VMID_MASK 0xff
+#define GDS_OA_CGPG_RESTORE__VMID__SHIFT 0x0
+#define GDS_OA_CGPG_RESTORE__MEID_MASK 0xf00
+#define GDS_OA_CGPG_RESTORE__MEID__SHIFT 0x8
+#define GDS_OA_CGPG_RESTORE__PIPEID_MASK 0xf000
+#define GDS_OA_CGPG_RESTORE__PIPEID__SHIFT 0xc
+#define GDS_OA_CGPG_RESTORE__QUEUEID_MASK 0xf0000
+#define GDS_OA_CGPG_RESTORE__QUEUEID__SHIFT 0x10
+#define GDS_OA_CGPG_RESTORE__UNUSED_MASK 0xfff00000
+#define GDS_OA_CGPG_RESTORE__UNUSED__SHIFT 0x14
+#define GDS_CS_CTXSW_STATUS__R_MASK 0x1
+#define GDS_CS_CTXSW_STATUS__R__SHIFT 0x0
+#define GDS_CS_CTXSW_STATUS__W_MASK 0x2
+#define GDS_CS_CTXSW_STATUS__W__SHIFT 0x1
+#define GDS_CS_CTXSW_STATUS__UNUSED_MASK 0xfffffffc
+#define GDS_CS_CTXSW_STATUS__UNUSED__SHIFT 0x2
+#define GDS_CS_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_CS_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_CS_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_CS_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_CS_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_CS_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_CS_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_CS_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_CS_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_CS_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_CS_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_CS_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_CS_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_CS_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_CS_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_CS_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_GFX_CTXSW_STATUS__R_MASK 0x1
+#define GDS_GFX_CTXSW_STATUS__R__SHIFT 0x0
+#define GDS_GFX_CTXSW_STATUS__W_MASK 0x2
+#define GDS_GFX_CTXSW_STATUS__W__SHIFT 0x1
+#define GDS_GFX_CTXSW_STATUS__UNUSED_MASK 0xfffffffc
+#define GDS_GFX_CTXSW_STATUS__UNUSED__SHIFT 0x2
+#define GDS_VS_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_VS_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_VS_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_VS_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_VS_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_VS_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_VS_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_VS_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_VS_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_VS_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_VS_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_VS_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_VS_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_VS_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_VS_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_VS_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS0_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS0_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS0_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS0_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS1_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS1_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS1_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS1_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS2_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS2_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS2_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS2_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS3_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS3_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS3_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS3_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS4_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS4_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS4_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS4_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS5_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS5_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS5_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS5_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS6_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS6_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS6_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS6_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS7_CTXSW_CNT0__UPDN_MASK 0xffff
+#define GDS_PS7_CTXSW_CNT0__UPDN__SHIFT 0x0
+#define GDS_PS7_CTXSW_CNT0__PTR_MASK 0xffff0000
+#define GDS_PS7_CTXSW_CNT0__PTR__SHIFT 0x10
+#define GDS_PS0_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS0_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS0_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS0_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS1_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS1_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS1_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS1_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS2_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS2_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS2_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS2_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS3_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS3_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS3_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS3_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS4_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS4_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS4_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS4_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS5_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS5_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS5_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS5_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS6_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS6_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS6_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS6_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS7_CTXSW_CNT1__UPDN_MASK 0xffff
+#define GDS_PS7_CTXSW_CNT1__UPDN__SHIFT 0x0
+#define GDS_PS7_CTXSW_CNT1__PTR_MASK 0xffff0000
+#define GDS_PS7_CTXSW_CNT1__PTR__SHIFT 0x10
+#define GDS_PS0_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS0_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS0_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS0_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS1_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS1_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS1_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS1_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS2_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS2_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS2_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS2_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS3_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS3_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS3_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS3_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS4_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS4_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS4_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS4_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS5_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS5_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS5_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS5_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS6_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS6_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS6_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS6_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS7_CTXSW_CNT2__UPDN_MASK 0xffff
+#define GDS_PS7_CTXSW_CNT2__UPDN__SHIFT 0x0
+#define GDS_PS7_CTXSW_CNT2__PTR_MASK 0xffff0000
+#define GDS_PS7_CTXSW_CNT2__PTR__SHIFT 0x10
+#define GDS_PS0_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS0_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS0_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS0_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS1_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS1_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS1_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS1_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS2_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS2_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS2_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS2_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS3_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS3_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS3_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS3_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS4_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS4_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS4_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS4_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS5_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS5_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS5_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS5_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS6_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS6_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS6_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS6_CTXSW_CNT3__PTR__SHIFT 0x10
+#define GDS_PS7_CTXSW_CNT3__UPDN_MASK 0xffff
+#define GDS_PS7_CTXSW_CNT3__UPDN__SHIFT 0x0
+#define GDS_PS7_CTXSW_CNT3__PTR_MASK 0xffff0000
+#define GDS_PS7_CTXSW_CNT3__PTR__SHIFT 0x10
+#define CS_COPY_STATE__SRC_STATE_ID_MASK 0x7
+#define CS_COPY_STATE__SRC_STATE_ID__SHIFT 0x0
+#define GFX_COPY_STATE__SRC_STATE_ID_MASK 0x7
+#define GFX_COPY_STATE__SRC_STATE_ID__SHIFT 0x0
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT_MASK 0x3
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT__SHIFT 0x0
+#define VGT_DRAW_INITIATOR__MAJOR_MODE_MASK 0xc
+#define VGT_DRAW_INITIATOR__MAJOR_MODE__SHIFT 0x2
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX_MASK 0x10
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX__SHIFT 0x4
+#define VGT_DRAW_INITIATOR__NOT_EOP_MASK 0x20
+#define VGT_DRAW_INITIATOR__NOT_EOP__SHIFT 0x5
+#define VGT_DRAW_INITIATOR__USE_OPAQUE_MASK 0x40
+#define VGT_DRAW_INITIATOR__USE_OPAQUE__SHIFT 0x6
+#define VGT_EVENT_INITIATOR__EVENT_TYPE_MASK 0x3f
+#define VGT_EVENT_INITIATOR__EVENT_TYPE__SHIFT 0x0
+#define VGT_EVENT_INITIATOR__ADDRESS_HI_MASK 0x7fc0000
+#define VGT_EVENT_INITIATOR__ADDRESS_HI__SHIFT 0x12
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT_MASK 0x8000000
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT 0x1b
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW_MASK 0xfffffff
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW__SHIFT 0x0
+#define VGT_DMA_BASE_HI__BASE_ADDR_MASK 0xff
+#define VGT_DMA_BASE_HI__BASE_ADDR__SHIFT 0x0
+#define VGT_DMA_BASE__BASE_ADDR_MASK 0xffffffff
+#define VGT_DMA_BASE__BASE_ADDR__SHIFT 0x0
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE_MASK 0x3
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE__SHIFT 0x0
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE_MASK 0xc
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE__SHIFT 0x2
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE_MASK 0x30
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE__SHIFT 0x4
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY_MASK 0x40
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY__SHIFT 0x6
+#define VGT_DMA_INDEX_TYPE__NOT_EOP_MASK 0x200
+#define VGT_DMA_INDEX_TYPE__NOT_EOP__SHIFT 0x9
+#define VGT_DMA_INDEX_TYPE__REQ_PATH_MASK 0x400
+#define VGT_DMA_INDEX_TYPE__REQ_PATH__SHIFT 0xa
+#define VGT_DMA_INDEX_TYPE__MTYPE_MASK 0x1800
+#define VGT_DMA_INDEX_TYPE__MTYPE__SHIFT 0xb
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES_MASK 0xffffffff
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES__SHIFT 0x0
+#define IA_ENHANCE__MISC_MASK 0xffffffff
+#define IA_ENHANCE__MISC__SHIFT 0x0
+#define VGT_DMA_SIZE__NUM_INDICES_MASK 0xffffffff
+#define VGT_DMA_SIZE__NUM_INDICES__SHIFT 0x0
+#define VGT_DMA_MAX_SIZE__MAX_SIZE_MASK 0xffffffff
+#define VGT_DMA_MAX_SIZE__MAX_SIZE__SHIFT 0x0
+#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE_MASK 0x3f
+#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT 0x0
+#define VGT_DMA_CONTROL__PRIMGROUP_SIZE_MASK 0xffff
+#define VGT_DMA_CONTROL__PRIMGROUP_SIZE__SHIFT 0x0
+#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP_MASK 0x20000
+#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP__SHIFT 0x11
+#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP_MASK 0x100000
+#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP__SHIFT 0x14
+#define VGT_IMMED_DATA__DATA_MASK 0xffffffff
+#define VGT_IMMED_DATA__DATA__SHIFT 0x0
+#define VGT_INDEX_TYPE__INDEX_TYPE_MASK 0x3
+#define VGT_INDEX_TYPE__INDEX_TYPE__SHIFT 0x0
+#define VGT_NUM_INDICES__NUM_INDICES_MASK 0xffffffff
+#define VGT_NUM_INDICES__NUM_INDICES__SHIFT 0x0
+#define VGT_NUM_INSTANCES__NUM_INSTANCES_MASK 0xffffffff
+#define VGT_NUM_INSTANCES__NUM_INSTANCES__SHIFT 0x0
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE_MASK 0x3f
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT 0x0
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN_MASK 0x1
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN__SHIFT 0x0
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI_MASK 0x2
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI__SHIFT 0x1
+#define VGT_PRIMITIVEID_RESET__VALUE_MASK 0xffffffff
+#define VGT_PRIMITIVEID_RESET__VALUE__SHIFT 0x0
+#define VGT_VTX_CNT_EN__VTX_CNT_EN_MASK 0x1
+#define VGT_VTX_CNT_EN__VTX_CNT_EN__SHIFT 0x0
+#define VGT_REUSE_OFF__REUSE_OFF_MASK 0x1
+#define VGT_REUSE_OFF__REUSE_OFF__SHIFT 0x0
+#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE_MASK 0xffffffff
+#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE__SHIFT 0x0
+#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE_MASK 0xffffffff
+#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE__SHIFT 0x0
+#define VGT_MAX_VTX_INDX__MAX_INDX_MASK 0xffffffff
+#define VGT_MAX_VTX_INDX__MAX_INDX__SHIFT 0x0
+#define VGT_MIN_VTX_INDX__MIN_INDX_MASK 0xffffffff
+#define VGT_MIN_VTX_INDX__MIN_INDX__SHIFT 0x0
+#define VGT_INDX_OFFSET__INDX_OFFSET_MASK 0xffffffff
+#define VGT_INDX_OFFSET__INDX_OFFSET__SHIFT 0x0
+#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH_MASK 0xff
+#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH__SHIFT 0x0
+#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST_MASK 0x7f
+#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST__SHIFT 0x0
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK 0xffffffff
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT 0x0
+#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN_MASK 0x1
+#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN__SHIFT 0x0
+#define VGT_ENHANCE__MISC_MASK 0xffffffff
+#define VGT_ENHANCE__MISC__SHIFT 0x0
+#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT_MASK 0x7
+#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT__SHIFT 0x0
+#define VGT_HOS_CNTL__TESS_MODE_MASK 0x3
+#define VGT_HOS_CNTL__TESS_MODE__SHIFT 0x0
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS_MASK 0xffffffff
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS__SHIFT 0x0
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS_MASK 0xffffffff
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS__SHIFT 0x0
+#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH_MASK 0xff
+#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH__SHIFT 0x0
+#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE_MASK 0x1f
+#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE__SHIFT 0x0
+#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER_MASK 0x4000
+#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER__SHIFT 0xe
+#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS_MASK 0x8000
+#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS__SHIFT 0xf
+#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER_MASK 0x70000
+#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER__SHIFT 0x10
+#define VGT_GROUP_FIRST_DECR__FIRST_DECR_MASK 0xf
+#define VGT_GROUP_FIRST_DECR__FIRST_DECR__SHIFT 0x0
+#define VGT_GROUP_DECR__DECR_MASK 0xf
+#define VGT_GROUP_DECR__DECR__SHIFT 0x0
+#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN_MASK 0x1
+#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN__SHIFT 0x0
+#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN_MASK 0x2
+#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN__SHIFT 0x1
+#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN_MASK 0x4
+#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN__SHIFT 0x2
+#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN_MASK 0x8
+#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN__SHIFT 0x3
+#define VGT_GROUP_VECT_0_CNTL__STRIDE_MASK 0xff00
+#define VGT_GROUP_VECT_0_CNTL__STRIDE__SHIFT 0x8
+#define VGT_GROUP_VECT_0_CNTL__SHIFT_MASK 0xff0000
+#define VGT_GROUP_VECT_0_CNTL__SHIFT__SHIFT 0x10
+#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN_MASK 0x1
+#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN__SHIFT 0x0
+#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN_MASK 0x2
+#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN__SHIFT 0x1
+#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN_MASK 0x4
+#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN__SHIFT 0x2
+#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN_MASK 0x8
+#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN__SHIFT 0x3
+#define VGT_GROUP_VECT_1_CNTL__STRIDE_MASK 0xff00
+#define VGT_GROUP_VECT_1_CNTL__STRIDE__SHIFT 0x8
+#define VGT_GROUP_VECT_1_CNTL__SHIFT_MASK 0xff0000
+#define VGT_GROUP_VECT_1_CNTL__SHIFT__SHIFT 0x10
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV_MASK 0xf
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV__SHIFT 0x0
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET_MASK 0xf0
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET__SHIFT 0x4
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV_MASK 0xf00
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV__SHIFT 0x8
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET_MASK 0xf000
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET__SHIFT 0xc
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV_MASK 0xf0000
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV__SHIFT 0x10
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET_MASK 0xf00000
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET__SHIFT 0x14
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV_MASK 0xf000000
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV__SHIFT 0x18
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET_MASK 0xf0000000
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET__SHIFT 0x1c
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV_MASK 0xf
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV__SHIFT 0x0
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET_MASK 0xf0
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET__SHIFT 0x4
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV_MASK 0xf00
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV__SHIFT 0x8
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET_MASK 0xf000
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET__SHIFT 0xc
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV_MASK 0xf0000
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV__SHIFT 0x10
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET_MASK 0xf00000
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET__SHIFT 0x14
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV_MASK 0xf000000
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV__SHIFT 0x18
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET_MASK 0xf0000000
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET__SHIFT 0x1c
+#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT_MASK 0x3ff
+#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT__SHIFT 0x0
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH_MASK 0x1ff
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH__SHIFT 0x0
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA2DRAW_FIFO_DEPTH_MASK 0x7fe00
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA2DRAW_FIFO_DEPTH__SHIFT 0x9
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH_MASK 0x3f
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH__SHIFT 0x0
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH_MASK 0x3f
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH__SHIFT 0x0
+#define VGT_LAST_COPY_STATE__SRC_STATE_ID_MASK 0x7
+#define VGT_LAST_COPY_STATE__SRC_STATE_ID__SHIFT 0x0
+#define VGT_LAST_COPY_STATE__DST_STATE_ID_MASK 0x70000
+#define VGT_LAST_COPY_STATE__DST_STATE_ID__SHIFT 0x10
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK 0xffff0000
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT 0x10
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK 0xffff0000
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT 0x10
+#define VGT_GS_MODE__MODE_MASK 0x7
+#define VGT_GS_MODE__MODE__SHIFT 0x0
+#define VGT_GS_MODE__RESERVED_0_MASK 0x8
+#define VGT_GS_MODE__RESERVED_0__SHIFT 0x3
+#define VGT_GS_MODE__CUT_MODE_MASK 0x30
+#define VGT_GS_MODE__CUT_MODE__SHIFT 0x4
+#define VGT_GS_MODE__RESERVED_1_MASK 0x7c0
+#define VGT_GS_MODE__RESERVED_1__SHIFT 0x6
+#define VGT_GS_MODE__GS_C_PACK_EN_MASK 0x800
+#define VGT_GS_MODE__GS_C_PACK_EN__SHIFT 0xb
+#define VGT_GS_MODE__RESERVED_2_MASK 0x1000
+#define VGT_GS_MODE__RESERVED_2__SHIFT 0xc
+#define VGT_GS_MODE__ES_PASSTHRU_MASK 0x2000
+#define VGT_GS_MODE__ES_PASSTHRU__SHIFT 0xd
+#define VGT_GS_MODE__RESERVED_3_MASK 0x4000
+#define VGT_GS_MODE__RESERVED_3__SHIFT 0xe
+#define VGT_GS_MODE__RESERVED_4_MASK 0x8000
+#define VGT_GS_MODE__RESERVED_4__SHIFT 0xf
+#define VGT_GS_MODE__RESERVED_5_MASK 0x10000
+#define VGT_GS_MODE__RESERVED_5__SHIFT 0x10
+#define VGT_GS_MODE__PARTIAL_THD_AT_EOI_MASK 0x20000
+#define VGT_GS_MODE__PARTIAL_THD_AT_EOI__SHIFT 0x11
+#define VGT_GS_MODE__SUPPRESS_CUTS_MASK 0x40000
+#define VGT_GS_MODE__SUPPRESS_CUTS__SHIFT 0x12
+#define VGT_GS_MODE__ES_WRITE_OPTIMIZE_MASK 0x80000
+#define VGT_GS_MODE__ES_WRITE_OPTIMIZE__SHIFT 0x13
+#define VGT_GS_MODE__GS_WRITE_OPTIMIZE_MASK 0x100000
+#define VGT_GS_MODE__GS_WRITE_OPTIMIZE__SHIFT 0x14
+#define VGT_GS_MODE__ONCHIP_MASK 0x600000
+#define VGT_GS_MODE__ONCHIP__SHIFT 0x15
+#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP_MASK 0x7ff
+#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP__SHIFT 0x0
+#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP_MASK 0x3ff800
+#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP__SHIFT 0xb
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_MASK 0x3f
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE__SHIFT 0x0
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1_MASK 0x3f00
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1__SHIFT 0x8
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2_MASK 0x3f0000
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2__SHIFT 0x10
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3_MASK 0xfc00000
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3__SHIFT 0x16
+#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM_MASK 0x80000000
+#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM__SHIFT 0x1f
+#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION_MASK 0x3
+#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT 0x0
+#define VGT_CACHE_INVALIDATION__DIS_INSTANCING_OPT_MASK 0x10
+#define VGT_CACHE_INVALIDATION__DIS_INSTANCING_OPT__SHIFT 0x4
+#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER_MASK 0x20
+#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER__SHIFT 0x5
+#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN_MASK 0xc0
+#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT 0x6
+#define VGT_CACHE_INVALIDATION__USE_GS_DONE_MASK 0x200
+#define VGT_CACHE_INVALIDATION__USE_GS_DONE__SHIFT 0x9
+#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD_MASK 0x800
+#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD__SHIFT 0xb
+#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN_MASK 0x1000
+#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN__SHIFT 0xc
+#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH_MASK 0x2000
+#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH__SHIFT 0xd
+#define VGT_CACHE_INVALIDATION__ES_LIMIT_MASK 0x1f0000
+#define VGT_CACHE_INVALIDATION__ES_LIMIT__SHIFT 0x10
+#define VGT_RESET_DEBUG__GS_DISABLE_MASK 0x1
+#define VGT_RESET_DEBUG__GS_DISABLE__SHIFT 0x0
+#define VGT_RESET_DEBUG__TESS_DISABLE_MASK 0x2
+#define VGT_RESET_DEBUG__TESS_DISABLE__SHIFT 0x1
+#define VGT_RESET_DEBUG__WD_DISABLE_MASK 0x4
+#define VGT_RESET_DEBUG__WD_DISABLE__SHIFT 0x2
+#define VGT_STRMOUT_DELAY__SKIP_DELAY_MASK 0xff
+#define VGT_STRMOUT_DELAY__SKIP_DELAY__SHIFT 0x0
+#define VGT_STRMOUT_DELAY__SE0_WD_DELAY_MASK 0x700
+#define VGT_STRMOUT_DELAY__SE0_WD_DELAY__SHIFT 0x8
+#define VGT_STRMOUT_DELAY__SE1_WD_DELAY_MASK 0x3800
+#define VGT_STRMOUT_DELAY__SE1_WD_DELAY__SHIFT 0xb
+#define VGT_STRMOUT_DELAY__SE2_WD_DELAY_MASK 0x1c000
+#define VGT_STRMOUT_DELAY__SE2_WD_DELAY__SHIFT 0xe
+#define VGT_STRMOUT_DELAY__SE3_WD_DELAY_MASK 0xe0000
+#define VGT_STRMOUT_DELAY__SE3_WD_DELAY__SHIFT 0x11
+#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH_MASK 0x7f
+#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH__SHIFT 0x0
+#define VGT_FIFO_DEPTHS__RESERVED_0_MASK 0x80
+#define VGT_FIFO_DEPTHS__RESERVED_0__SHIFT 0x7
+#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH_MASK 0x3fff00
+#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH__SHIFT 0x8
+#define VGT_FIFO_DEPTHS__HSINPUT_FIFO_DEPTH_MASK 0xfc00000
+#define VGT_FIFO_DEPTHS__HSINPUT_FIFO_DEPTH__SHIFT 0x16
+#define VGT_GS_PER_ES__GS_PER_ES_MASK 0x7ff
+#define VGT_GS_PER_ES__GS_PER_ES__SHIFT 0x0
+#define VGT_ES_PER_GS__ES_PER_GS_MASK 0x7ff
+#define VGT_ES_PER_GS__ES_PER_GS__SHIFT 0x0
+#define VGT_GS_PER_VS__GS_PER_VS_MASK 0xf
+#define VGT_GS_PER_VS__GS_PER_VS__SHIFT 0x0
+#define VGT_GS_VERTEX_REUSE__VERT_REUSE_MASK 0x1f
+#define VGT_GS_VERTEX_REUSE__VERT_REUSE__SHIFT 0x0
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES_MASK 0x3
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES__SHIFT 0x0
+#define IA_CNTL_STATUS__IA_BUSY_MASK 0x1
+#define IA_CNTL_STATUS__IA_BUSY__SHIFT 0x0
+#define IA_CNTL_STATUS__IA_DMA_BUSY_MASK 0x2
+#define IA_CNTL_STATUS__IA_DMA_BUSY__SHIFT 0x1
+#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY_MASK 0x4
+#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY__SHIFT 0x2
+#define IA_CNTL_STATUS__IA_GRP_BUSY_MASK 0x8
+#define IA_CNTL_STATUS__IA_GRP_BUSY__SHIFT 0x3
+#define IA_CNTL_STATUS__IA_ADC_BUSY_MASK 0x10
+#define IA_CNTL_STATUS__IA_ADC_BUSY__SHIFT 0x4
+#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN_MASK 0x1
+#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN__SHIFT 0x0
+#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN_MASK 0x2
+#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN__SHIFT 0x1
+#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN_MASK 0x4
+#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN__SHIFT 0x2
+#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN_MASK 0x8
+#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN__SHIFT 0x3
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK 0x70
+#define VGT_STRMOUT_CONFIG__RAST_STREAM__SHIFT 0x4
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK_MASK 0xf00
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK__SHIFT 0x8
+#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK_MASK 0x80000000
+#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK__SHIFT 0x1f
+#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET__SHIFT 0x0
+#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE_MASK 0x3ff
+#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE__SHIFT 0x0
+#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE_MASK 0x3ff
+#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE__SHIFT 0x0
+#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE_MASK 0x3ff
+#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE__SHIFT 0x0
+#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE_MASK 0x3ff
+#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN_MASK 0xf
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN_MASK 0xf0
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN__SHIFT 0x4
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN_MASK 0xf00
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN__SHIFT 0x8
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN_MASK 0xf000
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN__SHIFT 0xc
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET_MASK 0xffffffff
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET__SHIFT 0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE_MASK 0xffffffff
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE__SHIFT 0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE_MASK 0x1ff
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE__SHIFT 0x0
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT_MASK 0x7ff
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT__SHIFT 0x0
+#define VGT_SHADER_STAGES_EN__LS_EN_MASK 0x3
+#define VGT_SHADER_STAGES_EN__LS_EN__SHIFT 0x0
+#define VGT_SHADER_STAGES_EN__HS_EN_MASK 0x4
+#define VGT_SHADER_STAGES_EN__HS_EN__SHIFT 0x2
+#define VGT_SHADER_STAGES_EN__ES_EN_MASK 0x18
+#define VGT_SHADER_STAGES_EN__ES_EN__SHIFT 0x3
+#define VGT_SHADER_STAGES_EN__GS_EN_MASK 0x20
+#define VGT_SHADER_STAGES_EN__GS_EN__SHIFT 0x5
+#define VGT_SHADER_STAGES_EN__VS_EN_MASK 0xc0
+#define VGT_SHADER_STAGES_EN__VS_EN__SHIFT 0x6
+#define VGT_SHADER_STAGES_EN__DYNAMIC_HS_MASK 0x100
+#define VGT_SHADER_STAGES_EN__DYNAMIC_HS__SHIFT 0x8
+#define VGT_SHADER_STAGES_EN__DISPATCH_DRAW_EN_MASK 0x200
+#define VGT_SHADER_STAGES_EN__DISPATCH_DRAW_EN__SHIFT 0x9
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_0_MASK 0x400
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_0__SHIFT 0xa
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_1_MASK 0x800
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_1__SHIFT 0xb
+#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN_MASK 0x1000
+#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN__SHIFT 0xc
+#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX_MASK 0xffffffff
+#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX__SHIFT 0x0
+#define VGT_LS_HS_CONFIG__NUM_PATCHES_MASK 0xff
+#define VGT_LS_HS_CONFIG__NUM_PATCHES__SHIFT 0x0
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK 0x3f00
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT 0x8
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP_MASK 0xfc000
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP__SHIFT 0xe
+#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK 0x3f00
+#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT 0x8
+#define VGT_TF_PARAM__TYPE_MASK 0x3
+#define VGT_TF_PARAM__TYPE__SHIFT 0x0
+#define VGT_TF_PARAM__PARTITIONING_MASK 0x1c
+#define VGT_TF_PARAM__PARTITIONING__SHIFT 0x2
+#define VGT_TF_PARAM__TOPOLOGY_MASK 0xe0
+#define VGT_TF_PARAM__TOPOLOGY__SHIFT 0x5
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS_MASK 0x100
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS__SHIFT 0x8
+#define VGT_TF_PARAM__DEPRECATED_MASK 0x200
+#define VGT_TF_PARAM__DEPRECATED__SHIFT 0x9
+#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD_MASK 0x3c00
+#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD__SHIFT 0xa
+#define VGT_TF_PARAM__DISABLE_DONUTS_MASK 0x4000
+#define VGT_TF_PARAM__DISABLE_DONUTS__SHIFT 0xe
+#define VGT_TF_PARAM__RDREQ_POLICY_MASK 0x8000
+#define VGT_TF_PARAM__RDREQ_POLICY__SHIFT 0xf
+#define VGT_TF_PARAM__DISTRIBUTION_MODE_MASK 0x60000
+#define VGT_TF_PARAM__DISTRIBUTION_MODE__SHIFT 0x11
+#define VGT_TF_PARAM__MTYPE_MASK 0x180000
+#define VGT_TF_PARAM__MTYPE__SHIFT 0x13
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE_MASK 0xff
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE__SHIFT 0x0
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI_MASK 0xff00
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI__SHIFT 0x8
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD_MASK 0xff0000
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD__SHIFT 0x10
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT_MASK 0xff000000
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT__SHIFT 0x18
+#define VGT_TF_RING_SIZE__SIZE_MASK 0xffff
+#define VGT_TF_RING_SIZE__SIZE__SHIFT 0x0
+#define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK 0x1
+#define VGT_SYS_CONFIG__DUAL_CORE_EN__SHIFT 0x0
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP_MASK 0x7e
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP__SHIFT 0x1
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE_MASK 0x80
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE__SHIFT 0x7
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING_MASK 0x1ff
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING__SHIFT 0x0
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY_MASK 0x600
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY__SHIFT 0x9
+#define VGT_TF_MEMORY_BASE__BASE_MASK 0xffffffff
+#define VGT_TF_MEMORY_BASE__BASE__SHIFT 0x0
+#define VGT_GS_INSTANCE_CNT__ENABLE_MASK 0x1
+#define VGT_GS_INSTANCE_CNT__ENABLE__SHIFT 0x0
+#define VGT_GS_INSTANCE_CNT__CNT_MASK 0x1fc
+#define VGT_GS_INSTANCE_CNT__CNT__SHIFT 0x2
+#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE_MASK 0xffff
+#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE__SHIFT 0x0
+#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON_MASK 0x10000
+#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON__SHIFT 0x10
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP_MASK 0x20000
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP__SHIFT 0x11
+#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON_MASK 0x40000
+#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON__SHIFT 0x12
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI_MASK 0x80000
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI__SHIFT 0x13
+#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP_MASK 0x100000
+#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP__SHIFT 0x14
+#define IA_MULTI_VGT_PARAM__MAX_PRIMGRP_IN_WAVE_MASK 0xf0000000
+#define IA_MULTI_VGT_PARAM__MAX_PRIMGRP_IN_WAVE__SHIFT 0x1c
+#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID_MASK 0xfff
+#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT 0x0
+#define VGT_ESGS_RING_SIZE__MEM_SIZE_MASK 0xffffffff
+#define VGT_ESGS_RING_SIZE__MEM_SIZE__SHIFT 0x0
+#define VGT_GSVS_RING_SIZE__MEM_SIZE_MASK 0xffffffff
+#define VGT_GSVS_RING_SIZE__MEM_SIZE__SHIFT 0x0
+#define VGT_GSVS_RING_OFFSET_1__OFFSET_MASK 0x7fff
+#define VGT_GSVS_RING_OFFSET_1__OFFSET__SHIFT 0x0
+#define VGT_GSVS_RING_OFFSET_2__OFFSET_MASK 0x7fff
+#define VGT_GSVS_RING_OFFSET_2__OFFSET__SHIFT 0x0
+#define VGT_GSVS_RING_OFFSET_3__OFFSET_MASK 0x7fff
+#define VGT_GSVS_RING_OFFSET_3__OFFSET__SHIFT 0x0
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE_MASK 0x7fff
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE__SHIFT 0x0
+#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE_MASK 0x7fff
+#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE__SHIFT 0x0
+#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE_MASK 0x7fff
+#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE__SHIFT 0x0
+#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE_MASK 0x7fff
+#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE__SHIFT 0x0
+#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE_MASK 0x7fff
+#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE__SHIFT 0x0
+#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE_MASK 0x7fff
+#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE__SHIFT 0x0
+#define WD_CNTL_STATUS__WD_BUSY_MASK 0x1
+#define WD_CNTL_STATUS__WD_BUSY__SHIFT 0x0
+#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY_MASK 0x2
+#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY__SHIFT 0x1
+#define WD_CNTL_STATUS__WD_SPL_DI_BUSY_MASK 0x4
+#define WD_CNTL_STATUS__WD_SPL_DI_BUSY__SHIFT 0x2
+#define WD_CNTL_STATUS__WD_ADC_BUSY_MASK 0x8
+#define WD_CNTL_STATUS__WD_ADC_BUSY__SHIFT 0x3
+#define WD_ENHANCE__MISC_MASK 0xffffffff
+#define WD_ENHANCE__MISC__SHIFT 0x0
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT_MASK 0x1fff
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT__SHIFT 0x0
+#define GFX_PIPE_CONTROL__RESERVED_MASK 0xe000
+#define GFX_PIPE_CONTROL__RESERVED__SHIFT 0xd
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN_MASK 0x10000
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN__SHIFT 0x10
+#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK 0x2000000
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT 0x19
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE_MASK 0x4000000
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE__SHIFT 0x1a
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE_MASK 0x10000000
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE__SHIFT 0x1c
+#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE_MASK 0x20000000
+#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE__SHIFT 0x1d
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK 0x2000000
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT 0x19
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE_MASK 0x4000000
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE__SHIFT 0x1a
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK 0xf
+#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT 0x0
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK 0x2000000
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT 0x19
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE_MASK 0x4000000
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE__SHIFT 0x1a
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b
+#define CGTT_WD_CLK_CTRL__TESS_OVERRIDE_MASK 0x10000000
+#define CGTT_WD_CLK_CTRL__TESS_OVERRIDE__SHIFT 0x1c
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK 0x20000000
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1d
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK 0x40000000
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT 0x1e
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f
+#define VGT_DEBUG_CNTL__VGT_DEBUG_INDX_MASK 0x3f
+#define VGT_DEBUG_CNTL__VGT_DEBUG_INDX__SHIFT 0x0
+#define VGT_DEBUG_CNTL__VGT_DEBUG_SEL_BUS_B_MASK 0x40
+#define VGT_DEBUG_CNTL__VGT_DEBUG_SEL_BUS_B__SHIFT 0x6
+#define VGT_DEBUG_DATA__DATA_MASK 0xffffffff
+#define VGT_DEBUG_DATA__DATA__SHIFT 0x0
+#define IA_DEBUG_CNTL__IA_DEBUG_INDX_MASK 0x3f
+#define IA_DEBUG_CNTL__IA_DEBUG_INDX__SHIFT 0x0
+#define IA_DEBUG_CNTL__IA_DEBUG_SEL_BUS_B_MASK 0x40
+#define IA_DEBUG_CNTL__IA_DEBUG_SEL_BUS_B__SHIFT 0x6
+#define IA_DEBUG_DATA__DATA_MASK 0xffffffff
+#define IA_DEBUG_DATA__DATA__SHIFT 0x0
+#define VGT_CNTL_STATUS__VGT_BUSY_MASK 0x1
+#define VGT_CNTL_STATUS__VGT_BUSY__SHIFT 0x0
+#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY_MASK 0x2
+#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY__SHIFT 0x1
+#define VGT_CNTL_STATUS__VGT_OUT_BUSY_MASK 0x4
+#define VGT_CNTL_STATUS__VGT_OUT_BUSY__SHIFT 0x2
+#define VGT_CNTL_STATUS__VGT_PT_BUSY_MASK 0x8
+#define VGT_CNTL_STATUS__VGT_PT_BUSY__SHIFT 0x3
+#define VGT_CNTL_STATUS__VGT_TE_BUSY_MASK 0x10
+#define VGT_CNTL_STATUS__VGT_TE_BUSY__SHIFT 0x4
+#define VGT_CNTL_STATUS__VGT_VR_BUSY_MASK 0x20
+#define VGT_CNTL_STATUS__VGT_VR_BUSY__SHIFT 0x5
+#define VGT_CNTL_STATUS__VGT_PI_BUSY_MASK 0x40
+#define VGT_CNTL_STATUS__VGT_PI_BUSY__SHIFT 0x6
+#define VGT_CNTL_STATUS__VGT_GS_BUSY_MASK 0x80
+#define VGT_CNTL_STATUS__VGT_GS_BUSY__SHIFT 0x7
+#define VGT_CNTL_STATUS__VGT_HS_BUSY_MASK 0x100
+#define VGT_CNTL_STATUS__VGT_HS_BUSY__SHIFT 0x8
+#define VGT_CNTL_STATUS__VGT_TE11_BUSY_MASK 0x200
+#define VGT_CNTL_STATUS__VGT_TE11_BUSY__SHIFT 0x9
+#define WD_DEBUG_CNTL__WD_DEBUG_INDX_MASK 0x3f
+#define WD_DEBUG_CNTL__WD_DEBUG_INDX__SHIFT 0x0
+#define WD_DEBUG_CNTL__WD_DEBUG_SEL_BUS_B_MASK 0x40
+#define WD_DEBUG_CNTL__WD_DEBUG_SEL_BUS_B__SHIFT 0x6
+#define WD_DEBUG_DATA__DATA_MASK 0xffffffff
+#define WD_DEBUG_DATA__DATA__SHIFT 0x0
+#define WD_QOS__DRAW_STALL_MASK 0x1
+#define WD_QOS__DRAW_STALL__SHIFT 0x0
+#define CC_GC_PRIM_CONFIG__INACTIVE_IA_MASK 0x30000
+#define CC_GC_PRIM_CONFIG__INACTIVE_IA__SHIFT 0x10
+#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA_MASK 0xf000000
+#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT 0x18
+#define GC_USER_PRIM_CONFIG__INACTIVE_IA_MASK 0x30000
+#define GC_USER_PRIM_CONFIG__INACTIVE_IA__SHIFT 0x10
+#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA_MASK 0xf000000
+#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT 0x18
+#define WD_DEBUG_REG0__wd_busy_extended_MASK 0x1
+#define WD_DEBUG_REG0__wd_busy_extended__SHIFT 0x0
+#define WD_DEBUG_REG0__wd_nodma_busy_extended_MASK 0x2
+#define WD_DEBUG_REG0__wd_nodma_busy_extended__SHIFT 0x1
+#define WD_DEBUG_REG0__wd_busy_MASK 0x4
+#define WD_DEBUG_REG0__wd_busy__SHIFT 0x2
+#define WD_DEBUG_REG0__wd_nodma_busy_MASK 0x8
+#define WD_DEBUG_REG0__wd_nodma_busy__SHIFT 0x3
+#define WD_DEBUG_REG0__rbiu_busy_MASK 0x10
+#define WD_DEBUG_REG0__rbiu_busy__SHIFT 0x4
+#define WD_DEBUG_REG0__spl_dma_busy_MASK 0x20
+#define WD_DEBUG_REG0__spl_dma_busy__SHIFT 0x5
+#define WD_DEBUG_REG0__spl_di_busy_MASK 0x40
+#define WD_DEBUG_REG0__spl_di_busy__SHIFT 0x6
+#define WD_DEBUG_REG0__vgt0_active_q_MASK 0x80
+#define WD_DEBUG_REG0__vgt0_active_q__SHIFT 0x7
+#define WD_DEBUG_REG0__vgt1_active_q_MASK 0x100
+#define WD_DEBUG_REG0__vgt1_active_q__SHIFT 0x8
+#define WD_DEBUG_REG0__spl_dma_p1_busy_MASK 0x200
+#define WD_DEBUG_REG0__spl_dma_p1_busy__SHIFT 0x9
+#define WD_DEBUG_REG0__rbiu_dr_p1_fifo_busy_MASK 0x400
+#define WD_DEBUG_REG0__rbiu_dr_p1_fifo_busy__SHIFT 0xa
+#define WD_DEBUG_REG0__rbiu_di_p1_fifo_busy_MASK 0x800
+#define WD_DEBUG_REG0__rbiu_di_p1_fifo_busy__SHIFT 0xb
+#define WD_DEBUG_REG0__SPARE2_MASK 0x1000
+#define WD_DEBUG_REG0__SPARE2__SHIFT 0xc
+#define WD_DEBUG_REG0__rbiu_dr_fifo_busy_MASK 0x2000
+#define WD_DEBUG_REG0__rbiu_dr_fifo_busy__SHIFT 0xd
+#define WD_DEBUG_REG0__rbiu_spl_dr_valid_MASK 0x4000
+#define WD_DEBUG_REG0__rbiu_spl_dr_valid__SHIFT 0xe
+#define WD_DEBUG_REG0__spl_rbiu_dr_read_MASK 0x8000
+#define WD_DEBUG_REG0__spl_rbiu_dr_read__SHIFT 0xf
+#define WD_DEBUG_REG0__SPARE3_MASK 0x10000
+#define WD_DEBUG_REG0__SPARE3__SHIFT 0x10
+#define WD_DEBUG_REG0__rbiu_di_fifo_busy_MASK 0x20000
+#define WD_DEBUG_REG0__rbiu_di_fifo_busy__SHIFT 0x11
+#define WD_DEBUG_REG0__rbiu_spl_di_valid_MASK 0x40000
+#define WD_DEBUG_REG0__rbiu_spl_di_valid__SHIFT 0x12
+#define WD_DEBUG_REG0__spl_rbiu_di_read_MASK 0x80000
+#define WD_DEBUG_REG0__spl_rbiu_di_read__SHIFT 0x13
+#define WD_DEBUG_REG0__se0_synced_q_MASK 0x100000
+#define WD_DEBUG_REG0__se0_synced_q__SHIFT 0x14
+#define WD_DEBUG_REG0__se1_synced_q_MASK 0x200000
+#define WD_DEBUG_REG0__se1_synced_q__SHIFT 0x15
+#define WD_DEBUG_REG0__se2_synced_q_MASK 0x400000
+#define WD_DEBUG_REG0__se2_synced_q__SHIFT 0x16
+#define WD_DEBUG_REG0__se3_synced_q_MASK 0x800000
+#define WD_DEBUG_REG0__se3_synced_q__SHIFT 0x17
+#define WD_DEBUG_REG0__reg_clk_busy_MASK 0x1000000
+#define WD_DEBUG_REG0__reg_clk_busy__SHIFT 0x18
+#define WD_DEBUG_REG0__input_clk_busy_MASK 0x2000000
+#define WD_DEBUG_REG0__input_clk_busy__SHIFT 0x19
+#define WD_DEBUG_REG0__core_clk_busy_MASK 0x4000000
+#define WD_DEBUG_REG0__core_clk_busy__SHIFT 0x1a
+#define WD_DEBUG_REG0__vgt2_active_q_MASK 0x8000000
+#define WD_DEBUG_REG0__vgt2_active_q__SHIFT 0x1b
+#define WD_DEBUG_REG0__sclk_reg_vld_MASK 0x10000000
+#define WD_DEBUG_REG0__sclk_reg_vld__SHIFT 0x1c
+#define WD_DEBUG_REG0__sclk_input_vld_MASK 0x20000000
+#define WD_DEBUG_REG0__sclk_input_vld__SHIFT 0x1d
+#define WD_DEBUG_REG0__sclk_core_vld_MASK 0x40000000
+#define WD_DEBUG_REG0__sclk_core_vld__SHIFT 0x1e
+#define WD_DEBUG_REG0__vgt3_active_q_MASK 0x80000000
+#define WD_DEBUG_REG0__vgt3_active_q__SHIFT 0x1f
+#define WD_DEBUG_REG1__grbm_fifo_empty_MASK 0x1
+#define WD_DEBUG_REG1__grbm_fifo_empty__SHIFT 0x0
+#define WD_DEBUG_REG1__grbm_fifo_full_MASK 0x2
+#define WD_DEBUG_REG1__grbm_fifo_full__SHIFT 0x1
+#define WD_DEBUG_REG1__grbm_fifo_we_MASK 0x4
+#define WD_DEBUG_REG1__grbm_fifo_we__SHIFT 0x2
+#define WD_DEBUG_REG1__grbm_fifo_re_MASK 0x8
+#define WD_DEBUG_REG1__grbm_fifo_re__SHIFT 0x3
+#define WD_DEBUG_REG1__draw_initiator_valid_q_MASK 0x10
+#define WD_DEBUG_REG1__draw_initiator_valid_q__SHIFT 0x4
+#define WD_DEBUG_REG1__event_initiator_valid_q_MASK 0x20
+#define WD_DEBUG_REG1__event_initiator_valid_q__SHIFT 0x5
+#define WD_DEBUG_REG1__event_addr_valid_q_MASK 0x40
+#define WD_DEBUG_REG1__event_addr_valid_q__SHIFT 0x6
+#define WD_DEBUG_REG1__dma_request_valid_q_MASK 0x80
+#define WD_DEBUG_REG1__dma_request_valid_q__SHIFT 0x7
+#define WD_DEBUG_REG1__SPARE0_MASK 0x100
+#define WD_DEBUG_REG1__SPARE0__SHIFT 0x8
+#define WD_DEBUG_REG1__min_indx_valid_q_MASK 0x200
+#define WD_DEBUG_REG1__min_indx_valid_q__SHIFT 0x9
+#define WD_DEBUG_REG1__max_indx_valid_q_MASK 0x400
+#define WD_DEBUG_REG1__max_indx_valid_q__SHIFT 0xa
+#define WD_DEBUG_REG1__indx_offset_valid_q_MASK 0x800
+#define WD_DEBUG_REG1__indx_offset_valid_q__SHIFT 0xb
+#define WD_DEBUG_REG1__grbm_fifo_rdata_reg_id_MASK 0x1f000
+#define WD_DEBUG_REG1__grbm_fifo_rdata_reg_id__SHIFT 0xc
+#define WD_DEBUG_REG1__grbm_fifo_rdata_state_MASK 0xe0000
+#define WD_DEBUG_REG1__grbm_fifo_rdata_state__SHIFT 0x11
+#define WD_DEBUG_REG1__free_cnt_q_MASK 0x3f00000
+#define WD_DEBUG_REG1__free_cnt_q__SHIFT 0x14
+#define WD_DEBUG_REG1__rbiu_di_fifo_we_MASK 0x4000000
+#define WD_DEBUG_REG1__rbiu_di_fifo_we__SHIFT 0x1a
+#define WD_DEBUG_REG1__rbiu_dr_fifo_we_MASK 0x8000000
+#define WD_DEBUG_REG1__rbiu_dr_fifo_we__SHIFT 0x1b
+#define WD_DEBUG_REG1__rbiu_di_fifo_empty_MASK 0x10000000
+#define WD_DEBUG_REG1__rbiu_di_fifo_empty__SHIFT 0x1c
+#define WD_DEBUG_REG1__rbiu_di_fifo_full_MASK 0x20000000
+#define WD_DEBUG_REG1__rbiu_di_fifo_full__SHIFT 0x1d
+#define WD_DEBUG_REG1__rbiu_dr_fifo_empty_MASK 0x40000000
+#define WD_DEBUG_REG1__rbiu_dr_fifo_empty__SHIFT 0x1e
+#define WD_DEBUG_REG1__rbiu_dr_fifo_full_MASK 0x80000000
+#define WD_DEBUG_REG1__rbiu_dr_fifo_full__SHIFT 0x1f
+#define WD_DEBUG_REG2__p1_grbm_fifo_empty_MASK 0x1
+#define WD_DEBUG_REG2__p1_grbm_fifo_empty__SHIFT 0x0
+#define WD_DEBUG_REG2__p1_grbm_fifo_full_MASK 0x2
+#define WD_DEBUG_REG2__p1_grbm_fifo_full__SHIFT 0x1
+#define WD_DEBUG_REG2__p1_grbm_fifo_we_MASK 0x4
+#define WD_DEBUG_REG2__p1_grbm_fifo_we__SHIFT 0x2
+#define WD_DEBUG_REG2__p1_grbm_fifo_re_MASK 0x8
+#define WD_DEBUG_REG2__p1_grbm_fifo_re__SHIFT 0x3
+#define WD_DEBUG_REG2__p1_draw_initiator_valid_q_MASK 0x10
+#define WD_DEBUG_REG2__p1_draw_initiator_valid_q__SHIFT 0x4
+#define WD_DEBUG_REG2__p1_event_initiator_valid_q_MASK 0x20
+#define WD_DEBUG_REG2__p1_event_initiator_valid_q__SHIFT 0x5
+#define WD_DEBUG_REG2__p1_event_addr_valid_q_MASK 0x40
+#define WD_DEBUG_REG2__p1_event_addr_valid_q__SHIFT 0x6
+#define WD_DEBUG_REG2__p1_dma_request_valid_q_MASK 0x80
+#define WD_DEBUG_REG2__p1_dma_request_valid_q__SHIFT 0x7
+#define WD_DEBUG_REG2__SPARE0_MASK 0x100
+#define WD_DEBUG_REG2__SPARE0__SHIFT 0x8
+#define WD_DEBUG_REG2__p1_min_indx_valid_q_MASK 0x200
+#define WD_DEBUG_REG2__p1_min_indx_valid_q__SHIFT 0x9
+#define WD_DEBUG_REG2__p1_max_indx_valid_q_MASK 0x400
+#define WD_DEBUG_REG2__p1_max_indx_valid_q__SHIFT 0xa
+#define WD_DEBUG_REG2__p1_indx_offset_valid_q_MASK 0x800
+#define WD_DEBUG_REG2__p1_indx_offset_valid_q__SHIFT 0xb
+#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_reg_id_MASK 0x1f000
+#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_reg_id__SHIFT 0xc
+#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_state_MASK 0xe0000
+#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_state__SHIFT 0x11
+#define WD_DEBUG_REG2__p1_free_cnt_q_MASK 0x3f00000
+#define WD_DEBUG_REG2__p1_free_cnt_q__SHIFT 0x14
+#define WD_DEBUG_REG2__p1_rbiu_di_fifo_we_MASK 0x4000000
+#define WD_DEBUG_REG2__p1_rbiu_di_fifo_we__SHIFT 0x1a
+#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_we_MASK 0x8000000
+#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_we__SHIFT 0x1b
+#define WD_DEBUG_REG2__p1_rbiu_di_fifo_empty_MASK 0x10000000
+#define WD_DEBUG_REG2__p1_rbiu_di_fifo_empty__SHIFT 0x1c
+#define WD_DEBUG_REG2__p1_rbiu_di_fifo_full_MASK 0x20000000
+#define WD_DEBUG_REG2__p1_rbiu_di_fifo_full__SHIFT 0x1d
+#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_empty_MASK 0x40000000
+#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_empty__SHIFT 0x1e
+#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_full_MASK 0x80000000
+#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_full__SHIFT 0x1f
+#define WD_DEBUG_REG3__rbiu_spl_dr_valid_MASK 0x1
+#define WD_DEBUG_REG3__rbiu_spl_dr_valid__SHIFT 0x0
+#define WD_DEBUG_REG3__SPARE0_MASK 0x2
+#define WD_DEBUG_REG3__SPARE0__SHIFT 0x1
+#define WD_DEBUG_REG3__pipe0_dr_MASK 0x4
+#define WD_DEBUG_REG3__pipe0_dr__SHIFT 0x2
+#define WD_DEBUG_REG3__pipe0_rtr_MASK 0x8
+#define WD_DEBUG_REG3__pipe0_rtr__SHIFT 0x3
+#define WD_DEBUG_REG3__pipe1_dr_MASK 0x10
+#define WD_DEBUG_REG3__pipe1_dr__SHIFT 0x4
+#define WD_DEBUG_REG3__pipe1_rtr_MASK 0x20
+#define WD_DEBUG_REG3__pipe1_rtr__SHIFT 0x5
+#define WD_DEBUG_REG3__wd_subdma_fifo_empty_MASK 0x40
+#define WD_DEBUG_REG3__wd_subdma_fifo_empty__SHIFT 0x6
+#define WD_DEBUG_REG3__wd_subdma_fifo_full_MASK 0x80
+#define WD_DEBUG_REG3__wd_subdma_fifo_full__SHIFT 0x7
+#define WD_DEBUG_REG3__dma_buf_type_p0_q_MASK 0x300
+#define WD_DEBUG_REG3__dma_buf_type_p0_q__SHIFT 0x8
+#define WD_DEBUG_REG3__dma_zero_indices_p0_q_MASK 0x400
+#define WD_DEBUG_REG3__dma_zero_indices_p0_q__SHIFT 0xa
+#define WD_DEBUG_REG3__dma_req_path_p3_q_MASK 0x800
+#define WD_DEBUG_REG3__dma_req_path_p3_q__SHIFT 0xb
+#define WD_DEBUG_REG3__dma_not_eop_p1_q_MASK 0x1000
+#define WD_DEBUG_REG3__dma_not_eop_p1_q__SHIFT 0xc
+#define WD_DEBUG_REG3__out_of_range_p4_MASK 0x2000
+#define WD_DEBUG_REG3__out_of_range_p4__SHIFT 0xd
+#define WD_DEBUG_REG3__last_sub_dma_p3_q_MASK 0x4000
+#define WD_DEBUG_REG3__last_sub_dma_p3_q__SHIFT 0xe
+#define WD_DEBUG_REG3__last_rdreq_of_sub_dma_p4_MASK 0x8000
+#define WD_DEBUG_REG3__last_rdreq_of_sub_dma_p4__SHIFT 0xf
+#define WD_DEBUG_REG3__WD_IA_dma_send_d_MASK 0x10000
+#define WD_DEBUG_REG3__WD_IA_dma_send_d__SHIFT 0x10
+#define WD_DEBUG_REG3__WD_IA_dma_rtr_MASK 0x20000
+#define WD_DEBUG_REG3__WD_IA_dma_rtr__SHIFT 0x11
+#define WD_DEBUG_REG3__WD_IA1_dma_send_d_MASK 0x40000
+#define WD_DEBUG_REG3__WD_IA1_dma_send_d__SHIFT 0x12
+#define WD_DEBUG_REG3__WD_IA1_dma_rtr_MASK 0x80000
+#define WD_DEBUG_REG3__WD_IA1_dma_rtr__SHIFT 0x13
+#define WD_DEBUG_REG3__last_inst_of_dma_p2_MASK 0x100000
+#define WD_DEBUG_REG3__last_inst_of_dma_p2__SHIFT 0x14
+#define WD_DEBUG_REG3__last_sd_of_inst_p2_MASK 0x200000
+#define WD_DEBUG_REG3__last_sd_of_inst_p2__SHIFT 0x15
+#define WD_DEBUG_REG3__last_sd_of_dma_p2_MASK 0x400000
+#define WD_DEBUG_REG3__last_sd_of_dma_p2__SHIFT 0x16
+#define WD_DEBUG_REG3__SPARE1_MASK 0x800000
+#define WD_DEBUG_REG3__SPARE1__SHIFT 0x17
+#define WD_DEBUG_REG3__WD_IA_dma_busy_MASK 0x1000000
+#define WD_DEBUG_REG3__WD_IA_dma_busy__SHIFT 0x18
+#define WD_DEBUG_REG3__WD_IA1_dma_busy_MASK 0x2000000
+#define WD_DEBUG_REG3__WD_IA1_dma_busy__SHIFT 0x19
+#define WD_DEBUG_REG3__send_to_ia1_p3_q_MASK 0x4000000
+#define WD_DEBUG_REG3__send_to_ia1_p3_q__SHIFT 0x1a
+#define WD_DEBUG_REG3__dma_wd_switch_on_eop_p3_q_MASK 0x8000000
+#define WD_DEBUG_REG3__dma_wd_switch_on_eop_p3_q__SHIFT 0x1b
+#define WD_DEBUG_REG3__pipe3_dr_MASK 0x10000000
+#define WD_DEBUG_REG3__pipe3_dr__SHIFT 0x1c
+#define WD_DEBUG_REG3__pipe3_rtr_MASK 0x20000000
+#define WD_DEBUG_REG3__pipe3_rtr__SHIFT 0x1d
+#define WD_DEBUG_REG3__wd_dma2draw_fifo_empty_MASK 0x40000000
+#define WD_DEBUG_REG3__wd_dma2draw_fifo_empty__SHIFT 0x1e
+#define WD_DEBUG_REG3__wd_dma2draw_fifo_full_MASK 0x80000000
+#define WD_DEBUG_REG3__wd_dma2draw_fifo_full__SHIFT 0x1f
+#define WD_DEBUG_REG4__rbiu_spl_di_valid_MASK 0x1
+#define WD_DEBUG_REG4__rbiu_spl_di_valid__SHIFT 0x0
+#define WD_DEBUG_REG4__spl_rbiu_di_read_MASK 0x2
+#define WD_DEBUG_REG4__spl_rbiu_di_read__SHIFT 0x1
+#define WD_DEBUG_REG4__rbiu_spl_p1_di_valid_MASK 0x4
+#define WD_DEBUG_REG4__rbiu_spl_p1_di_valid__SHIFT 0x2
+#define WD_DEBUG_REG4__spl_rbiu_p1_di_read_MASK 0x8
+#define WD_DEBUG_REG4__spl_rbiu_p1_di_read__SHIFT 0x3
+#define WD_DEBUG_REG4__pipe0_dr_MASK 0x10
+#define WD_DEBUG_REG4__pipe0_dr__SHIFT 0x4
+#define WD_DEBUG_REG4__pipe0_rtr_MASK 0x20
+#define WD_DEBUG_REG4__pipe0_rtr__SHIFT 0x5
+#define WD_DEBUG_REG4__pipe1_dr_MASK 0x40
+#define WD_DEBUG_REG4__pipe1_dr__SHIFT 0x6
+#define WD_DEBUG_REG4__pipe1_rtr_MASK 0x80
+#define WD_DEBUG_REG4__pipe1_rtr__SHIFT 0x7
+#define WD_DEBUG_REG4__pipe2_dr_MASK 0x100
+#define WD_DEBUG_REG4__pipe2_dr__SHIFT 0x8
+#define WD_DEBUG_REG4__pipe2_rtr_MASK 0x200
+#define WD_DEBUG_REG4__pipe2_rtr__SHIFT 0x9
+#define WD_DEBUG_REG4__pipe3_ld_MASK 0x400
+#define WD_DEBUG_REG4__pipe3_ld__SHIFT 0xa
+#define WD_DEBUG_REG4__pipe3_rtr_MASK 0x800
+#define WD_DEBUG_REG4__pipe3_rtr__SHIFT 0xb
+#define WD_DEBUG_REG4__WD_IA_draw_send_d_MASK 0x1000
+#define WD_DEBUG_REG4__WD_IA_draw_send_d__SHIFT 0xc
+#define WD_DEBUG_REG4__WD_IA_draw_rtr_MASK 0x2000
+#define WD_DEBUG_REG4__WD_IA_draw_rtr__SHIFT 0xd
+#define WD_DEBUG_REG4__di_type_p0_MASK 0xc000
+#define WD_DEBUG_REG4__di_type_p0__SHIFT 0xe
+#define WD_DEBUG_REG4__di_state_sel_p1_q_MASK 0x70000
+#define WD_DEBUG_REG4__di_state_sel_p1_q__SHIFT 0x10
+#define WD_DEBUG_REG4__di_wd_switch_on_eop_p1_q_MASK 0x80000
+#define WD_DEBUG_REG4__di_wd_switch_on_eop_p1_q__SHIFT 0x13
+#define WD_DEBUG_REG4__rbiu_spl_pipe0_lockout_MASK 0x100000
+#define WD_DEBUG_REG4__rbiu_spl_pipe0_lockout__SHIFT 0x14
+#define WD_DEBUG_REG4__last_inst_of_di_p2_MASK 0x200000
+#define WD_DEBUG_REG4__last_inst_of_di_p2__SHIFT 0x15
+#define WD_DEBUG_REG4__last_sd_of_inst_p2_MASK 0x400000
+#define WD_DEBUG_REG4__last_sd_of_inst_p2__SHIFT 0x16
+#define WD_DEBUG_REG4__last_sd_of_di_p2_MASK 0x800000
+#define WD_DEBUG_REG4__last_sd_of_di_p2__SHIFT 0x17
+#define WD_DEBUG_REG4__not_eop_wait_p1_q_MASK 0x1000000
+#define WD_DEBUG_REG4__not_eop_wait_p1_q__SHIFT 0x18
+#define WD_DEBUG_REG4__not_eop_wait_q_MASK 0x2000000
+#define WD_DEBUG_REG4__not_eop_wait_q__SHIFT 0x19
+#define WD_DEBUG_REG4__ext_event_wait_p1_q_MASK 0x4000000
+#define WD_DEBUG_REG4__ext_event_wait_p1_q__SHIFT 0x1a
+#define WD_DEBUG_REG4__ext_event_wait_q_MASK 0x8000000
+#define WD_DEBUG_REG4__ext_event_wait_q__SHIFT 0x1b
+#define WD_DEBUG_REG4__WD_IA1_draw_send_d_MASK 0x10000000
+#define WD_DEBUG_REG4__WD_IA1_draw_send_d__SHIFT 0x1c
+#define WD_DEBUG_REG4__WD_IA1_draw_rtr_MASK 0x20000000
+#define WD_DEBUG_REG4__WD_IA1_draw_rtr__SHIFT 0x1d
+#define WD_DEBUG_REG4__send_to_ia1_q_MASK 0x40000000
+#define WD_DEBUG_REG4__send_to_ia1_q__SHIFT 0x1e
+#define WD_DEBUG_REG4__dual_ia_mode_MASK 0x80000000
+#define WD_DEBUG_REG4__dual_ia_mode__SHIFT 0x1f
+#define WD_DEBUG_REG5__p1_rbiu_spl_dr_valid_MASK 0x1
+#define WD_DEBUG_REG5__p1_rbiu_spl_dr_valid__SHIFT 0x0
+#define WD_DEBUG_REG5__SPARE0_MASK 0x2
+#define WD_DEBUG_REG5__SPARE0__SHIFT 0x1
+#define WD_DEBUG_REG5__p1_pipe0_dr_MASK 0x4
+#define WD_DEBUG_REG5__p1_pipe0_dr__SHIFT 0x2
+#define WD_DEBUG_REG5__p1_pipe0_rtr_MASK 0x8
+#define WD_DEBUG_REG5__p1_pipe0_rtr__SHIFT 0x3
+#define WD_DEBUG_REG5__p1_pipe1_dr_MASK 0x10
+#define WD_DEBUG_REG5__p1_pipe1_dr__SHIFT 0x4
+#define WD_DEBUG_REG5__p1_pipe1_rtr_MASK 0x20
+#define WD_DEBUG_REG5__p1_pipe1_rtr__SHIFT 0x5
+#define WD_DEBUG_REG5__p1_wd_subdma_fifo_empty_MASK 0x40
+#define WD_DEBUG_REG5__p1_wd_subdma_fifo_empty__SHIFT 0x6
+#define WD_DEBUG_REG5__p1_wd_subdma_fifo_full_MASK 0x80
+#define WD_DEBUG_REG5__p1_wd_subdma_fifo_full__SHIFT 0x7
+#define WD_DEBUG_REG5__p1_dma_buf_type_p0_q_MASK 0x300
+#define WD_DEBUG_REG5__p1_dma_buf_type_p0_q__SHIFT 0x8
+#define WD_DEBUG_REG5__p1_dma_zero_indices_p0_q_MASK 0x400
+#define WD_DEBUG_REG5__p1_dma_zero_indices_p0_q__SHIFT 0xa
+#define WD_DEBUG_REG5__p1_dma_req_path_p3_q_MASK 0x800
+#define WD_DEBUG_REG5__p1_dma_req_path_p3_q__SHIFT 0xb
+#define WD_DEBUG_REG5__p1_dma_not_eop_p1_q_MASK 0x1000
+#define WD_DEBUG_REG5__p1_dma_not_eop_p1_q__SHIFT 0xc
+#define WD_DEBUG_REG5__p1_out_of_range_p4_MASK 0x2000
+#define WD_DEBUG_REG5__p1_out_of_range_p4__SHIFT 0xd
+#define WD_DEBUG_REG5__p1_last_sub_dma_p3_q_MASK 0x4000
+#define WD_DEBUG_REG5__p1_last_sub_dma_p3_q__SHIFT 0xe
+#define WD_DEBUG_REG5__p1_last_rdreq_of_sub_dma_p4_MASK 0x8000
+#define WD_DEBUG_REG5__p1_last_rdreq_of_sub_dma_p4__SHIFT 0xf
+#define WD_DEBUG_REG5__p1_WD_IA_dma_send_d_MASK 0x10000
+#define WD_DEBUG_REG5__p1_WD_IA_dma_send_d__SHIFT 0x10
+#define WD_DEBUG_REG5__p1_WD_IA_dma_rtr_MASK 0x20000
+#define WD_DEBUG_REG5__p1_WD_IA_dma_rtr__SHIFT 0x11
+#define WD_DEBUG_REG5__p1_WD_IA1_dma_send_d_MASK 0x40000
+#define WD_DEBUG_REG5__p1_WD_IA1_dma_send_d__SHIFT 0x12
+#define WD_DEBUG_REG5__p1_WD_IA1_dma_rtr_MASK 0x80000
+#define WD_DEBUG_REG5__p1_WD_IA1_dma_rtr__SHIFT 0x13
+#define WD_DEBUG_REG5__p1_last_inst_of_dma_p2_MASK 0x100000
+#define WD_DEBUG_REG5__p1_last_inst_of_dma_p2__SHIFT 0x14
+#define WD_DEBUG_REG5__p1_last_sd_of_inst_p2_MASK 0x200000
+#define WD_DEBUG_REG5__p1_last_sd_of_inst_p2__SHIFT 0x15
+#define WD_DEBUG_REG5__p1_last_sd_of_dma_p2_MASK 0x400000
+#define WD_DEBUG_REG5__p1_last_sd_of_dma_p2__SHIFT 0x16
+#define WD_DEBUG_REG5__SPARE1_MASK 0x800000
+#define WD_DEBUG_REG5__SPARE1__SHIFT 0x17
+#define WD_DEBUG_REG5__p1_WD_IA_dma_busy_MASK 0x1000000
+#define WD_DEBUG_REG5__p1_WD_IA_dma_busy__SHIFT 0x18
+#define WD_DEBUG_REG5__p1_WD_IA1_dma_busy_MASK 0x2000000
+#define WD_DEBUG_REG5__p1_WD_IA1_dma_busy__SHIFT 0x19
+#define WD_DEBUG_REG5__p1_send_to_ia1_p3_q_MASK 0x4000000
+#define WD_DEBUG_REG5__p1_send_to_ia1_p3_q__SHIFT 0x1a
+#define WD_DEBUG_REG5__p1_dma_wd_switch_on_eop_p3_q_MASK 0x8000000
+#define WD_DEBUG_REG5__p1_dma_wd_switch_on_eop_p3_q__SHIFT 0x1b
+#define WD_DEBUG_REG5__p1_pipe3_dr_MASK 0x10000000
+#define WD_DEBUG_REG5__p1_pipe3_dr__SHIFT 0x1c
+#define WD_DEBUG_REG5__p1_pipe3_rtr_MASK 0x20000000
+#define WD_DEBUG_REG5__p1_pipe3_rtr__SHIFT 0x1d
+#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_empty_MASK 0x40000000
+#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_empty__SHIFT 0x1e
+#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_full_MASK 0x80000000
+#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_full__SHIFT 0x1f
+#define WD_DEBUG_REG6__WD_IA_draw_eop_MASK 0xffffffff
+#define WD_DEBUG_REG6__WD_IA_draw_eop__SHIFT 0x0
+#define WD_DEBUG_REG7__SE0VGT_WD_thdgrp_send_in_MASK 0x1
+#define WD_DEBUG_REG7__SE0VGT_WD_thdgrp_send_in__SHIFT 0x0
+#define WD_DEBUG_REG7__wd_arb_se0_input_fifo_re_MASK 0x2
+#define WD_DEBUG_REG7__wd_arb_se0_input_fifo_re__SHIFT 0x1
+#define WD_DEBUG_REG7__wd_arb_se0_input_fifo_empty_MASK 0x4
+#define WD_DEBUG_REG7__wd_arb_se0_input_fifo_empty__SHIFT 0x2
+#define WD_DEBUG_REG7__wd_arb_se0_input_fifo_full_MASK 0x8
+#define WD_DEBUG_REG7__wd_arb_se0_input_fifo_full__SHIFT 0x3
+#define WD_DEBUG_REG7__SE1VGT_WD_thdgrp_send_in_MASK 0x10
+#define WD_DEBUG_REG7__SE1VGT_WD_thdgrp_send_in__SHIFT 0x4
+#define WD_DEBUG_REG7__wd_arb_se1_input_fifo_re_MASK 0x20
+#define WD_DEBUG_REG7__wd_arb_se1_input_fifo_re__SHIFT 0x5
+#define WD_DEBUG_REG7__wd_arb_se1_input_fifo_empty_MASK 0x40
+#define WD_DEBUG_REG7__wd_arb_se1_input_fifo_empty__SHIFT 0x6
+#define WD_DEBUG_REG7__wd_arb_se1_input_fifo_full_MASK 0x80
+#define WD_DEBUG_REG7__wd_arb_se1_input_fifo_full__SHIFT 0x7
+#define WD_DEBUG_REG7__SPARE1_MASK 0xf00
+#define WD_DEBUG_REG7__SPARE1__SHIFT 0x8
+#define WD_DEBUG_REG7__SPARE2_MASK 0xf000
+#define WD_DEBUG_REG7__SPARE2__SHIFT 0xc
+#define WD_DEBUG_REG7__te11_arb_state_q_MASK 0x70000
+#define WD_DEBUG_REG7__te11_arb_state_q__SHIFT 0x10
+#define WD_DEBUG_REG7__SPARE5_MASK 0x80000
+#define WD_DEBUG_REG7__SPARE5__SHIFT 0x13
+#define WD_DEBUG_REG7__se0_thdgrp_is_event_MASK 0x100000
+#define WD_DEBUG_REG7__se0_thdgrp_is_event__SHIFT 0x14
+#define WD_DEBUG_REG7__se0_thdgrp_eop_MASK 0x200000
+#define WD_DEBUG_REG7__se0_thdgrp_eop__SHIFT 0x15
+#define WD_DEBUG_REG7__se1_thdgrp_is_event_MASK 0x400000
+#define WD_DEBUG_REG7__se1_thdgrp_is_event__SHIFT 0x16
+#define WD_DEBUG_REG7__se1_thdgrp_eop_MASK 0x800000
+#define WD_DEBUG_REG7__se1_thdgrp_eop__SHIFT 0x17
+#define WD_DEBUG_REG7__SPARE6_MASK 0xf000000
+#define WD_DEBUG_REG7__SPARE6__SHIFT 0x18
+#define WD_DEBUG_REG7__tfreq_arb_tgroup_rtr_MASK 0x10000000
+#define WD_DEBUG_REG7__tfreq_arb_tgroup_rtr__SHIFT 0x1c
+#define WD_DEBUG_REG7__arb_tfreq_tgroup_rts_MASK 0x20000000
+#define WD_DEBUG_REG7__arb_tfreq_tgroup_rts__SHIFT 0x1d
+#define WD_DEBUG_REG7__arb_tfreq_tgroup_event_MASK 0x40000000
+#define WD_DEBUG_REG7__arb_tfreq_tgroup_event__SHIFT 0x1e
+#define WD_DEBUG_REG7__te11_arb_busy_MASK 0x80000000
+#define WD_DEBUG_REG7__te11_arb_busy__SHIFT 0x1f
+#define WD_DEBUG_REG8__pipe0_dr_MASK 0x1
+#define WD_DEBUG_REG8__pipe0_dr__SHIFT 0x0
+#define WD_DEBUG_REG8__pipe1_dr_MASK 0x2
+#define WD_DEBUG_REG8__pipe1_dr__SHIFT 0x1
+#define WD_DEBUG_REG8__pipe0_rtr_MASK 0x4
+#define WD_DEBUG_REG8__pipe0_rtr__SHIFT 0x2
+#define WD_DEBUG_REG8__pipe1_rtr_MASK 0x8
+#define WD_DEBUG_REG8__pipe1_rtr__SHIFT 0x3
+#define WD_DEBUG_REG8__tfreq_tg_fifo_empty_MASK 0x10
+#define WD_DEBUG_REG8__tfreq_tg_fifo_empty__SHIFT 0x4
+#define WD_DEBUG_REG8__tfreq_tg_fifo_full_MASK 0x20
+#define WD_DEBUG_REG8__tfreq_tg_fifo_full__SHIFT 0x5
+#define WD_DEBUG_REG8__tf_data_fifo_busy_q_MASK 0x40
+#define WD_DEBUG_REG8__tf_data_fifo_busy_q__SHIFT 0x6
+#define WD_DEBUG_REG8__tf_data_fifo_rtr_q_MASK 0x80
+#define WD_DEBUG_REG8__tf_data_fifo_rtr_q__SHIFT 0x7
+#define WD_DEBUG_REG8__tf_skid_fifo_empty_MASK 0x100
+#define WD_DEBUG_REG8__tf_skid_fifo_empty__SHIFT 0x8
+#define WD_DEBUG_REG8__tf_skid_fifo_full_MASK 0x200
+#define WD_DEBUG_REG8__tf_skid_fifo_full__SHIFT 0x9
+#define WD_DEBUG_REG8__wd_tc_rdreq_rtr_q_MASK 0x400
+#define WD_DEBUG_REG8__wd_tc_rdreq_rtr_q__SHIFT 0xa
+#define WD_DEBUG_REG8__last_req_of_tg_p2_MASK 0x800
+#define WD_DEBUG_REG8__last_req_of_tg_p2__SHIFT 0xb
+#define WD_DEBUG_REG8__se0spi_wd_hs_done_cnt_q_MASK 0x3f000
+#define WD_DEBUG_REG8__se0spi_wd_hs_done_cnt_q__SHIFT 0xc
+#define WD_DEBUG_REG8__event_flag_p1_q_MASK 0x40000
+#define WD_DEBUG_REG8__event_flag_p1_q__SHIFT 0x12
+#define WD_DEBUG_REG8__null_flag_p1_q_MASK 0x80000
+#define WD_DEBUG_REG8__null_flag_p1_q__SHIFT 0x13
+#define WD_DEBUG_REG8__tf_data_fifo_cnt_q_MASK 0x7f00000
+#define WD_DEBUG_REG8__tf_data_fifo_cnt_q__SHIFT 0x14
+#define WD_DEBUG_REG8__second_tf_ret_data_q_MASK 0x8000000
+#define WD_DEBUG_REG8__second_tf_ret_data_q__SHIFT 0x1b
+#define WD_DEBUG_REG8__first_req_of_tg_p1_q_MASK 0x10000000
+#define WD_DEBUG_REG8__first_req_of_tg_p1_q__SHIFT 0x1c
+#define WD_DEBUG_REG8__WD_TC_rdreq_send_out_MASK 0x20000000
+#define WD_DEBUG_REG8__WD_TC_rdreq_send_out__SHIFT 0x1d
+#define WD_DEBUG_REG8__WD_TC_rdnfo_stall_out_MASK 0x40000000
+#define WD_DEBUG_REG8__WD_TC_rdnfo_stall_out__SHIFT 0x1e
+#define WD_DEBUG_REG8__TC_WD_rdret_valid_in_MASK 0x80000000
+#define WD_DEBUG_REG8__TC_WD_rdret_valid_in__SHIFT 0x1f
+#define WD_DEBUG_REG9__pipe0_dr_MASK 0x1
+#define WD_DEBUG_REG9__pipe0_dr__SHIFT 0x0
+#define WD_DEBUG_REG9__pipec_tf_dr_MASK 0x2
+#define WD_DEBUG_REG9__pipec_tf_dr__SHIFT 0x1
+#define WD_DEBUG_REG9__pipe2_dr_MASK 0x4
+#define WD_DEBUG_REG9__pipe2_dr__SHIFT 0x2
+#define WD_DEBUG_REG9__event_or_null_flags_p0_q_MASK 0x8
+#define WD_DEBUG_REG9__event_or_null_flags_p0_q__SHIFT 0x3
+#define WD_DEBUG_REG9__pipe0_rtr_MASK 0x10
+#define WD_DEBUG_REG9__pipe0_rtr__SHIFT 0x4
+#define WD_DEBUG_REG9__pipe1_rtr_MASK 0x20
+#define WD_DEBUG_REG9__pipe1_rtr__SHIFT 0x5
+#define WD_DEBUG_REG9__pipec_tf_rtr_MASK 0x40
+#define WD_DEBUG_REG9__pipec_tf_rtr__SHIFT 0x6
+#define WD_DEBUG_REG9__pipe2_rtr_MASK 0x80
+#define WD_DEBUG_REG9__pipe2_rtr__SHIFT 0x7
+#define WD_DEBUG_REG9__ttp_patch_fifo_full_MASK 0x100
+#define WD_DEBUG_REG9__ttp_patch_fifo_full__SHIFT 0x8
+#define WD_DEBUG_REG9__ttp_patch_fifo_empty_MASK 0x200
+#define WD_DEBUG_REG9__ttp_patch_fifo_empty__SHIFT 0x9
+#define WD_DEBUG_REG9__ttp_tf_fifo_empty_MASK 0x400
+#define WD_DEBUG_REG9__ttp_tf_fifo_empty__SHIFT 0xa
+#define WD_DEBUG_REG9__SPARE0_MASK 0xf800
+#define WD_DEBUG_REG9__SPARE0__SHIFT 0xb
+#define WD_DEBUG_REG9__tf_fetch_state_q_MASK 0x70000
+#define WD_DEBUG_REG9__tf_fetch_state_q__SHIFT 0x10
+#define WD_DEBUG_REG9__last_patch_of_tg_MASK 0x80000
+#define WD_DEBUG_REG9__last_patch_of_tg__SHIFT 0x13
+#define WD_DEBUG_REG9__tf_pointer_p0_q_MASK 0xf00000
+#define WD_DEBUG_REG9__tf_pointer_p0_q__SHIFT 0x14
+#define WD_DEBUG_REG9__dynamic_hs_p0_q_MASK 0x1000000
+#define WD_DEBUG_REG9__dynamic_hs_p0_q__SHIFT 0x18
+#define WD_DEBUG_REG9__first_fetch_of_tg_p0_q_MASK 0x2000000
+#define WD_DEBUG_REG9__first_fetch_of_tg_p0_q__SHIFT 0x19
+#define WD_DEBUG_REG9__mem_is_even_MASK 0x4000000
+#define WD_DEBUG_REG9__mem_is_even__SHIFT 0x1a
+#define WD_DEBUG_REG9__SPARE1_MASK 0x8000000
+#define WD_DEBUG_REG9__SPARE1__SHIFT 0x1b
+#define WD_DEBUG_REG9__SPARE2_MASK 0x30000000
+#define WD_DEBUG_REG9__SPARE2__SHIFT 0x1c
+#define WD_DEBUG_REG9__pipe4_dr_MASK 0x40000000
+#define WD_DEBUG_REG9__pipe4_dr__SHIFT 0x1e
+#define WD_DEBUG_REG9__pipe4_rtr_MASK 0x80000000
+#define WD_DEBUG_REG9__pipe4_rtr__SHIFT 0x1f
+#define WD_DEBUG_REG10__ttp_pd_patch_rts_MASK 0x1
+#define WD_DEBUG_REG10__ttp_pd_patch_rts__SHIFT 0x0
+#define WD_DEBUG_REG10__ttp_pd_is_event_MASK 0x2
+#define WD_DEBUG_REG10__ttp_pd_is_event__SHIFT 0x1
+#define WD_DEBUG_REG10__ttp_pd_eopg_MASK 0x4
+#define WD_DEBUG_REG10__ttp_pd_eopg__SHIFT 0x2
+#define WD_DEBUG_REG10__ttp_pd_eop_MASK 0x8
+#define WD_DEBUG_REG10__ttp_pd_eop__SHIFT 0x3
+#define WD_DEBUG_REG10__pipe0_dr_MASK 0x10
+#define WD_DEBUG_REG10__pipe0_dr__SHIFT 0x4
+#define WD_DEBUG_REG10__pipe1_dr_MASK 0x20
+#define WD_DEBUG_REG10__pipe1_dr__SHIFT 0x5
+#define WD_DEBUG_REG10__pipe0_rtr_MASK 0x40
+#define WD_DEBUG_REG10__pipe0_rtr__SHIFT 0x6
+#define WD_DEBUG_REG10__pipe1_rtr_MASK 0x80
+#define WD_DEBUG_REG10__pipe1_rtr__SHIFT 0x7
+#define WD_DEBUG_REG10__donut_en_p1_q_MASK 0x100
+#define WD_DEBUG_REG10__donut_en_p1_q__SHIFT 0x8
+#define WD_DEBUG_REG10__donut_se_switch_p2_MASK 0x200
+#define WD_DEBUG_REG10__donut_se_switch_p2__SHIFT 0x9
+#define WD_DEBUG_REG10__patch_se_switch_p2_MASK 0x400
+#define WD_DEBUG_REG10__patch_se_switch_p2__SHIFT 0xa
+#define WD_DEBUG_REG10__last_donut_switch_p2_MASK 0x800
+#define WD_DEBUG_REG10__last_donut_switch_p2__SHIFT 0xb
+#define WD_DEBUG_REG10__last_donut_of_patch_p2_MASK 0x1000
+#define WD_DEBUG_REG10__last_donut_of_patch_p2__SHIFT 0xc
+#define WD_DEBUG_REG10__is_event_p1_q_MASK 0x2000
+#define WD_DEBUG_REG10__is_event_p1_q__SHIFT 0xd
+#define WD_DEBUG_REG10__eopg_p1_q_MASK 0x4000
+#define WD_DEBUG_REG10__eopg_p1_q__SHIFT 0xe
+#define WD_DEBUG_REG10__eop_p1_q_MASK 0x8000
+#define WD_DEBUG_REG10__eop_p1_q__SHIFT 0xf
+#define WD_DEBUG_REG10__patch_accum_q_MASK 0xff0000
+#define WD_DEBUG_REG10__patch_accum_q__SHIFT 0x10
+#define WD_DEBUG_REG10__wd_te11_out_se0_fifo_full_MASK 0x1000000
+#define WD_DEBUG_REG10__wd_te11_out_se0_fifo_full__SHIFT 0x18
+#define WD_DEBUG_REG10__wd_te11_out_se0_fifo_empty_MASK 0x2000000
+#define WD_DEBUG_REG10__wd_te11_out_se0_fifo_empty__SHIFT 0x19
+#define WD_DEBUG_REG10__wd_te11_out_se1_fifo_full_MASK 0x4000000
+#define WD_DEBUG_REG10__wd_te11_out_se1_fifo_full__SHIFT 0x1a
+#define WD_DEBUG_REG10__wd_te11_out_se1_fifo_empty_MASK 0x8000000
+#define WD_DEBUG_REG10__wd_te11_out_se1_fifo_empty__SHIFT 0x1b
+#define WD_DEBUG_REG10__wd_te11_out_se2_fifo_full_MASK 0x10000000
+#define WD_DEBUG_REG10__wd_te11_out_se2_fifo_full__SHIFT 0x1c
+#define WD_DEBUG_REG10__wd_te11_out_se2_fifo_empty_MASK 0x20000000
+#define WD_DEBUG_REG10__wd_te11_out_se2_fifo_empty__SHIFT 0x1d
+#define WD_DEBUG_REG10__wd_te11_out_se3_fifo_full_MASK 0x40000000
+#define WD_DEBUG_REG10__wd_te11_out_se3_fifo_full__SHIFT 0x1e
+#define WD_DEBUG_REG10__wd_te11_out_se3_fifo_empty_MASK 0x80000000
+#define WD_DEBUG_REG10__wd_te11_out_se3_fifo_empty__SHIFT 0x1f
+#define IA_DEBUG_REG0__ia_busy_extended_MASK 0x1
+#define IA_DEBUG_REG0__ia_busy_extended__SHIFT 0x0
+#define IA_DEBUG_REG0__ia_nodma_busy_extended_MASK 0x2
+#define IA_DEBUG_REG0__ia_nodma_busy_extended__SHIFT 0x1
+#define IA_DEBUG_REG0__ia_busy_MASK 0x4
+#define IA_DEBUG_REG0__ia_busy__SHIFT 0x2
+#define IA_DEBUG_REG0__ia_nodma_busy_MASK 0x8
+#define IA_DEBUG_REG0__ia_nodma_busy__SHIFT 0x3
+#define IA_DEBUG_REG0__SPARE0_MASK 0x10
+#define IA_DEBUG_REG0__SPARE0__SHIFT 0x4
+#define IA_DEBUG_REG0__dma_req_busy_MASK 0x20
+#define IA_DEBUG_REG0__dma_req_busy__SHIFT 0x5
+#define IA_DEBUG_REG0__dma_busy_MASK 0x40
+#define IA_DEBUG_REG0__dma_busy__SHIFT 0x6
+#define IA_DEBUG_REG0__mc_xl8r_busy_MASK 0x80
+#define IA_DEBUG_REG0__mc_xl8r_busy__SHIFT 0x7
+#define IA_DEBUG_REG0__grp_busy_MASK 0x100
+#define IA_DEBUG_REG0__grp_busy__SHIFT 0x8
+#define IA_DEBUG_REG0__SPARE1_MASK 0x200
+#define IA_DEBUG_REG0__SPARE1__SHIFT 0x9
+#define IA_DEBUG_REG0__dma_grp_valid_MASK 0x400
+#define IA_DEBUG_REG0__dma_grp_valid__SHIFT 0xa
+#define IA_DEBUG_REG0__grp_dma_read_MASK 0x800
+#define IA_DEBUG_REG0__grp_dma_read__SHIFT 0xb
+#define IA_DEBUG_REG0__dma_grp_hp_valid_MASK 0x1000
+#define IA_DEBUG_REG0__dma_grp_hp_valid__SHIFT 0xc
+#define IA_DEBUG_REG0__grp_dma_hp_read_MASK 0x2000
+#define IA_DEBUG_REG0__grp_dma_hp_read__SHIFT 0xd
+#define IA_DEBUG_REG0__SPARE2_MASK 0xffc000
+#define IA_DEBUG_REG0__SPARE2__SHIFT 0xe
+#define IA_DEBUG_REG0__reg_clk_busy_MASK 0x1000000
+#define IA_DEBUG_REG0__reg_clk_busy__SHIFT 0x18
+#define IA_DEBUG_REG0__core_clk_busy_MASK 0x2000000
+#define IA_DEBUG_REG0__core_clk_busy__SHIFT 0x19
+#define IA_DEBUG_REG0__SPARE3_MASK 0x4000000
+#define IA_DEBUG_REG0__SPARE3__SHIFT 0x1a
+#define IA_DEBUG_REG0__SPARE4_MASK 0x8000000
+#define IA_DEBUG_REG0__SPARE4__SHIFT 0x1b
+#define IA_DEBUG_REG0__sclk_reg_vld_MASK 0x10000000
+#define IA_DEBUG_REG0__sclk_reg_vld__SHIFT 0x1c
+#define IA_DEBUG_REG0__sclk_core_vld_MASK 0x20000000
+#define IA_DEBUG_REG0__sclk_core_vld__SHIFT 0x1d
+#define IA_DEBUG_REG0__SPARE5_MASK 0x40000000
+#define IA_DEBUG_REG0__SPARE5__SHIFT 0x1e
+#define IA_DEBUG_REG0__SPARE6_MASK 0x80000000
+#define IA_DEBUG_REG0__SPARE6__SHIFT 0x1f
+#define IA_DEBUG_REG1__dma_input_fifo_empty_MASK 0x1
+#define IA_DEBUG_REG1__dma_input_fifo_empty__SHIFT 0x0
+#define IA_DEBUG_REG1__dma_input_fifo_full_MASK 0x2
+#define IA_DEBUG_REG1__dma_input_fifo_full__SHIFT 0x1
+#define IA_DEBUG_REG1__start_new_packet_MASK 0x4
+#define IA_DEBUG_REG1__start_new_packet__SHIFT 0x2
+#define IA_DEBUG_REG1__dma_rdreq_dr_q_MASK 0x8
+#define IA_DEBUG_REG1__dma_rdreq_dr_q__SHIFT 0x3
+#define IA_DEBUG_REG1__dma_zero_indices_q_MASK 0x10
+#define IA_DEBUG_REG1__dma_zero_indices_q__SHIFT 0x4
+#define IA_DEBUG_REG1__dma_buf_type_q_MASK 0x60
+#define IA_DEBUG_REG1__dma_buf_type_q__SHIFT 0x5
+#define IA_DEBUG_REG1__dma_req_path_q_MASK 0x80
+#define IA_DEBUG_REG1__dma_req_path_q__SHIFT 0x7
+#define IA_DEBUG_REG1__discard_1st_chunk_MASK 0x100
+#define IA_DEBUG_REG1__discard_1st_chunk__SHIFT 0x8
+#define IA_DEBUG_REG1__discard_2nd_chunk_MASK 0x200
+#define IA_DEBUG_REG1__discard_2nd_chunk__SHIFT 0x9
+#define IA_DEBUG_REG1__second_tc_ret_data_q_MASK 0x400
+#define IA_DEBUG_REG1__second_tc_ret_data_q__SHIFT 0xa
+#define IA_DEBUG_REG1__dma_tc_ret_sel_q_MASK 0x800
+#define IA_DEBUG_REG1__dma_tc_ret_sel_q__SHIFT 0xb
+#define IA_DEBUG_REG1__last_rdreq_in_dma_op_MASK 0x1000
+#define IA_DEBUG_REG1__last_rdreq_in_dma_op__SHIFT 0xc
+#define IA_DEBUG_REG1__dma_mask_fifo_empty_MASK 0x2000
+#define IA_DEBUG_REG1__dma_mask_fifo_empty__SHIFT 0xd
+#define IA_DEBUG_REG1__dma_data_fifo_empty_q_MASK 0x4000
+#define IA_DEBUG_REG1__dma_data_fifo_empty_q__SHIFT 0xe
+#define IA_DEBUG_REG1__dma_data_fifo_full_MASK 0x8000
+#define IA_DEBUG_REG1__dma_data_fifo_full__SHIFT 0xf
+#define IA_DEBUG_REG1__dma_req_fifo_empty_MASK 0x10000
+#define IA_DEBUG_REG1__dma_req_fifo_empty__SHIFT 0x10
+#define IA_DEBUG_REG1__dma_req_fifo_full_MASK 0x20000
+#define IA_DEBUG_REG1__dma_req_fifo_full__SHIFT 0x11
+#define IA_DEBUG_REG1__stage2_dr_MASK 0x40000
+#define IA_DEBUG_REG1__stage2_dr__SHIFT 0x12
+#define IA_DEBUG_REG1__stage2_rtr_MASK 0x80000
+#define IA_DEBUG_REG1__stage2_rtr__SHIFT 0x13
+#define IA_DEBUG_REG1__stage3_dr_MASK 0x100000
+#define IA_DEBUG_REG1__stage3_dr__SHIFT 0x14
+#define IA_DEBUG_REG1__stage3_rtr_MASK 0x200000
+#define IA_DEBUG_REG1__stage3_rtr__SHIFT 0x15
+#define IA_DEBUG_REG1__stage4_dr_MASK 0x400000
+#define IA_DEBUG_REG1__stage4_dr__SHIFT 0x16
+#define IA_DEBUG_REG1__stage4_rtr_MASK 0x800000
+#define IA_DEBUG_REG1__stage4_rtr__SHIFT 0x17
+#define IA_DEBUG_REG1__dma_skid_fifo_empty_MASK 0x1000000
+#define IA_DEBUG_REG1__dma_skid_fifo_empty__SHIFT 0x18
+#define IA_DEBUG_REG1__dma_skid_fifo_full_MASK 0x2000000
+#define IA_DEBUG_REG1__dma_skid_fifo_full__SHIFT 0x19
+#define IA_DEBUG_REG1__dma_grp_valid_MASK 0x4000000
+#define IA_DEBUG_REG1__dma_grp_valid__SHIFT 0x1a
+#define IA_DEBUG_REG1__grp_dma_read_MASK 0x8000000
+#define IA_DEBUG_REG1__grp_dma_read__SHIFT 0x1b
+#define IA_DEBUG_REG1__current_data_valid_MASK 0x10000000
+#define IA_DEBUG_REG1__current_data_valid__SHIFT 0x1c
+#define IA_DEBUG_REG1__out_of_range_r2_q_MASK 0x20000000
+#define IA_DEBUG_REG1__out_of_range_r2_q__SHIFT 0x1d
+#define IA_DEBUG_REG1__dma_mask_fifo_we_MASK 0x40000000
+#define IA_DEBUG_REG1__dma_mask_fifo_we__SHIFT 0x1e
+#define IA_DEBUG_REG1__dma_ret_data_we_q_MASK 0x80000000
+#define IA_DEBUG_REG1__dma_ret_data_we_q__SHIFT 0x1f
+#define IA_DEBUG_REG2__hp_dma_input_fifo_empty_MASK 0x1
+#define IA_DEBUG_REG2__hp_dma_input_fifo_empty__SHIFT 0x0
+#define IA_DEBUG_REG2__hp_dma_input_fifo_full_MASK 0x2
+#define IA_DEBUG_REG2__hp_dma_input_fifo_full__SHIFT 0x1
+#define IA_DEBUG_REG2__hp_start_new_packet_MASK 0x4
+#define IA_DEBUG_REG2__hp_start_new_packet__SHIFT 0x2
+#define IA_DEBUG_REG2__hp_dma_rdreq_dr_q_MASK 0x8
+#define IA_DEBUG_REG2__hp_dma_rdreq_dr_q__SHIFT 0x3
+#define IA_DEBUG_REG2__hp_dma_zero_indices_q_MASK 0x10
+#define IA_DEBUG_REG2__hp_dma_zero_indices_q__SHIFT 0x4
+#define IA_DEBUG_REG2__hp_dma_buf_type_q_MASK 0x60
+#define IA_DEBUG_REG2__hp_dma_buf_type_q__SHIFT 0x5
+#define IA_DEBUG_REG2__hp_dma_req_path_q_MASK 0x80
+#define IA_DEBUG_REG2__hp_dma_req_path_q__SHIFT 0x7
+#define IA_DEBUG_REG2__hp_discard_1st_chunk_MASK 0x100
+#define IA_DEBUG_REG2__hp_discard_1st_chunk__SHIFT 0x8
+#define IA_DEBUG_REG2__hp_discard_2nd_chunk_MASK 0x200
+#define IA_DEBUG_REG2__hp_discard_2nd_chunk__SHIFT 0x9
+#define IA_DEBUG_REG2__hp_second_tc_ret_data_q_MASK 0x400
+#define IA_DEBUG_REG2__hp_second_tc_ret_data_q__SHIFT 0xa
+#define IA_DEBUG_REG2__hp_dma_tc_ret_sel_q_MASK 0x800
+#define IA_DEBUG_REG2__hp_dma_tc_ret_sel_q__SHIFT 0xb
+#define IA_DEBUG_REG2__hp_last_rdreq_in_dma_op_MASK 0x1000
+#define IA_DEBUG_REG2__hp_last_rdreq_in_dma_op__SHIFT 0xc
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_empty_MASK 0x2000
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_empty__SHIFT 0xd
+#define IA_DEBUG_REG2__hp_dma_data_fifo_empty_q_MASK 0x4000
+#define IA_DEBUG_REG2__hp_dma_data_fifo_empty_q__SHIFT 0xe
+#define IA_DEBUG_REG2__hp_dma_data_fifo_full_MASK 0x8000
+#define IA_DEBUG_REG2__hp_dma_data_fifo_full__SHIFT 0xf
+#define IA_DEBUG_REG2__hp_dma_req_fifo_empty_MASK 0x10000
+#define IA_DEBUG_REG2__hp_dma_req_fifo_empty__SHIFT 0x10
+#define IA_DEBUG_REG2__hp_dma_req_fifo_full_MASK 0x20000
+#define IA_DEBUG_REG2__hp_dma_req_fifo_full__SHIFT 0x11
+#define IA_DEBUG_REG2__hp_stage2_dr_MASK 0x40000
+#define IA_DEBUG_REG2__hp_stage2_dr__SHIFT 0x12
+#define IA_DEBUG_REG2__hp_stage2_rtr_MASK 0x80000
+#define IA_DEBUG_REG2__hp_stage2_rtr__SHIFT 0x13
+#define IA_DEBUG_REG2__hp_stage3_dr_MASK 0x100000
+#define IA_DEBUG_REG2__hp_stage3_dr__SHIFT 0x14
+#define IA_DEBUG_REG2__hp_stage3_rtr_MASK 0x200000
+#define IA_DEBUG_REG2__hp_stage3_rtr__SHIFT 0x15
+#define IA_DEBUG_REG2__hp_stage4_dr_MASK 0x400000
+#define IA_DEBUG_REG2__hp_stage4_dr__SHIFT 0x16
+#define IA_DEBUG_REG2__hp_stage4_rtr_MASK 0x800000
+#define IA_DEBUG_REG2__hp_stage4_rtr__SHIFT 0x17
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_empty_MASK 0x1000000
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_empty__SHIFT 0x18
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_full_MASK 0x2000000
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_full__SHIFT 0x19
+#define IA_DEBUG_REG2__hp_dma_grp_valid_MASK 0x4000000
+#define IA_DEBUG_REG2__hp_dma_grp_valid__SHIFT 0x1a
+#define IA_DEBUG_REG2__hp_grp_dma_read_MASK 0x8000000
+#define IA_DEBUG_REG2__hp_grp_dma_read__SHIFT 0x1b
+#define IA_DEBUG_REG2__hp_current_data_valid_MASK 0x10000000
+#define IA_DEBUG_REG2__hp_current_data_valid__SHIFT 0x1c
+#define IA_DEBUG_REG2__hp_out_of_range_r2_q_MASK 0x20000000
+#define IA_DEBUG_REG2__hp_out_of_range_r2_q__SHIFT 0x1d
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_we_MASK 0x40000000
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_we__SHIFT 0x1e
+#define IA_DEBUG_REG2__hp_dma_ret_data_we_q_MASK 0x80000000
+#define IA_DEBUG_REG2__hp_dma_ret_data_we_q__SHIFT 0x1f
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_valid_MASK 0x1
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_valid__SHIFT 0x0
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_read_MASK 0x2
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_read__SHIFT 0x1
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_null_out_MASK 0x4
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_null_out__SHIFT 0x2
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_eop_out_MASK 0x8
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_eop_out__SHIFT 0x3
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_use_tc_out_MASK 0x10
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_use_tc_out__SHIFT 0x4
+#define IA_DEBUG_REG3__grp_dma_draw_is_pipe0_MASK 0x20
+#define IA_DEBUG_REG3__grp_dma_draw_is_pipe0__SHIFT 0x5
+#define IA_DEBUG_REG3__must_service_pipe0_req_MASK 0x40
+#define IA_DEBUG_REG3__must_service_pipe0_req__SHIFT 0x6
+#define IA_DEBUG_REG3__send_pipe1_req_MASK 0x80
+#define IA_DEBUG_REG3__send_pipe1_req__SHIFT 0x7
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_valid_MASK 0x100
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_valid__SHIFT 0x8
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_read_MASK 0x200
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_read__SHIFT 0x9
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_null_out_MASK 0x400
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_null_out__SHIFT 0xa
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_eop_out_MASK 0x800
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_eop_out__SHIFT 0xb
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_use_tc_out_MASK 0x1000
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_use_tc_out__SHIFT 0xc
+#define IA_DEBUG_REG3__ia_mc_rdreq_rtr_q_MASK 0x2000
+#define IA_DEBUG_REG3__ia_mc_rdreq_rtr_q__SHIFT 0xd
+#define IA_DEBUG_REG3__mc_out_rtr_MASK 0x4000
+#define IA_DEBUG_REG3__mc_out_rtr__SHIFT 0xe
+#define IA_DEBUG_REG3__dma_rdreq_send_out_MASK 0x8000
+#define IA_DEBUG_REG3__dma_rdreq_send_out__SHIFT 0xf
+#define IA_DEBUG_REG3__pipe0_dr_MASK 0x10000
+#define IA_DEBUG_REG3__pipe0_dr__SHIFT 0x10
+#define IA_DEBUG_REG3__pipe0_rtr_MASK 0x20000
+#define IA_DEBUG_REG3__pipe0_rtr__SHIFT 0x11
+#define IA_DEBUG_REG3__ia_tc_rdreq_rtr_q_MASK 0x40000
+#define IA_DEBUG_REG3__ia_tc_rdreq_rtr_q__SHIFT 0x12
+#define IA_DEBUG_REG3__tc_out_rtr_MASK 0x80000
+#define IA_DEBUG_REG3__tc_out_rtr__SHIFT 0x13
+#define IA_DEBUG_REG3__pair0_valid_p1_MASK 0x100000
+#define IA_DEBUG_REG3__pair0_valid_p1__SHIFT 0x14
+#define IA_DEBUG_REG3__pair1_valid_p1_MASK 0x200000
+#define IA_DEBUG_REG3__pair1_valid_p1__SHIFT 0x15
+#define IA_DEBUG_REG3__pair2_valid_p1_MASK 0x400000
+#define IA_DEBUG_REG3__pair2_valid_p1__SHIFT 0x16
+#define IA_DEBUG_REG3__pair3_valid_p1_MASK 0x800000
+#define IA_DEBUG_REG3__pair3_valid_p1__SHIFT 0x17
+#define IA_DEBUG_REG3__tc_req_count_q_MASK 0x3000000
+#define IA_DEBUG_REG3__tc_req_count_q__SHIFT 0x18
+#define IA_DEBUG_REG3__discard_1st_chunk_MASK 0x4000000
+#define IA_DEBUG_REG3__discard_1st_chunk__SHIFT 0x1a
+#define IA_DEBUG_REG3__discard_2nd_chunk_MASK 0x8000000
+#define IA_DEBUG_REG3__discard_2nd_chunk__SHIFT 0x1b
+#define IA_DEBUG_REG3__last_tc_req_p1_MASK 0x10000000
+#define IA_DEBUG_REG3__last_tc_req_p1__SHIFT 0x1c
+#define IA_DEBUG_REG3__IA_TC_rdreq_send_out_MASK 0x20000000
+#define IA_DEBUG_REG3__IA_TC_rdreq_send_out__SHIFT 0x1d
+#define IA_DEBUG_REG3__TC_IA_rdret_valid_in_MASK 0x40000000
+#define IA_DEBUG_REG3__TC_IA_rdret_valid_in__SHIFT 0x1e
+#define IA_DEBUG_REG3__TAP_IA_rdret_vld_in_MASK 0x80000000
+#define IA_DEBUG_REG3__TAP_IA_rdret_vld_in__SHIFT 0x1f
+#define IA_DEBUG_REG4__pipe0_dr_MASK 0x1
+#define IA_DEBUG_REG4__pipe0_dr__SHIFT 0x0
+#define IA_DEBUG_REG4__pipe1_dr_MASK 0x2
+#define IA_DEBUG_REG4__pipe1_dr__SHIFT 0x1
+#define IA_DEBUG_REG4__pipe2_dr_MASK 0x4
+#define IA_DEBUG_REG4__pipe2_dr__SHIFT 0x2
+#define IA_DEBUG_REG4__pipe3_dr_MASK 0x8
+#define IA_DEBUG_REG4__pipe3_dr__SHIFT 0x3
+#define IA_DEBUG_REG4__pipe4_dr_MASK 0x10
+#define IA_DEBUG_REG4__pipe4_dr__SHIFT 0x4
+#define IA_DEBUG_REG4__pipe5_dr_MASK 0x20
+#define IA_DEBUG_REG4__pipe5_dr__SHIFT 0x5
+#define IA_DEBUG_REG4__grp_se0_fifo_empty_MASK 0x40
+#define IA_DEBUG_REG4__grp_se0_fifo_empty__SHIFT 0x6
+#define IA_DEBUG_REG4__grp_se0_fifo_full_MASK 0x80
+#define IA_DEBUG_REG4__grp_se0_fifo_full__SHIFT 0x7
+#define IA_DEBUG_REG4__pipe0_rtr_MASK 0x100
+#define IA_DEBUG_REG4__pipe0_rtr__SHIFT 0x8
+#define IA_DEBUG_REG4__pipe1_rtr_MASK 0x200
+#define IA_DEBUG_REG4__pipe1_rtr__SHIFT 0x9
+#define IA_DEBUG_REG4__pipe2_rtr_MASK 0x400
+#define IA_DEBUG_REG4__pipe2_rtr__SHIFT 0xa
+#define IA_DEBUG_REG4__pipe3_rtr_MASK 0x800
+#define IA_DEBUG_REG4__pipe3_rtr__SHIFT 0xb
+#define IA_DEBUG_REG4__pipe4_rtr_MASK 0x1000
+#define IA_DEBUG_REG4__pipe4_rtr__SHIFT 0xc
+#define IA_DEBUG_REG4__pipe5_rtr_MASK 0x2000
+#define IA_DEBUG_REG4__pipe5_rtr__SHIFT 0xd
+#define IA_DEBUG_REG4__ia_vgt_prim_rtr_q_MASK 0x4000
+#define IA_DEBUG_REG4__ia_vgt_prim_rtr_q__SHIFT 0xe
+#define IA_DEBUG_REG4__ia_se1vgt_prim_rtr_q_MASK 0x8000
+#define IA_DEBUG_REG4__ia_se1vgt_prim_rtr_q__SHIFT 0xf
+#define IA_DEBUG_REG4__di_major_mode_p1_q_MASK 0x10000
+#define IA_DEBUG_REG4__di_major_mode_p1_q__SHIFT 0x10
+#define IA_DEBUG_REG4__gs_mode_p1_q_MASK 0xe0000
+#define IA_DEBUG_REG4__gs_mode_p1_q__SHIFT 0x11
+#define IA_DEBUG_REG4__di_event_flag_p1_q_MASK 0x100000
+#define IA_DEBUG_REG4__di_event_flag_p1_q__SHIFT 0x14
+#define IA_DEBUG_REG4__di_state_sel_p1_q_MASK 0xe00000
+#define IA_DEBUG_REG4__di_state_sel_p1_q__SHIFT 0x15
+#define IA_DEBUG_REG4__draw_opaq_en_p1_q_MASK 0x1000000
+#define IA_DEBUG_REG4__draw_opaq_en_p1_q__SHIFT 0x18
+#define IA_DEBUG_REG4__draw_opaq_active_q_MASK 0x2000000
+#define IA_DEBUG_REG4__draw_opaq_active_q__SHIFT 0x19
+#define IA_DEBUG_REG4__di_source_select_p1_q_MASK 0xc000000
+#define IA_DEBUG_REG4__di_source_select_p1_q__SHIFT 0x1a
+#define IA_DEBUG_REG4__ready_to_read_di_MASK 0x10000000
+#define IA_DEBUG_REG4__ready_to_read_di__SHIFT 0x1c
+#define IA_DEBUG_REG4__di_first_group_of_draw_q_MASK 0x20000000
+#define IA_DEBUG_REG4__di_first_group_of_draw_q__SHIFT 0x1d
+#define IA_DEBUG_REG4__last_shift_of_draw_MASK 0x40000000
+#define IA_DEBUG_REG4__last_shift_of_draw__SHIFT 0x1e
+#define IA_DEBUG_REG4__current_shift_is_vect1_q_MASK 0x80000000
+#define IA_DEBUG_REG4__current_shift_is_vect1_q__SHIFT 0x1f
+#define IA_DEBUG_REG5__di_index_counter_q_15_0_MASK 0xffff
+#define IA_DEBUG_REG5__di_index_counter_q_15_0__SHIFT 0x0
+#define IA_DEBUG_REG5__instanceid_13_0_MASK 0x3fff0000
+#define IA_DEBUG_REG5__instanceid_13_0__SHIFT 0x10
+#define IA_DEBUG_REG5__draw_input_fifo_full_MASK 0x40000000
+#define IA_DEBUG_REG5__draw_input_fifo_full__SHIFT 0x1e
+#define IA_DEBUG_REG5__draw_input_fifo_empty_MASK 0x80000000
+#define IA_DEBUG_REG5__draw_input_fifo_empty__SHIFT 0x1f
+#define IA_DEBUG_REG6__current_shift_q_MASK 0xf
+#define IA_DEBUG_REG6__current_shift_q__SHIFT 0x0
+#define IA_DEBUG_REG6__current_stride_pre_MASK 0xf0
+#define IA_DEBUG_REG6__current_stride_pre__SHIFT 0x4
+#define IA_DEBUG_REG6__current_stride_q_MASK 0x1f00
+#define IA_DEBUG_REG6__current_stride_q__SHIFT 0x8
+#define IA_DEBUG_REG6__first_group_partial_MASK 0x2000
+#define IA_DEBUG_REG6__first_group_partial__SHIFT 0xd
+#define IA_DEBUG_REG6__second_group_partial_MASK 0x4000
+#define IA_DEBUG_REG6__second_group_partial__SHIFT 0xe
+#define IA_DEBUG_REG6__curr_prim_partial_MASK 0x8000
+#define IA_DEBUG_REG6__curr_prim_partial__SHIFT 0xf
+#define IA_DEBUG_REG6__next_stride_q_MASK 0x1f0000
+#define IA_DEBUG_REG6__next_stride_q__SHIFT 0x10
+#define IA_DEBUG_REG6__next_group_partial_MASK 0x200000
+#define IA_DEBUG_REG6__next_group_partial__SHIFT 0x15
+#define IA_DEBUG_REG6__after_group_partial_MASK 0x400000
+#define IA_DEBUG_REG6__after_group_partial__SHIFT 0x16
+#define IA_DEBUG_REG6__extract_group_MASK 0x800000
+#define IA_DEBUG_REG6__extract_group__SHIFT 0x17
+#define IA_DEBUG_REG6__grp_shift_debug_data_MASK 0xff000000
+#define IA_DEBUG_REG6__grp_shift_debug_data__SHIFT 0x18
+#define IA_DEBUG_REG7__reset_indx_state_q_MASK 0xf
+#define IA_DEBUG_REG7__reset_indx_state_q__SHIFT 0x0
+#define IA_DEBUG_REG7__shift_vect_valid_p2_q_MASK 0xf0
+#define IA_DEBUG_REG7__shift_vect_valid_p2_q__SHIFT 0x4
+#define IA_DEBUG_REG7__shift_vect1_valid_p2_q_MASK 0xf00
+#define IA_DEBUG_REG7__shift_vect1_valid_p2_q__SHIFT 0x8
+#define IA_DEBUG_REG7__shift_vect0_reset_match_p2_q_MASK 0xf000
+#define IA_DEBUG_REG7__shift_vect0_reset_match_p2_q__SHIFT 0xc
+#define IA_DEBUG_REG7__shift_vect1_reset_match_p2_q_MASK 0xf0000
+#define IA_DEBUG_REG7__shift_vect1_reset_match_p2_q__SHIFT 0x10
+#define IA_DEBUG_REG7__num_indx_in_group_p2_q_MASK 0x700000
+#define IA_DEBUG_REG7__num_indx_in_group_p2_q__SHIFT 0x14
+#define IA_DEBUG_REG7__last_group_of_draw_p2_q_MASK 0x800000
+#define IA_DEBUG_REG7__last_group_of_draw_p2_q__SHIFT 0x17
+#define IA_DEBUG_REG7__shift_event_flag_p2_q_MASK 0x1000000
+#define IA_DEBUG_REG7__shift_event_flag_p2_q__SHIFT 0x18
+#define IA_DEBUG_REG7__indx_shift_is_one_p2_q_MASK 0x2000000
+#define IA_DEBUG_REG7__indx_shift_is_one_p2_q__SHIFT 0x19
+#define IA_DEBUG_REG7__indx_shift_is_two_p2_q_MASK 0x4000000
+#define IA_DEBUG_REG7__indx_shift_is_two_p2_q__SHIFT 0x1a
+#define IA_DEBUG_REG7__indx_stride_is_four_p2_q_MASK 0x8000000
+#define IA_DEBUG_REG7__indx_stride_is_four_p2_q__SHIFT 0x1b
+#define IA_DEBUG_REG7__shift_prim1_reset_p3_q_MASK 0x10000000
+#define IA_DEBUG_REG7__shift_prim1_reset_p3_q__SHIFT 0x1c
+#define IA_DEBUG_REG7__shift_prim1_partial_p3_q_MASK 0x20000000
+#define IA_DEBUG_REG7__shift_prim1_partial_p3_q__SHIFT 0x1d
+#define IA_DEBUG_REG7__shift_prim0_reset_p3_q_MASK 0x40000000
+#define IA_DEBUG_REG7__shift_prim0_reset_p3_q__SHIFT 0x1e
+#define IA_DEBUG_REG7__shift_prim0_partial_p3_q_MASK 0x80000000
+#define IA_DEBUG_REG7__shift_prim0_partial_p3_q__SHIFT 0x1f
+#define IA_DEBUG_REG8__di_prim_type_p1_q_MASK 0x1f
+#define IA_DEBUG_REG8__di_prim_type_p1_q__SHIFT 0x0
+#define IA_DEBUG_REG8__two_cycle_xfer_p1_q_MASK 0x20
+#define IA_DEBUG_REG8__two_cycle_xfer_p1_q__SHIFT 0x5
+#define IA_DEBUG_REG8__two_prim_input_p1_q_MASK 0x40
+#define IA_DEBUG_REG8__two_prim_input_p1_q__SHIFT 0x6
+#define IA_DEBUG_REG8__shift_vect_end_of_packet_p5_q_MASK 0x80
+#define IA_DEBUG_REG8__shift_vect_end_of_packet_p5_q__SHIFT 0x7
+#define IA_DEBUG_REG8__last_group_of_inst_p5_q_MASK 0x100
+#define IA_DEBUG_REG8__last_group_of_inst_p5_q__SHIFT 0x8
+#define IA_DEBUG_REG8__shift_prim1_null_flag_p5_q_MASK 0x200
+#define IA_DEBUG_REG8__shift_prim1_null_flag_p5_q__SHIFT 0x9
+#define IA_DEBUG_REG8__shift_prim0_null_flag_p5_q_MASK 0x400
+#define IA_DEBUG_REG8__shift_prim0_null_flag_p5_q__SHIFT 0xa
+#define IA_DEBUG_REG8__grp_continued_MASK 0x800
+#define IA_DEBUG_REG8__grp_continued__SHIFT 0xb
+#define IA_DEBUG_REG8__grp_state_sel_MASK 0x7000
+#define IA_DEBUG_REG8__grp_state_sel__SHIFT 0xc
+#define IA_DEBUG_REG8__grp_sub_prim_type_MASK 0x1f8000
+#define IA_DEBUG_REG8__grp_sub_prim_type__SHIFT 0xf
+#define IA_DEBUG_REG8__grp_output_path_MASK 0xe00000
+#define IA_DEBUG_REG8__grp_output_path__SHIFT 0x15
+#define IA_DEBUG_REG8__grp_null_primitive_MASK 0x1000000
+#define IA_DEBUG_REG8__grp_null_primitive__SHIFT 0x18
+#define IA_DEBUG_REG8__grp_eop_MASK 0x2000000
+#define IA_DEBUG_REG8__grp_eop__SHIFT 0x19
+#define IA_DEBUG_REG8__grp_eopg_MASK 0x4000000
+#define IA_DEBUG_REG8__grp_eopg__SHIFT 0x1a
+#define IA_DEBUG_REG8__grp_event_flag_MASK 0x8000000
+#define IA_DEBUG_REG8__grp_event_flag__SHIFT 0x1b
+#define IA_DEBUG_REG8__grp_components_valid_MASK 0xf0000000
+#define IA_DEBUG_REG8__grp_components_valid__SHIFT 0x1c
+#define IA_DEBUG_REG9__send_to_se1_p6_MASK 0x1
+#define IA_DEBUG_REG9__send_to_se1_p6__SHIFT 0x0
+#define IA_DEBUG_REG9__gfx_se_switch_p6_MASK 0x2
+#define IA_DEBUG_REG9__gfx_se_switch_p6__SHIFT 0x1
+#define IA_DEBUG_REG9__null_eoi_xfer_prim1_p6_MASK 0x4
+#define IA_DEBUG_REG9__null_eoi_xfer_prim1_p6__SHIFT 0x2
+#define IA_DEBUG_REG9__null_eoi_xfer_prim0_p6_MASK 0x8
+#define IA_DEBUG_REG9__null_eoi_xfer_prim0_p6__SHIFT 0x3
+#define IA_DEBUG_REG9__prim1_eoi_p6_MASK 0x10
+#define IA_DEBUG_REG9__prim1_eoi_p6__SHIFT 0x4
+#define IA_DEBUG_REG9__prim0_eoi_p6_MASK 0x20
+#define IA_DEBUG_REG9__prim0_eoi_p6__SHIFT 0x5
+#define IA_DEBUG_REG9__prim1_valid_eopg_p6_MASK 0x40
+#define IA_DEBUG_REG9__prim1_valid_eopg_p6__SHIFT 0x6
+#define IA_DEBUG_REG9__prim0_valid_eopg_p6_MASK 0x80
+#define IA_DEBUG_REG9__prim0_valid_eopg_p6__SHIFT 0x7
+#define IA_DEBUG_REG9__prim1_to_other_se_p6_MASK 0x100
+#define IA_DEBUG_REG9__prim1_to_other_se_p6__SHIFT 0x8
+#define IA_DEBUG_REG9__eopg_on_last_prim_p6_MASK 0x200
+#define IA_DEBUG_REG9__eopg_on_last_prim_p6__SHIFT 0x9
+#define IA_DEBUG_REG9__eopg_between_prims_p6_MASK 0x400
+#define IA_DEBUG_REG9__eopg_between_prims_p6__SHIFT 0xa
+#define IA_DEBUG_REG9__prim_count_eq_group_size_p6_MASK 0x800
+#define IA_DEBUG_REG9__prim_count_eq_group_size_p6__SHIFT 0xb
+#define IA_DEBUG_REG9__prim_count_gt_group_size_p6_MASK 0x1000
+#define IA_DEBUG_REG9__prim_count_gt_group_size_p6__SHIFT 0xc
+#define IA_DEBUG_REG9__two_prim_output_p5_q_MASK 0x2000
+#define IA_DEBUG_REG9__two_prim_output_p5_q__SHIFT 0xd
+#define IA_DEBUG_REG9__SPARE0_MASK 0x4000
+#define IA_DEBUG_REG9__SPARE0__SHIFT 0xe
+#define IA_DEBUG_REG9__SPARE1_MASK 0x8000
+#define IA_DEBUG_REG9__SPARE1__SHIFT 0xf
+#define IA_DEBUG_REG9__shift_vect_end_of_packet_p5_q_MASK 0x10000
+#define IA_DEBUG_REG9__shift_vect_end_of_packet_p5_q__SHIFT 0x10
+#define IA_DEBUG_REG9__prim1_xfer_p6_MASK 0x20000
+#define IA_DEBUG_REG9__prim1_xfer_p6__SHIFT 0x11
+#define IA_DEBUG_REG9__grp_se1_fifo_empty_MASK 0x40000
+#define IA_DEBUG_REG9__grp_se1_fifo_empty__SHIFT 0x12
+#define IA_DEBUG_REG9__grp_se1_fifo_full_MASK 0x80000
+#define IA_DEBUG_REG9__grp_se1_fifo_full__SHIFT 0x13
+#define IA_DEBUG_REG9__prim_counter_q_MASK 0xfff00000
+#define IA_DEBUG_REG9__prim_counter_q__SHIFT 0x14
+#define VGT_DEBUG_REG0__vgt_busy_extended_MASK 0x1
+#define VGT_DEBUG_REG0__vgt_busy_extended__SHIFT 0x0
+#define VGT_DEBUG_REG0__SPARE9_MASK 0x2
+#define VGT_DEBUG_REG0__SPARE9__SHIFT 0x1
+#define VGT_DEBUG_REG0__vgt_busy_MASK 0x4
+#define VGT_DEBUG_REG0__vgt_busy__SHIFT 0x2
+#define VGT_DEBUG_REG0__SPARE8_MASK 0x8
+#define VGT_DEBUG_REG0__SPARE8__SHIFT 0x3
+#define VGT_DEBUG_REG0__SPARE7_MASK 0x10
+#define VGT_DEBUG_REG0__SPARE7__SHIFT 0x4
+#define VGT_DEBUG_REG0__SPARE6_MASK 0x20
+#define VGT_DEBUG_REG0__SPARE6__SHIFT 0x5
+#define VGT_DEBUG_REG0__SPARE5_MASK 0x40
+#define VGT_DEBUG_REG0__SPARE5__SHIFT 0x6
+#define VGT_DEBUG_REG0__SPARE4_MASK 0x80
+#define VGT_DEBUG_REG0__SPARE4__SHIFT 0x7
+#define VGT_DEBUG_REG0__pi_busy_MASK 0x100
+#define VGT_DEBUG_REG0__pi_busy__SHIFT 0x8
+#define VGT_DEBUG_REG0__vr_pi_busy_MASK 0x200
+#define VGT_DEBUG_REG0__vr_pi_busy__SHIFT 0x9
+#define VGT_DEBUG_REG0__pt_pi_busy_MASK 0x400
+#define VGT_DEBUG_REG0__pt_pi_busy__SHIFT 0xa
+#define VGT_DEBUG_REG0__te_pi_busy_MASK 0x800
+#define VGT_DEBUG_REG0__te_pi_busy__SHIFT 0xb
+#define VGT_DEBUG_REG0__gs_busy_MASK 0x1000
+#define VGT_DEBUG_REG0__gs_busy__SHIFT 0xc
+#define VGT_DEBUG_REG0__rcm_busy_MASK 0x2000
+#define VGT_DEBUG_REG0__rcm_busy__SHIFT 0xd
+#define VGT_DEBUG_REG0__tm_busy_MASK 0x4000
+#define VGT_DEBUG_REG0__tm_busy__SHIFT 0xe
+#define VGT_DEBUG_REG0__cm_busy_MASK 0x8000
+#define VGT_DEBUG_REG0__cm_busy__SHIFT 0xf
+#define VGT_DEBUG_REG0__gog_busy_MASK 0x10000
+#define VGT_DEBUG_REG0__gog_busy__SHIFT 0x10
+#define VGT_DEBUG_REG0__frmt_busy_MASK 0x20000
+#define VGT_DEBUG_REG0__frmt_busy__SHIFT 0x11
+#define VGT_DEBUG_REG0__SPARE10_MASK 0x40000
+#define VGT_DEBUG_REG0__SPARE10__SHIFT 0x12
+#define VGT_DEBUG_REG0__te11_pi_busy_MASK 0x80000
+#define VGT_DEBUG_REG0__te11_pi_busy__SHIFT 0x13
+#define VGT_DEBUG_REG0__SPARE3_MASK 0x100000
+#define VGT_DEBUG_REG0__SPARE3__SHIFT 0x14
+#define VGT_DEBUG_REG0__combined_out_busy_MASK 0x200000
+#define VGT_DEBUG_REG0__combined_out_busy__SHIFT 0x15
+#define VGT_DEBUG_REG0__spi_vs_interfaces_busy_MASK 0x400000
+#define VGT_DEBUG_REG0__spi_vs_interfaces_busy__SHIFT 0x16
+#define VGT_DEBUG_REG0__pa_interfaces_busy_MASK 0x800000
+#define VGT_DEBUG_REG0__pa_interfaces_busy__SHIFT 0x17
+#define VGT_DEBUG_REG0__reg_clk_busy_MASK 0x1000000
+#define VGT_DEBUG_REG0__reg_clk_busy__SHIFT 0x18
+#define VGT_DEBUG_REG0__SPARE2_MASK 0x2000000
+#define VGT_DEBUG_REG0__SPARE2__SHIFT 0x19
+#define VGT_DEBUG_REG0__core_clk_busy_MASK 0x4000000
+#define VGT_DEBUG_REG0__core_clk_busy__SHIFT 0x1a
+#define VGT_DEBUG_REG0__gs_clk_busy_MASK 0x8000000
+#define VGT_DEBUG_REG0__gs_clk_busy__SHIFT 0x1b
+#define VGT_DEBUG_REG0__SPARE1_MASK 0x10000000
+#define VGT_DEBUG_REG0__SPARE1__SHIFT 0x1c
+#define VGT_DEBUG_REG0__sclk_core_vld_MASK 0x20000000
+#define VGT_DEBUG_REG0__sclk_core_vld__SHIFT 0x1d
+#define VGT_DEBUG_REG0__sclk_gs_vld_MASK 0x40000000
+#define VGT_DEBUG_REG0__sclk_gs_vld__SHIFT 0x1e
+#define VGT_DEBUG_REG0__SPARE0_MASK 0x80000000
+#define VGT_DEBUG_REG0__SPARE0__SHIFT 0x1f
+#define VGT_DEBUG_REG1__SPARE9_MASK 0x1
+#define VGT_DEBUG_REG1__SPARE9__SHIFT 0x0
+#define VGT_DEBUG_REG1__SPARE8_MASK 0x2
+#define VGT_DEBUG_REG1__SPARE8__SHIFT 0x1
+#define VGT_DEBUG_REG1__SPARE7_MASK 0x4
+#define VGT_DEBUG_REG1__SPARE7__SHIFT 0x2
+#define VGT_DEBUG_REG1__SPARE6_MASK 0x8
+#define VGT_DEBUG_REG1__SPARE6__SHIFT 0x3
+#define VGT_DEBUG_REG1__SPARE5_MASK 0x10
+#define VGT_DEBUG_REG1__SPARE5__SHIFT 0x4
+#define VGT_DEBUG_REG1__SPARE4_MASK 0x20
+#define VGT_DEBUG_REG1__SPARE4__SHIFT 0x5
+#define VGT_DEBUG_REG1__SPARE3_MASK 0x40
+#define VGT_DEBUG_REG1__SPARE3__SHIFT 0x6
+#define VGT_DEBUG_REG1__SPARE2_MASK 0x80
+#define VGT_DEBUG_REG1__SPARE2__SHIFT 0x7
+#define VGT_DEBUG_REG1__SPARE1_MASK 0x100
+#define VGT_DEBUG_REG1__SPARE1__SHIFT 0x8
+#define VGT_DEBUG_REG1__SPARE0_MASK 0x200
+#define VGT_DEBUG_REG1__SPARE0__SHIFT 0x9
+#define VGT_DEBUG_REG1__pi_vr_valid_MASK 0x400
+#define VGT_DEBUG_REG1__pi_vr_valid__SHIFT 0xa
+#define VGT_DEBUG_REG1__vr_pi_read_MASK 0x800
+#define VGT_DEBUG_REG1__vr_pi_read__SHIFT 0xb
+#define VGT_DEBUG_REG1__pi_pt_valid_MASK 0x1000
+#define VGT_DEBUG_REG1__pi_pt_valid__SHIFT 0xc
+#define VGT_DEBUG_REG1__pt_pi_read_MASK 0x2000
+#define VGT_DEBUG_REG1__pt_pi_read__SHIFT 0xd
+#define VGT_DEBUG_REG1__pi_te_valid_MASK 0x4000
+#define VGT_DEBUG_REG1__pi_te_valid__SHIFT 0xe
+#define VGT_DEBUG_REG1__te_grp_read_MASK 0x8000
+#define VGT_DEBUG_REG1__te_grp_read__SHIFT 0xf
+#define VGT_DEBUG_REG1__vr_out_indx_valid_MASK 0x10000
+#define VGT_DEBUG_REG1__vr_out_indx_valid__SHIFT 0x10
+#define VGT_DEBUG_REG1__SPARE12_MASK 0x20000
+#define VGT_DEBUG_REG1__SPARE12__SHIFT 0x11
+#define VGT_DEBUG_REG1__vr_out_prim_valid_MASK 0x40000
+#define VGT_DEBUG_REG1__vr_out_prim_valid__SHIFT 0x12
+#define VGT_DEBUG_REG1__SPARE11_MASK 0x80000
+#define VGT_DEBUG_REG1__SPARE11__SHIFT 0x13
+#define VGT_DEBUG_REG1__pt_out_indx_valid_MASK 0x100000
+#define VGT_DEBUG_REG1__pt_out_indx_valid__SHIFT 0x14
+#define VGT_DEBUG_REG1__SPARE10_MASK 0x200000
+#define VGT_DEBUG_REG1__SPARE10__SHIFT 0x15
+#define VGT_DEBUG_REG1__pt_out_prim_valid_MASK 0x400000
+#define VGT_DEBUG_REG1__pt_out_prim_valid__SHIFT 0x16
+#define VGT_DEBUG_REG1__SPARE23_MASK 0x800000
+#define VGT_DEBUG_REG1__SPARE23__SHIFT 0x17
+#define VGT_DEBUG_REG1__te_out_data_valid_MASK 0x1000000
+#define VGT_DEBUG_REG1__te_out_data_valid__SHIFT 0x18
+#define VGT_DEBUG_REG1__SPARE25_MASK 0x2000000
+#define VGT_DEBUG_REG1__SPARE25__SHIFT 0x19
+#define VGT_DEBUG_REG1__pi_gs_valid_MASK 0x4000000
+#define VGT_DEBUG_REG1__pi_gs_valid__SHIFT 0x1a
+#define VGT_DEBUG_REG1__gs_pi_read_MASK 0x8000000
+#define VGT_DEBUG_REG1__gs_pi_read__SHIFT 0x1b
+#define VGT_DEBUG_REG1__gog_out_indx_valid_MASK 0x10000000
+#define VGT_DEBUG_REG1__gog_out_indx_valid__SHIFT 0x1c
+#define VGT_DEBUG_REG1__out_indx_read_MASK 0x20000000
+#define VGT_DEBUG_REG1__out_indx_read__SHIFT 0x1d
+#define VGT_DEBUG_REG1__gog_out_prim_valid_MASK 0x40000000
+#define VGT_DEBUG_REG1__gog_out_prim_valid__SHIFT 0x1e
+#define VGT_DEBUG_REG1__out_prim_read_MASK 0x80000000
+#define VGT_DEBUG_REG1__out_prim_read__SHIFT 0x1f
+#define VGT_DEBUG_REG2__hs_grp_busy_MASK 0x1
+#define VGT_DEBUG_REG2__hs_grp_busy__SHIFT 0x0
+#define VGT_DEBUG_REG2__hs_noif_busy_MASK 0x2
+#define VGT_DEBUG_REG2__hs_noif_busy__SHIFT 0x1
+#define VGT_DEBUG_REG2__tfmmIsBusy_MASK 0x4
+#define VGT_DEBUG_REG2__tfmmIsBusy__SHIFT 0x2
+#define VGT_DEBUG_REG2__lsVertIfBusy_0_MASK 0x8
+#define VGT_DEBUG_REG2__lsVertIfBusy_0__SHIFT 0x3
+#define VGT_DEBUG_REG2__te11_hs_tess_input_rtr_MASK 0x10
+#define VGT_DEBUG_REG2__te11_hs_tess_input_rtr__SHIFT 0x4
+#define VGT_DEBUG_REG2__lsWaveIfBusy_0_MASK 0x20
+#define VGT_DEBUG_REG2__lsWaveIfBusy_0__SHIFT 0x5
+#define VGT_DEBUG_REG2__hs_te11_tess_input_rts_MASK 0x40
+#define VGT_DEBUG_REG2__hs_te11_tess_input_rts__SHIFT 0x6
+#define VGT_DEBUG_REG2__grpModBusy_MASK 0x80
+#define VGT_DEBUG_REG2__grpModBusy__SHIFT 0x7
+#define VGT_DEBUG_REG2__lsVertFifoEmpty_MASK 0x100
+#define VGT_DEBUG_REG2__lsVertFifoEmpty__SHIFT 0x8
+#define VGT_DEBUG_REG2__lsWaveFifoEmpty_MASK 0x200
+#define VGT_DEBUG_REG2__lsWaveFifoEmpty__SHIFT 0x9
+#define VGT_DEBUG_REG2__hsVertFifoEmpty_MASK 0x400
+#define VGT_DEBUG_REG2__hsVertFifoEmpty__SHIFT 0xa
+#define VGT_DEBUG_REG2__hsWaveFifoEmpty_MASK 0x800
+#define VGT_DEBUG_REG2__hsWaveFifoEmpty__SHIFT 0xb
+#define VGT_DEBUG_REG2__hsInputFifoEmpty_MASK 0x1000
+#define VGT_DEBUG_REG2__hsInputFifoEmpty__SHIFT 0xc
+#define VGT_DEBUG_REG2__hsTifFifoEmpty_MASK 0x2000
+#define VGT_DEBUG_REG2__hsTifFifoEmpty__SHIFT 0xd
+#define VGT_DEBUG_REG2__lsVertFifoFull_MASK 0x4000
+#define VGT_DEBUG_REG2__lsVertFifoFull__SHIFT 0xe
+#define VGT_DEBUG_REG2__lsWaveFifoFull_MASK 0x8000
+#define VGT_DEBUG_REG2__lsWaveFifoFull__SHIFT 0xf
+#define VGT_DEBUG_REG2__hsVertFifoFull_MASK 0x10000
+#define VGT_DEBUG_REG2__hsVertFifoFull__SHIFT 0x10
+#define VGT_DEBUG_REG2__hsWaveFifoFull_MASK 0x20000
+#define VGT_DEBUG_REG2__hsWaveFifoFull__SHIFT 0x11
+#define VGT_DEBUG_REG2__hsInputFifoFull_MASK 0x40000
+#define VGT_DEBUG_REG2__hsInputFifoFull__SHIFT 0x12
+#define VGT_DEBUG_REG2__hsTifFifoFull_MASK 0x80000
+#define VGT_DEBUG_REG2__hsTifFifoFull__SHIFT 0x13
+#define VGT_DEBUG_REG2__p0_rtr_MASK 0x100000
+#define VGT_DEBUG_REG2__p0_rtr__SHIFT 0x14
+#define VGT_DEBUG_REG2__p1_rtr_MASK 0x200000
+#define VGT_DEBUG_REG2__p1_rtr__SHIFT 0x15
+#define VGT_DEBUG_REG2__p0_dr_MASK 0x400000
+#define VGT_DEBUG_REG2__p0_dr__SHIFT 0x16
+#define VGT_DEBUG_REG2__p1_dr_MASK 0x800000
+#define VGT_DEBUG_REG2__p1_dr__SHIFT 0x17
+#define VGT_DEBUG_REG2__p0_rts_MASK 0x1000000
+#define VGT_DEBUG_REG2__p0_rts__SHIFT 0x18
+#define VGT_DEBUG_REG2__p1_rts_MASK 0x2000000
+#define VGT_DEBUG_REG2__p1_rts__SHIFT 0x19
+#define VGT_DEBUG_REG2__ls_sh_id_MASK 0x4000000
+#define VGT_DEBUG_REG2__ls_sh_id__SHIFT 0x1a
+#define VGT_DEBUG_REG2__lsFwaveFlag_MASK 0x8000000
+#define VGT_DEBUG_REG2__lsFwaveFlag__SHIFT 0x1b
+#define VGT_DEBUG_REG2__lsWaveSendFlush_MASK 0x10000000
+#define VGT_DEBUG_REG2__lsWaveSendFlush__SHIFT 0x1c
+#define VGT_DEBUG_REG2__SPARE_MASK 0xe0000000
+#define VGT_DEBUG_REG2__SPARE__SHIFT 0x1d
+#define VGT_DEBUG_REG3__lsTgRelInd_MASK 0xfff
+#define VGT_DEBUG_REG3__lsTgRelInd__SHIFT 0x0
+#define VGT_DEBUG_REG3__lsWaveRelInd_MASK 0x3f000
+#define VGT_DEBUG_REG3__lsWaveRelInd__SHIFT 0xc
+#define VGT_DEBUG_REG3__lsPatchCnt_MASK 0x3fc0000
+#define VGT_DEBUG_REG3__lsPatchCnt__SHIFT 0x12
+#define VGT_DEBUG_REG3__hsWaveRelInd_MASK 0xfc000000
+#define VGT_DEBUG_REG3__hsWaveRelInd__SHIFT 0x1a
+#define VGT_DEBUG_REG4__hsPatchCnt_MASK 0xff
+#define VGT_DEBUG_REG4__hsPatchCnt__SHIFT 0x0
+#define VGT_DEBUG_REG4__hsPrimId_15_0_MASK 0xffff00
+#define VGT_DEBUG_REG4__hsPrimId_15_0__SHIFT 0x8
+#define VGT_DEBUG_REG4__hsCpCnt_MASK 0x1f000000
+#define VGT_DEBUG_REG4__hsCpCnt__SHIFT 0x18
+#define VGT_DEBUG_REG4__hsWaveSendFlush_MASK 0x20000000
+#define VGT_DEBUG_REG4__hsWaveSendFlush__SHIFT 0x1d
+#define VGT_DEBUG_REG4__hsFwaveFlag_MASK 0x40000000
+#define VGT_DEBUG_REG4__hsFwaveFlag__SHIFT 0x1e
+#define VGT_DEBUG_REG4__SPARE_MASK 0x80000000
+#define VGT_DEBUG_REG4__SPARE__SHIFT 0x1f
+#define VGT_DEBUG_REG5__SPARE4_MASK 0x7
+#define VGT_DEBUG_REG5__SPARE4__SHIFT 0x0
+#define VGT_DEBUG_REG5__hsWaveCreditCnt_0_MASK 0xf8
+#define VGT_DEBUG_REG5__hsWaveCreditCnt_0__SHIFT 0x3
+#define VGT_DEBUG_REG5__SPARE3_MASK 0x700
+#define VGT_DEBUG_REG5__SPARE3__SHIFT 0x8
+#define VGT_DEBUG_REG5__hsVertCreditCnt_0_MASK 0xf800
+#define VGT_DEBUG_REG5__hsVertCreditCnt_0__SHIFT 0xb
+#define VGT_DEBUG_REG5__SPARE2_MASK 0x70000
+#define VGT_DEBUG_REG5__SPARE2__SHIFT 0x10
+#define VGT_DEBUG_REG5__lsWaveCreditCnt_0_MASK 0xf80000
+#define VGT_DEBUG_REG5__lsWaveCreditCnt_0__SHIFT 0x13
+#define VGT_DEBUG_REG5__SPARE1_MASK 0x7000000
+#define VGT_DEBUG_REG5__SPARE1__SHIFT 0x18
+#define VGT_DEBUG_REG5__lsVertCreditCnt_0_MASK 0xf8000000
+#define VGT_DEBUG_REG5__lsVertCreditCnt_0__SHIFT 0x1b
+#define VGT_DEBUG_REG6__debug_BASE_MASK 0xffff
+#define VGT_DEBUG_REG6__debug_BASE__SHIFT 0x0
+#define VGT_DEBUG_REG6__debug_SIZE_MASK 0xffff0000
+#define VGT_DEBUG_REG6__debug_SIZE__SHIFT 0x10
+#define VGT_DEBUG_REG7__debug_tfmmFifoEmpty_MASK 0x1
+#define VGT_DEBUG_REG7__debug_tfmmFifoEmpty__SHIFT 0x0
+#define VGT_DEBUG_REG7__debug_tfmmFifoFull_MASK 0x2
+#define VGT_DEBUG_REG7__debug_tfmmFifoFull__SHIFT 0x1
+#define VGT_DEBUG_REG7__hs_pipe0_dr_MASK 0x4
+#define VGT_DEBUG_REG7__hs_pipe0_dr__SHIFT 0x2
+#define VGT_DEBUG_REG7__hs_pipe0_rtr_MASK 0x8
+#define VGT_DEBUG_REG7__hs_pipe0_rtr__SHIFT 0x3
+#define VGT_DEBUG_REG7__hs_pipe1_rtr_MASK 0x10
+#define VGT_DEBUG_REG7__hs_pipe1_rtr__SHIFT 0x4
+#define VGT_DEBUG_REG7__SPARE_MASK 0xffe0
+#define VGT_DEBUG_REG7__SPARE__SHIFT 0x5
+#define VGT_DEBUG_REG7__TF_addr_MASK 0xffff0000
+#define VGT_DEBUG_REG7__TF_addr__SHIFT 0x10
+#define VGT_DEBUG_REG8__rcm_busy_q_MASK 0x1
+#define VGT_DEBUG_REG8__rcm_busy_q__SHIFT 0x0
+#define VGT_DEBUG_REG8__rcm_noif_busy_q_MASK 0x2
+#define VGT_DEBUG_REG8__rcm_noif_busy_q__SHIFT 0x1
+#define VGT_DEBUG_REG8__r1_inst_rtr_MASK 0x4
+#define VGT_DEBUG_REG8__r1_inst_rtr__SHIFT 0x2
+#define VGT_DEBUG_REG8__spi_gsprim_fifo_busy_q_MASK 0x8
+#define VGT_DEBUG_REG8__spi_gsprim_fifo_busy_q__SHIFT 0x3
+#define VGT_DEBUG_REG8__spi_esvert_fifo_busy_q_MASK 0x10
+#define VGT_DEBUG_REG8__spi_esvert_fifo_busy_q__SHIFT 0x4
+#define VGT_DEBUG_REG8__gs_tbl_valid_r3_q_MASK 0x20
+#define VGT_DEBUG_REG8__gs_tbl_valid_r3_q__SHIFT 0x5
+#define VGT_DEBUG_REG8__valid_r0_q_MASK 0x40
+#define VGT_DEBUG_REG8__valid_r0_q__SHIFT 0x6
+#define VGT_DEBUG_REG8__valid_r1_q_MASK 0x80
+#define VGT_DEBUG_REG8__valid_r1_q__SHIFT 0x7
+#define VGT_DEBUG_REG8__valid_r2_MASK 0x100
+#define VGT_DEBUG_REG8__valid_r2__SHIFT 0x8
+#define VGT_DEBUG_REG8__valid_r2_q_MASK 0x200
+#define VGT_DEBUG_REG8__valid_r2_q__SHIFT 0x9
+#define VGT_DEBUG_REG8__r0_rtr_MASK 0x400
+#define VGT_DEBUG_REG8__r0_rtr__SHIFT 0xa
+#define VGT_DEBUG_REG8__r1_rtr_MASK 0x800
+#define VGT_DEBUG_REG8__r1_rtr__SHIFT 0xb
+#define VGT_DEBUG_REG8__r2_indx_rtr_MASK 0x1000
+#define VGT_DEBUG_REG8__r2_indx_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG8__r2_rtr_MASK 0x2000
+#define VGT_DEBUG_REG8__r2_rtr__SHIFT 0xd
+#define VGT_DEBUG_REG8__es_gs_rtr_MASK 0x4000
+#define VGT_DEBUG_REG8__es_gs_rtr__SHIFT 0xe
+#define VGT_DEBUG_REG8__gs_event_fifo_rtr_MASK 0x8000
+#define VGT_DEBUG_REG8__gs_event_fifo_rtr__SHIFT 0xf
+#define VGT_DEBUG_REG8__tm_rcm_gs_event_rtr_MASK 0x10000
+#define VGT_DEBUG_REG8__tm_rcm_gs_event_rtr__SHIFT 0x10
+#define VGT_DEBUG_REG8__gs_tbl_r3_rtr_MASK 0x20000
+#define VGT_DEBUG_REG8__gs_tbl_r3_rtr__SHIFT 0x11
+#define VGT_DEBUG_REG8__prim_skid_fifo_empty_MASK 0x40000
+#define VGT_DEBUG_REG8__prim_skid_fifo_empty__SHIFT 0x12
+#define VGT_DEBUG_REG8__VGT_SPI_gsprim_rtr_q_MASK 0x80000
+#define VGT_DEBUG_REG8__VGT_SPI_gsprim_rtr_q__SHIFT 0x13
+#define VGT_DEBUG_REG8__tm_rcm_gs_tbl_rtr_MASK 0x100000
+#define VGT_DEBUG_REG8__tm_rcm_gs_tbl_rtr__SHIFT 0x14
+#define VGT_DEBUG_REG8__tm_rcm_es_tbl_rtr_MASK 0x200000
+#define VGT_DEBUG_REG8__tm_rcm_es_tbl_rtr__SHIFT 0x15
+#define VGT_DEBUG_REG8__VGT_SPI_esvert_rtr_q_MASK 0x400000
+#define VGT_DEBUG_REG8__VGT_SPI_esvert_rtr_q__SHIFT 0x16
+#define VGT_DEBUG_REG8__r2_no_bp_rtr_MASK 0x800000
+#define VGT_DEBUG_REG8__r2_no_bp_rtr__SHIFT 0x17
+#define VGT_DEBUG_REG8__hold_for_es_flush_MASK 0x1000000
+#define VGT_DEBUG_REG8__hold_for_es_flush__SHIFT 0x18
+#define VGT_DEBUG_REG8__gs_event_fifo_empty_MASK 0x2000000
+#define VGT_DEBUG_REG8__gs_event_fifo_empty__SHIFT 0x19
+#define VGT_DEBUG_REG8__gsprim_buff_empty_q_MASK 0x4000000
+#define VGT_DEBUG_REG8__gsprim_buff_empty_q__SHIFT 0x1a
+#define VGT_DEBUG_REG8__gsprim_buff_full_q_MASK 0x8000000
+#define VGT_DEBUG_REG8__gsprim_buff_full_q__SHIFT 0x1b
+#define VGT_DEBUG_REG8__te_prim_fifo_empty_MASK 0x10000000
+#define VGT_DEBUG_REG8__te_prim_fifo_empty__SHIFT 0x1c
+#define VGT_DEBUG_REG8__te_prim_fifo_full_MASK 0x20000000
+#define VGT_DEBUG_REG8__te_prim_fifo_full__SHIFT 0x1d
+#define VGT_DEBUG_REG8__te_vert_fifo_empty_MASK 0x40000000
+#define VGT_DEBUG_REG8__te_vert_fifo_empty__SHIFT 0x1e
+#define VGT_DEBUG_REG8__te_vert_fifo_full_MASK 0x80000000
+#define VGT_DEBUG_REG8__te_vert_fifo_full__SHIFT 0x1f
+#define VGT_DEBUG_REG9__indices_to_send_r2_q_MASK 0x3
+#define VGT_DEBUG_REG9__indices_to_send_r2_q__SHIFT 0x0
+#define VGT_DEBUG_REG9__valid_indices_r3_MASK 0x4
+#define VGT_DEBUG_REG9__valid_indices_r3__SHIFT 0x2
+#define VGT_DEBUG_REG9__gs_eov_r3_MASK 0x8
+#define VGT_DEBUG_REG9__gs_eov_r3__SHIFT 0x3
+#define VGT_DEBUG_REG9__eop_indx_r3_MASK 0x10
+#define VGT_DEBUG_REG9__eop_indx_r3__SHIFT 0x4
+#define VGT_DEBUG_REG9__eop_prim_r3_MASK 0x20
+#define VGT_DEBUG_REG9__eop_prim_r3__SHIFT 0x5
+#define VGT_DEBUG_REG9__es_eov_r3_MASK 0x40
+#define VGT_DEBUG_REG9__es_eov_r3__SHIFT 0x6
+#define VGT_DEBUG_REG9__es_tbl_state_r3_q_0_MASK 0x80
+#define VGT_DEBUG_REG9__es_tbl_state_r3_q_0__SHIFT 0x7
+#define VGT_DEBUG_REG9__pending_es_send_r3_q_MASK 0x100
+#define VGT_DEBUG_REG9__pending_es_send_r3_q__SHIFT 0x8
+#define VGT_DEBUG_REG9__pending_es_flush_r3_MASK 0x200
+#define VGT_DEBUG_REG9__pending_es_flush_r3__SHIFT 0x9
+#define VGT_DEBUG_REG9__gs_tbl_num_es_per_gs_r3_q_not_0_MASK 0x400
+#define VGT_DEBUG_REG9__gs_tbl_num_es_per_gs_r3_q_not_0__SHIFT 0xa
+#define VGT_DEBUG_REG9__gs_tbl_prim_cnt_r3_q_MASK 0x3f800
+#define VGT_DEBUG_REG9__gs_tbl_prim_cnt_r3_q__SHIFT 0xb
+#define VGT_DEBUG_REG9__gs_tbl_eop_r3_q_MASK 0x40000
+#define VGT_DEBUG_REG9__gs_tbl_eop_r3_q__SHIFT 0x12
+#define VGT_DEBUG_REG9__gs_tbl_state_r3_q_MASK 0x380000
+#define VGT_DEBUG_REG9__gs_tbl_state_r3_q__SHIFT 0x13
+#define VGT_DEBUG_REG9__gs_pending_state_r3_q_MASK 0x400000
+#define VGT_DEBUG_REG9__gs_pending_state_r3_q__SHIFT 0x16
+#define VGT_DEBUG_REG9__invalidate_rb_roll_over_q_MASK 0x800000
+#define VGT_DEBUG_REG9__invalidate_rb_roll_over_q__SHIFT 0x17
+#define VGT_DEBUG_REG9__gs_instancing_state_q_MASK 0x1000000
+#define VGT_DEBUG_REG9__gs_instancing_state_q__SHIFT 0x18
+#define VGT_DEBUG_REG9__es_per_gs_vert_cnt_r3_q_not_0_MASK 0x2000000
+#define VGT_DEBUG_REG9__es_per_gs_vert_cnt_r3_q_not_0__SHIFT 0x19
+#define VGT_DEBUG_REG9__gs_prim_per_es_ctr_r3_q_not_0_MASK 0x4000000
+#define VGT_DEBUG_REG9__gs_prim_per_es_ctr_r3_q_not_0__SHIFT 0x1a
+#define VGT_DEBUG_REG9__pre_r0_rtr_MASK 0x8000000
+#define VGT_DEBUG_REG9__pre_r0_rtr__SHIFT 0x1b
+#define VGT_DEBUG_REG9__valid_r3_q_MASK 0x10000000
+#define VGT_DEBUG_REG9__valid_r3_q__SHIFT 0x1c
+#define VGT_DEBUG_REG9__valid_pre_r0_q_MASK 0x20000000
+#define VGT_DEBUG_REG9__valid_pre_r0_q__SHIFT 0x1d
+#define VGT_DEBUG_REG9__SPARE0_MASK 0x40000000
+#define VGT_DEBUG_REG9__SPARE0__SHIFT 0x1e
+#define VGT_DEBUG_REG9__off_chip_hs_r2_q_MASK 0x80000000
+#define VGT_DEBUG_REG9__off_chip_hs_r2_q__SHIFT 0x1f
+#define VGT_DEBUG_REG10__index_buffer_depth_r1_q_MASK 0x1f
+#define VGT_DEBUG_REG10__index_buffer_depth_r1_q__SHIFT 0x0
+#define VGT_DEBUG_REG10__eopg_r2_q_MASK 0x20
+#define VGT_DEBUG_REG10__eopg_r2_q__SHIFT 0x5
+#define VGT_DEBUG_REG10__eotg_r2_q_MASK 0x40
+#define VGT_DEBUG_REG10__eotg_r2_q__SHIFT 0x6
+#define VGT_DEBUG_REG10__onchip_gs_en_r0_q_MASK 0x180
+#define VGT_DEBUG_REG10__onchip_gs_en_r0_q__SHIFT 0x7
+#define VGT_DEBUG_REG10__SPARE2_MASK 0x600
+#define VGT_DEBUG_REG10__SPARE2__SHIFT 0x9
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_qq_MASK 0x800
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_qq__SHIFT 0xb
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_q_MASK 0x1000
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_q__SHIFT 0xc
+#define VGT_DEBUG_REG10__gs_rb_space_avail_r3_q_9_0_MASK 0x7fe000
+#define VGT_DEBUG_REG10__gs_rb_space_avail_r3_q_9_0__SHIFT 0xd
+#define VGT_DEBUG_REG10__es_rb_space_avail_r2_q_8_0_MASK 0xff800000
+#define VGT_DEBUG_REG10__es_rb_space_avail_r2_q_8_0__SHIFT 0x17
+#define VGT_DEBUG_REG11__tm_busy_q_MASK 0x1
+#define VGT_DEBUG_REG11__tm_busy_q__SHIFT 0x0
+#define VGT_DEBUG_REG11__tm_noif_busy_q_MASK 0x2
+#define VGT_DEBUG_REG11__tm_noif_busy_q__SHIFT 0x1
+#define VGT_DEBUG_REG11__tm_out_busy_q_MASK 0x4
+#define VGT_DEBUG_REG11__tm_out_busy_q__SHIFT 0x2
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_busy_MASK 0x8
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_busy__SHIFT 0x3
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_busy_MASK 0x10
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_busy__SHIFT 0x4
+#define VGT_DEBUG_REG11__SPARE1_MASK 0x20
+#define VGT_DEBUG_REG11__SPARE1__SHIFT 0x5
+#define VGT_DEBUG_REG11__spi_gsthread_fifo_busy_MASK 0x40
+#define VGT_DEBUG_REG11__spi_gsthread_fifo_busy__SHIFT 0x6
+#define VGT_DEBUG_REG11__spi_esthread_fifo_busy_MASK 0x80
+#define VGT_DEBUG_REG11__spi_esthread_fifo_busy__SHIFT 0x7
+#define VGT_DEBUG_REG11__hold_eswave_MASK 0x100
+#define VGT_DEBUG_REG11__hold_eswave__SHIFT 0x8
+#define VGT_DEBUG_REG11__es_rb_roll_over_r3_MASK 0x200
+#define VGT_DEBUG_REG11__es_rb_roll_over_r3__SHIFT 0x9
+#define VGT_DEBUG_REG11__counters_busy_r0_MASK 0x400
+#define VGT_DEBUG_REG11__counters_busy_r0__SHIFT 0xa
+#define VGT_DEBUG_REG11__counters_avail_r0_MASK 0x800
+#define VGT_DEBUG_REG11__counters_avail_r0__SHIFT 0xb
+#define VGT_DEBUG_REG11__counters_available_r0_MASK 0x1000
+#define VGT_DEBUG_REG11__counters_available_r0__SHIFT 0xc
+#define VGT_DEBUG_REG11__vs_event_fifo_rtr_MASK 0x2000
+#define VGT_DEBUG_REG11__vs_event_fifo_rtr__SHIFT 0xd
+#define VGT_DEBUG_REG11__VGT_SPI_gsthread_rtr_q_MASK 0x4000
+#define VGT_DEBUG_REG11__VGT_SPI_gsthread_rtr_q__SHIFT 0xe
+#define VGT_DEBUG_REG11__VGT_SPI_esthread_rtr_q_MASK 0x8000
+#define VGT_DEBUG_REG11__VGT_SPI_esthread_rtr_q__SHIFT 0xf
+#define VGT_DEBUG_REG11__gs_issue_rtr_MASK 0x10000
+#define VGT_DEBUG_REG11__gs_issue_rtr__SHIFT 0x10
+#define VGT_DEBUG_REG11__tm_pt_event_rtr_MASK 0x20000
+#define VGT_DEBUG_REG11__tm_pt_event_rtr__SHIFT 0x11
+#define VGT_DEBUG_REG11__SPARE0_MASK 0x40000
+#define VGT_DEBUG_REG11__SPARE0__SHIFT 0x12
+#define VGT_DEBUG_REG11__gs_r0_rtr_MASK 0x80000
+#define VGT_DEBUG_REG11__gs_r0_rtr__SHIFT 0x13
+#define VGT_DEBUG_REG11__es_r0_rtr_MASK 0x100000
+#define VGT_DEBUG_REG11__es_r0_rtr__SHIFT 0x14
+#define VGT_DEBUG_REG11__gog_tm_vs_event_rtr_MASK 0x200000
+#define VGT_DEBUG_REG11__gog_tm_vs_event_rtr__SHIFT 0x15
+#define VGT_DEBUG_REG11__tm_rcm_gs_event_rtr_MASK 0x400000
+#define VGT_DEBUG_REG11__tm_rcm_gs_event_rtr__SHIFT 0x16
+#define VGT_DEBUG_REG11__tm_rcm_gs_tbl_rtr_MASK 0x800000
+#define VGT_DEBUG_REG11__tm_rcm_gs_tbl_rtr__SHIFT 0x17
+#define VGT_DEBUG_REG11__tm_rcm_es_tbl_rtr_MASK 0x1000000
+#define VGT_DEBUG_REG11__tm_rcm_es_tbl_rtr__SHIFT 0x18
+#define VGT_DEBUG_REG11__vs_event_fifo_empty_MASK 0x2000000
+#define VGT_DEBUG_REG11__vs_event_fifo_empty__SHIFT 0x19
+#define VGT_DEBUG_REG11__vs_event_fifo_full_MASK 0x4000000
+#define VGT_DEBUG_REG11__vs_event_fifo_full__SHIFT 0x1a
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_full_MASK 0x8000000
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_full__SHIFT 0x1b
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_full_MASK 0x10000000
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_full__SHIFT 0x1c
+#define VGT_DEBUG_REG11__send_event_q_MASK 0x20000000
+#define VGT_DEBUG_REG11__send_event_q__SHIFT 0x1d
+#define VGT_DEBUG_REG11__es_tbl_empty_MASK 0x40000000
+#define VGT_DEBUG_REG11__es_tbl_empty__SHIFT 0x1e
+#define VGT_DEBUG_REG11__no_active_states_r0_MASK 0x80000000
+#define VGT_DEBUG_REG11__no_active_states_r0__SHIFT 0x1f
+#define VGT_DEBUG_REG12__gs_state0_r0_q_MASK 0x7
+#define VGT_DEBUG_REG12__gs_state0_r0_q__SHIFT 0x0
+#define VGT_DEBUG_REG12__gs_state1_r0_q_MASK 0x38
+#define VGT_DEBUG_REG12__gs_state1_r0_q__SHIFT 0x3
+#define VGT_DEBUG_REG12__gs_state2_r0_q_MASK 0x1c0
+#define VGT_DEBUG_REG12__gs_state2_r0_q__SHIFT 0x6
+#define VGT_DEBUG_REG12__gs_state3_r0_q_MASK 0xe00
+#define VGT_DEBUG_REG12__gs_state3_r0_q__SHIFT 0x9
+#define VGT_DEBUG_REG12__gs_state4_r0_q_MASK 0x7000
+#define VGT_DEBUG_REG12__gs_state4_r0_q__SHIFT 0xc
+#define VGT_DEBUG_REG12__gs_state5_r0_q_MASK 0x38000
+#define VGT_DEBUG_REG12__gs_state5_r0_q__SHIFT 0xf
+#define VGT_DEBUG_REG12__gs_state6_r0_q_MASK 0x1c0000
+#define VGT_DEBUG_REG12__gs_state6_r0_q__SHIFT 0x12
+#define VGT_DEBUG_REG12__gs_state7_r0_q_MASK 0xe00000
+#define VGT_DEBUG_REG12__gs_state7_r0_q__SHIFT 0x15
+#define VGT_DEBUG_REG12__gs_state8_r0_q_MASK 0x7000000
+#define VGT_DEBUG_REG12__gs_state8_r0_q__SHIFT 0x18
+#define VGT_DEBUG_REG12__gs_state9_r0_q_MASK 0x38000000
+#define VGT_DEBUG_REG12__gs_state9_r0_q__SHIFT 0x1b
+#define VGT_DEBUG_REG12__hold_eswave_eop_MASK 0x40000000
+#define VGT_DEBUG_REG12__hold_eswave_eop__SHIFT 0x1e
+#define VGT_DEBUG_REG12__SPARE0_MASK 0x80000000
+#define VGT_DEBUG_REG12__SPARE0__SHIFT 0x1f
+#define VGT_DEBUG_REG13__gs_state10_r0_q_MASK 0x7
+#define VGT_DEBUG_REG13__gs_state10_r0_q__SHIFT 0x0
+#define VGT_DEBUG_REG13__gs_state11_r0_q_MASK 0x38
+#define VGT_DEBUG_REG13__gs_state11_r0_q__SHIFT 0x3
+#define VGT_DEBUG_REG13__gs_state12_r0_q_MASK 0x1c0
+#define VGT_DEBUG_REG13__gs_state12_r0_q__SHIFT 0x6
+#define VGT_DEBUG_REG13__gs_state13_r0_q_MASK 0xe00
+#define VGT_DEBUG_REG13__gs_state13_r0_q__SHIFT 0x9
+#define VGT_DEBUG_REG13__gs_state14_r0_q_MASK 0x7000
+#define VGT_DEBUG_REG13__gs_state14_r0_q__SHIFT 0xc
+#define VGT_DEBUG_REG13__gs_state15_r0_q_MASK 0x38000
+#define VGT_DEBUG_REG13__gs_state15_r0_q__SHIFT 0xf
+#define VGT_DEBUG_REG13__gs_tbl_wrptr_r0_q_3_0_MASK 0x3c0000
+#define VGT_DEBUG_REG13__gs_tbl_wrptr_r0_q_3_0__SHIFT 0x12
+#define VGT_DEBUG_REG13__gsfetch_done_fifo_cnt_q_not_0_MASK 0x400000
+#define VGT_DEBUG_REG13__gsfetch_done_fifo_cnt_q_not_0__SHIFT 0x16
+#define VGT_DEBUG_REG13__gsfetch_done_cnt_q_not_0_MASK 0x800000
+#define VGT_DEBUG_REG13__gsfetch_done_cnt_q_not_0__SHIFT 0x17
+#define VGT_DEBUG_REG13__es_tbl_full_MASK 0x1000000
+#define VGT_DEBUG_REG13__es_tbl_full__SHIFT 0x18
+#define VGT_DEBUG_REG13__SPARE1_MASK 0x2000000
+#define VGT_DEBUG_REG13__SPARE1__SHIFT 0x19
+#define VGT_DEBUG_REG13__SPARE0_MASK 0x4000000
+#define VGT_DEBUG_REG13__SPARE0__SHIFT 0x1a
+#define VGT_DEBUG_REG13__active_cm_sm_r0_q_MASK 0xf8000000
+#define VGT_DEBUG_REG13__active_cm_sm_r0_q__SHIFT 0x1b
+#define VGT_DEBUG_REG14__SPARE3_MASK 0xf
+#define VGT_DEBUG_REG14__SPARE3__SHIFT 0x0
+#define VGT_DEBUG_REG14__gsfetch_done_fifo_full_MASK 0x10
+#define VGT_DEBUG_REG14__gsfetch_done_fifo_full__SHIFT 0x4
+#define VGT_DEBUG_REG14__gs_rb_space_avail_r0_MASK 0x20
+#define VGT_DEBUG_REG14__gs_rb_space_avail_r0__SHIFT 0x5
+#define VGT_DEBUG_REG14__smx_es_done_cnt_r0_q_not_0_MASK 0x40
+#define VGT_DEBUG_REG14__smx_es_done_cnt_r0_q_not_0__SHIFT 0x6
+#define VGT_DEBUG_REG14__SPARE8_MASK 0x180
+#define VGT_DEBUG_REG14__SPARE8__SHIFT 0x7
+#define VGT_DEBUG_REG14__vs_done_cnt_q_not_0_MASK 0x200
+#define VGT_DEBUG_REG14__vs_done_cnt_q_not_0__SHIFT 0x9
+#define VGT_DEBUG_REG14__es_flush_cnt_busy_q_MASK 0x400
+#define VGT_DEBUG_REG14__es_flush_cnt_busy_q__SHIFT 0xa
+#define VGT_DEBUG_REG14__gs_tbl_full_r0_MASK 0x800
+#define VGT_DEBUG_REG14__gs_tbl_full_r0__SHIFT 0xb
+#define VGT_DEBUG_REG14__SPARE2_MASK 0x1ff000
+#define VGT_DEBUG_REG14__SPARE2__SHIFT 0xc
+#define VGT_DEBUG_REG14__se1spi_gsthread_fifo_busy_MASK 0x200000
+#define VGT_DEBUG_REG14__se1spi_gsthread_fifo_busy__SHIFT 0x15
+#define VGT_DEBUG_REG14__SPARE_MASK 0x1c00000
+#define VGT_DEBUG_REG14__SPARE__SHIFT 0x16
+#define VGT_DEBUG_REG14__VGT_SE1SPI_gsthread_rtr_q_MASK 0x2000000
+#define VGT_DEBUG_REG14__VGT_SE1SPI_gsthread_rtr_q__SHIFT 0x19
+#define VGT_DEBUG_REG14__smx1_es_done_cnt_r0_q_not_0_MASK 0x4000000
+#define VGT_DEBUG_REG14__smx1_es_done_cnt_r0_q_not_0__SHIFT 0x1a
+#define VGT_DEBUG_REG14__se1spi_esthread_fifo_busy_MASK 0x8000000
+#define VGT_DEBUG_REG14__se1spi_esthread_fifo_busy__SHIFT 0x1b
+#define VGT_DEBUG_REG14__SPARE1_MASK 0x10000000
+#define VGT_DEBUG_REG14__SPARE1__SHIFT 0x1c
+#define VGT_DEBUG_REG14__gsfetch_done_se1_cnt_q_not_0_MASK 0x20000000
+#define VGT_DEBUG_REG14__gsfetch_done_se1_cnt_q_not_0__SHIFT 0x1d
+#define VGT_DEBUG_REG14__SPARE0_MASK 0x40000000
+#define VGT_DEBUG_REG14__SPARE0__SHIFT 0x1e
+#define VGT_DEBUG_REG14__VGT_SE1SPI_esthread_rtr_q_MASK 0x80000000
+#define VGT_DEBUG_REG14__VGT_SE1SPI_esthread_rtr_q__SHIFT 0x1f
+#define VGT_DEBUG_REG15__cm_busy_q_MASK 0x1
+#define VGT_DEBUG_REG15__cm_busy_q__SHIFT 0x0
+#define VGT_DEBUG_REG15__counters_busy_q_MASK 0x2
+#define VGT_DEBUG_REG15__counters_busy_q__SHIFT 0x1
+#define VGT_DEBUG_REG15__output_fifo_empty_MASK 0x4
+#define VGT_DEBUG_REG15__output_fifo_empty__SHIFT 0x2
+#define VGT_DEBUG_REG15__output_fifo_full_MASK 0x8
+#define VGT_DEBUG_REG15__output_fifo_full__SHIFT 0x3
+#define VGT_DEBUG_REG15__counters_full_MASK 0x10
+#define VGT_DEBUG_REG15__counters_full__SHIFT 0x4
+#define VGT_DEBUG_REG15__active_sm_q_MASK 0x3e0
+#define VGT_DEBUG_REG15__active_sm_q__SHIFT 0x5
+#define VGT_DEBUG_REG15__entry_rdptr_q_MASK 0x7c00
+#define VGT_DEBUG_REG15__entry_rdptr_q__SHIFT 0xa
+#define VGT_DEBUG_REG15__cntr_tbl_wrptr_q_MASK 0xf8000
+#define VGT_DEBUG_REG15__cntr_tbl_wrptr_q__SHIFT 0xf
+#define VGT_DEBUG_REG15__SPARE25_MASK 0x3f00000
+#define VGT_DEBUG_REG15__SPARE25__SHIFT 0x14
+#define VGT_DEBUG_REG15__st_cut_mode_q_MASK 0xc000000
+#define VGT_DEBUG_REG15__st_cut_mode_q__SHIFT 0x1a
+#define VGT_DEBUG_REG15__gs_done_array_q_not_0_MASK 0x10000000
+#define VGT_DEBUG_REG15__gs_done_array_q_not_0__SHIFT 0x1c
+#define VGT_DEBUG_REG15__SPARE31_MASK 0xe0000000
+#define VGT_DEBUG_REG15__SPARE31__SHIFT 0x1d
+#define VGT_DEBUG_REG16__gog_busy_MASK 0x1
+#define VGT_DEBUG_REG16__gog_busy__SHIFT 0x0
+#define VGT_DEBUG_REG16__gog_state_q_MASK 0xe
+#define VGT_DEBUG_REG16__gog_state_q__SHIFT 0x1
+#define VGT_DEBUG_REG16__r0_rtr_MASK 0x10
+#define VGT_DEBUG_REG16__r0_rtr__SHIFT 0x4
+#define VGT_DEBUG_REG16__r1_rtr_MASK 0x20
+#define VGT_DEBUG_REG16__r1_rtr__SHIFT 0x5
+#define VGT_DEBUG_REG16__r1_upstream_rtr_MASK 0x40
+#define VGT_DEBUG_REG16__r1_upstream_rtr__SHIFT 0x6
+#define VGT_DEBUG_REG16__r2_vs_tbl_rtr_MASK 0x80
+#define VGT_DEBUG_REG16__r2_vs_tbl_rtr__SHIFT 0x7
+#define VGT_DEBUG_REG16__r2_prim_rtr_MASK 0x100
+#define VGT_DEBUG_REG16__r2_prim_rtr__SHIFT 0x8
+#define VGT_DEBUG_REG16__r2_indx_rtr_MASK 0x200
+#define VGT_DEBUG_REG16__r2_indx_rtr__SHIFT 0x9
+#define VGT_DEBUG_REG16__r2_rtr_MASK 0x400
+#define VGT_DEBUG_REG16__r2_rtr__SHIFT 0xa
+#define VGT_DEBUG_REG16__gog_tm_vs_event_rtr_MASK 0x800
+#define VGT_DEBUG_REG16__gog_tm_vs_event_rtr__SHIFT 0xb
+#define VGT_DEBUG_REG16__r3_force_vs_tbl_we_rtr_MASK 0x1000
+#define VGT_DEBUG_REG16__r3_force_vs_tbl_we_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG16__indx_valid_r2_q_MASK 0x2000
+#define VGT_DEBUG_REG16__indx_valid_r2_q__SHIFT 0xd
+#define VGT_DEBUG_REG16__prim_valid_r2_q_MASK 0x4000
+#define VGT_DEBUG_REG16__prim_valid_r2_q__SHIFT 0xe
+#define VGT_DEBUG_REG16__valid_r2_q_MASK 0x8000
+#define VGT_DEBUG_REG16__valid_r2_q__SHIFT 0xf
+#define VGT_DEBUG_REG16__prim_valid_r1_q_MASK 0x10000
+#define VGT_DEBUG_REG16__prim_valid_r1_q__SHIFT 0x10
+#define VGT_DEBUG_REG16__indx_valid_r1_q_MASK 0x20000
+#define VGT_DEBUG_REG16__indx_valid_r1_q__SHIFT 0x11
+#define VGT_DEBUG_REG16__valid_r1_q_MASK 0x40000
+#define VGT_DEBUG_REG16__valid_r1_q__SHIFT 0x12
+#define VGT_DEBUG_REG16__indx_valid_r0_q_MASK 0x80000
+#define VGT_DEBUG_REG16__indx_valid_r0_q__SHIFT 0x13
+#define VGT_DEBUG_REG16__prim_valid_r0_q_MASK 0x100000
+#define VGT_DEBUG_REG16__prim_valid_r0_q__SHIFT 0x14
+#define VGT_DEBUG_REG16__valid_r0_q_MASK 0x200000
+#define VGT_DEBUG_REG16__valid_r0_q__SHIFT 0x15
+#define VGT_DEBUG_REG16__send_event_q_MASK 0x400000
+#define VGT_DEBUG_REG16__send_event_q__SHIFT 0x16
+#define VGT_DEBUG_REG16__SPARE24_MASK 0x800000
+#define VGT_DEBUG_REG16__SPARE24__SHIFT 0x17
+#define VGT_DEBUG_REG16__vert_seen_since_sopg_r2_q_MASK 0x1000000
+#define VGT_DEBUG_REG16__vert_seen_since_sopg_r2_q__SHIFT 0x18
+#define VGT_DEBUG_REG16__gog_out_prim_state_sel_MASK 0xe000000
+#define VGT_DEBUG_REG16__gog_out_prim_state_sel__SHIFT 0x19
+#define VGT_DEBUG_REG16__multiple_streams_en_r1_q_MASK 0x10000000
+#define VGT_DEBUG_REG16__multiple_streams_en_r1_q__SHIFT 0x1c
+#define VGT_DEBUG_REG16__vs_vert_count_r2_q_not_0_MASK 0x20000000
+#define VGT_DEBUG_REG16__vs_vert_count_r2_q_not_0__SHIFT 0x1d
+#define VGT_DEBUG_REG16__num_gs_r2_q_not_0_MASK 0x40000000
+#define VGT_DEBUG_REG16__num_gs_r2_q_not_0__SHIFT 0x1e
+#define VGT_DEBUG_REG16__new_vs_thread_r2_MASK 0x80000000
+#define VGT_DEBUG_REG16__new_vs_thread_r2__SHIFT 0x1f
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx2_5_0_MASK 0x3f
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx2_5_0__SHIFT 0x0
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx1_5_0_MASK 0xfc0
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx1_5_0__SHIFT 0x6
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx0_5_0_MASK 0x3f000
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx0_5_0__SHIFT 0xc
+#define VGT_DEBUG_REG17__gog_out_indx_13_0_MASK 0xfffc0000
+#define VGT_DEBUG_REG17__gog_out_indx_13_0__SHIFT 0x12
+#define VGT_DEBUG_REG18__grp_vr_valid_MASK 0x1
+#define VGT_DEBUG_REG18__grp_vr_valid__SHIFT 0x0
+#define VGT_DEBUG_REG18__pipe0_dr_MASK 0x2
+#define VGT_DEBUG_REG18__pipe0_dr__SHIFT 0x1
+#define VGT_DEBUG_REG18__pipe1_dr_MASK 0x4
+#define VGT_DEBUG_REG18__pipe1_dr__SHIFT 0x2
+#define VGT_DEBUG_REG18__vr_grp_read_MASK 0x8
+#define VGT_DEBUG_REG18__vr_grp_read__SHIFT 0x3
+#define VGT_DEBUG_REG18__pipe0_rtr_MASK 0x10
+#define VGT_DEBUG_REG18__pipe0_rtr__SHIFT 0x4
+#define VGT_DEBUG_REG18__pipe1_rtr_MASK 0x20
+#define VGT_DEBUG_REG18__pipe1_rtr__SHIFT 0x5
+#define VGT_DEBUG_REG18__out_vr_indx_read_MASK 0x40
+#define VGT_DEBUG_REG18__out_vr_indx_read__SHIFT 0x6
+#define VGT_DEBUG_REG18__out_vr_prim_read_MASK 0x80
+#define VGT_DEBUG_REG18__out_vr_prim_read__SHIFT 0x7
+#define VGT_DEBUG_REG18__indices_to_send_q_MASK 0x700
+#define VGT_DEBUG_REG18__indices_to_send_q__SHIFT 0x8
+#define VGT_DEBUG_REG18__valid_indices_MASK 0x800
+#define VGT_DEBUG_REG18__valid_indices__SHIFT 0xb
+#define VGT_DEBUG_REG18__last_indx_of_prim_MASK 0x1000
+#define VGT_DEBUG_REG18__last_indx_of_prim__SHIFT 0xc
+#define VGT_DEBUG_REG18__indx0_new_d_MASK 0x2000
+#define VGT_DEBUG_REG18__indx0_new_d__SHIFT 0xd
+#define VGT_DEBUG_REG18__indx1_new_d_MASK 0x4000
+#define VGT_DEBUG_REG18__indx1_new_d__SHIFT 0xe
+#define VGT_DEBUG_REG18__indx2_new_d_MASK 0x8000
+#define VGT_DEBUG_REG18__indx2_new_d__SHIFT 0xf
+#define VGT_DEBUG_REG18__indx2_hit_d_MASK 0x10000
+#define VGT_DEBUG_REG18__indx2_hit_d__SHIFT 0x10
+#define VGT_DEBUG_REG18__indx1_hit_d_MASK 0x20000
+#define VGT_DEBUG_REG18__indx1_hit_d__SHIFT 0x11
+#define VGT_DEBUG_REG18__indx0_hit_d_MASK 0x40000
+#define VGT_DEBUG_REG18__indx0_hit_d__SHIFT 0x12
+#define VGT_DEBUG_REG18__st_vertex_reuse_off_r0_q_MASK 0x80000
+#define VGT_DEBUG_REG18__st_vertex_reuse_off_r0_q__SHIFT 0x13
+#define VGT_DEBUG_REG18__last_group_of_instance_r0_q_MASK 0x100000
+#define VGT_DEBUG_REG18__last_group_of_instance_r0_q__SHIFT 0x14
+#define VGT_DEBUG_REG18__null_primitive_r0_q_MASK 0x200000
+#define VGT_DEBUG_REG18__null_primitive_r0_q__SHIFT 0x15
+#define VGT_DEBUG_REG18__eop_r0_q_MASK 0x400000
+#define VGT_DEBUG_REG18__eop_r0_q__SHIFT 0x16
+#define VGT_DEBUG_REG18__eject_vtx_vect_r1_d_MASK 0x800000
+#define VGT_DEBUG_REG18__eject_vtx_vect_r1_d__SHIFT 0x17
+#define VGT_DEBUG_REG18__sub_prim_type_r0_q_MASK 0x7000000
+#define VGT_DEBUG_REG18__sub_prim_type_r0_q__SHIFT 0x18
+#define VGT_DEBUG_REG18__gs_scenario_a_r0_q_MASK 0x8000000
+#define VGT_DEBUG_REG18__gs_scenario_a_r0_q__SHIFT 0x1b
+#define VGT_DEBUG_REG18__gs_scenario_b_r0_q_MASK 0x10000000
+#define VGT_DEBUG_REG18__gs_scenario_b_r0_q__SHIFT 0x1c
+#define VGT_DEBUG_REG18__components_valid_r0_q_MASK 0xe0000000
+#define VGT_DEBUG_REG18__components_valid_r0_q__SHIFT 0x1d
+#define VGT_DEBUG_REG19__separate_out_busy_q_MASK 0x1
+#define VGT_DEBUG_REG19__separate_out_busy_q__SHIFT 0x0
+#define VGT_DEBUG_REG19__separate_out_indx_busy_q_MASK 0x2
+#define VGT_DEBUG_REG19__separate_out_indx_busy_q__SHIFT 0x1
+#define VGT_DEBUG_REG19__prim_buffer_empty_MASK 0x4
+#define VGT_DEBUG_REG19__prim_buffer_empty__SHIFT 0x2
+#define VGT_DEBUG_REG19__prim_buffer_full_MASK 0x8
+#define VGT_DEBUG_REG19__prim_buffer_full__SHIFT 0x3
+#define VGT_DEBUG_REG19__pa_clips_fifo_busy_q_MASK 0x10
+#define VGT_DEBUG_REG19__pa_clips_fifo_busy_q__SHIFT 0x4
+#define VGT_DEBUG_REG19__pa_clipp_fifo_busy_q_MASK 0x20
+#define VGT_DEBUG_REG19__pa_clipp_fifo_busy_q__SHIFT 0x5
+#define VGT_DEBUG_REG19__VGT_PA_clips_rtr_q_MASK 0x40
+#define VGT_DEBUG_REG19__VGT_PA_clips_rtr_q__SHIFT 0x6
+#define VGT_DEBUG_REG19__VGT_PA_clipp_rtr_q_MASK 0x80
+#define VGT_DEBUG_REG19__VGT_PA_clipp_rtr_q__SHIFT 0x7
+#define VGT_DEBUG_REG19__spi_vsthread_fifo_busy_q_MASK 0x100
+#define VGT_DEBUG_REG19__spi_vsthread_fifo_busy_q__SHIFT 0x8
+#define VGT_DEBUG_REG19__spi_vsvert_fifo_busy_q_MASK 0x200
+#define VGT_DEBUG_REG19__spi_vsvert_fifo_busy_q__SHIFT 0x9
+#define VGT_DEBUG_REG19__pa_clipv_fifo_busy_q_MASK 0x400
+#define VGT_DEBUG_REG19__pa_clipv_fifo_busy_q__SHIFT 0xa
+#define VGT_DEBUG_REG19__hold_prim_MASK 0x800
+#define VGT_DEBUG_REG19__hold_prim__SHIFT 0xb
+#define VGT_DEBUG_REG19__VGT_SPI_vsthread_rtr_q_MASK 0x1000
+#define VGT_DEBUG_REG19__VGT_SPI_vsthread_rtr_q__SHIFT 0xc
+#define VGT_DEBUG_REG19__VGT_SPI_vsvert_rtr_q_MASK 0x2000
+#define VGT_DEBUG_REG19__VGT_SPI_vsvert_rtr_q__SHIFT 0xd
+#define VGT_DEBUG_REG19__VGT_PA_clipv_rtr_q_MASK 0x4000
+#define VGT_DEBUG_REG19__VGT_PA_clipv_rtr_q__SHIFT 0xe
+#define VGT_DEBUG_REG19__new_packet_q_MASK 0x8000
+#define VGT_DEBUG_REG19__new_packet_q__SHIFT 0xf
+#define VGT_DEBUG_REG19__buffered_prim_event_MASK 0x10000
+#define VGT_DEBUG_REG19__buffered_prim_event__SHIFT 0x10
+#define VGT_DEBUG_REG19__buffered_prim_null_primitive_MASK 0x20000
+#define VGT_DEBUG_REG19__buffered_prim_null_primitive__SHIFT 0x11
+#define VGT_DEBUG_REG19__buffered_prim_eop_MASK 0x40000
+#define VGT_DEBUG_REG19__buffered_prim_eop__SHIFT 0x12
+#define VGT_DEBUG_REG19__buffered_prim_eject_vtx_vect_MASK 0x80000
+#define VGT_DEBUG_REG19__buffered_prim_eject_vtx_vect__SHIFT 0x13
+#define VGT_DEBUG_REG19__buffered_prim_type_event_MASK 0x3f00000
+#define VGT_DEBUG_REG19__buffered_prim_type_event__SHIFT 0x14
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vswave_rtr_q_MASK 0x4000000
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vswave_rtr_q__SHIFT 0x1a
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vsvert_rtr_q_MASK 0x8000000
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vsvert_rtr_q__SHIFT 0x1b
+#define VGT_DEBUG_REG19__num_new_unique_rel_indx_MASK 0x30000000
+#define VGT_DEBUG_REG19__num_new_unique_rel_indx__SHIFT 0x1c
+#define VGT_DEBUG_REG19__null_terminate_vtx_vector_MASK 0x40000000
+#define VGT_DEBUG_REG19__null_terminate_vtx_vector__SHIFT 0x1e
+#define VGT_DEBUG_REG19__filter_event_MASK 0x80000000
+#define VGT_DEBUG_REG19__filter_event__SHIFT 0x1f
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexindex_MASK 0xffff
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexindex__SHIFT 0x0
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexcount_not_0_MASK 0x10000
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexcount_not_0__SHIFT 0x10
+#define VGT_DEBUG_REG20__SPARE17_MASK 0x20000
+#define VGT_DEBUG_REG20__SPARE17__SHIFT 0x11
+#define VGT_DEBUG_REG20__alloc_counter_q_MASK 0x3c0000
+#define VGT_DEBUG_REG20__alloc_counter_q__SHIFT 0x12
+#define VGT_DEBUG_REG20__curr_dealloc_distance_q_MASK 0x1fc00000
+#define VGT_DEBUG_REG20__curr_dealloc_distance_q__SHIFT 0x16
+#define VGT_DEBUG_REG20__new_allocate_q_MASK 0x20000000
+#define VGT_DEBUG_REG20__new_allocate_q__SHIFT 0x1d
+#define VGT_DEBUG_REG20__curr_slot_in_vtx_vect_q_not_0_MASK 0x40000000
+#define VGT_DEBUG_REG20__curr_slot_in_vtx_vect_q_not_0__SHIFT 0x1e
+#define VGT_DEBUG_REG20__int_vtx_counter_q_not_0_MASK 0x80000000
+#define VGT_DEBUG_REG20__int_vtx_counter_q_not_0__SHIFT 0x1f
+#define VGT_DEBUG_REG21__out_indx_fifo_empty_MASK 0x1
+#define VGT_DEBUG_REG21__out_indx_fifo_empty__SHIFT 0x0
+#define VGT_DEBUG_REG21__indx_side_fifo_empty_MASK 0x2
+#define VGT_DEBUG_REG21__indx_side_fifo_empty__SHIFT 0x1
+#define VGT_DEBUG_REG21__pipe0_dr_MASK 0x4
+#define VGT_DEBUG_REG21__pipe0_dr__SHIFT 0x2
+#define VGT_DEBUG_REG21__pipe1_dr_MASK 0x8
+#define VGT_DEBUG_REG21__pipe1_dr__SHIFT 0x3
+#define VGT_DEBUG_REG21__pipe2_dr_MASK 0x10
+#define VGT_DEBUG_REG21__pipe2_dr__SHIFT 0x4
+#define VGT_DEBUG_REG21__vsthread_buff_empty_MASK 0x20
+#define VGT_DEBUG_REG21__vsthread_buff_empty__SHIFT 0x5
+#define VGT_DEBUG_REG21__out_indx_fifo_full_MASK 0x40
+#define VGT_DEBUG_REG21__out_indx_fifo_full__SHIFT 0x6
+#define VGT_DEBUG_REG21__indx_side_fifo_full_MASK 0x80
+#define VGT_DEBUG_REG21__indx_side_fifo_full__SHIFT 0x7
+#define VGT_DEBUG_REG21__pipe0_rtr_MASK 0x100
+#define VGT_DEBUG_REG21__pipe0_rtr__SHIFT 0x8
+#define VGT_DEBUG_REG21__pipe1_rtr_MASK 0x200
+#define VGT_DEBUG_REG21__pipe1_rtr__SHIFT 0x9
+#define VGT_DEBUG_REG21__pipe2_rtr_MASK 0x400
+#define VGT_DEBUG_REG21__pipe2_rtr__SHIFT 0xa
+#define VGT_DEBUG_REG21__vsthread_buff_full_MASK 0x800
+#define VGT_DEBUG_REG21__vsthread_buff_full__SHIFT 0xb
+#define VGT_DEBUG_REG21__interfaces_rtr_MASK 0x1000
+#define VGT_DEBUG_REG21__interfaces_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG21__indx_count_q_not_0_MASK 0x2000
+#define VGT_DEBUG_REG21__indx_count_q_not_0__SHIFT 0xd
+#define VGT_DEBUG_REG21__wait_for_external_eopg_q_MASK 0x4000
+#define VGT_DEBUG_REG21__wait_for_external_eopg_q__SHIFT 0xe
+#define VGT_DEBUG_REG21__full_state_p1_q_MASK 0x8000
+#define VGT_DEBUG_REG21__full_state_p1_q__SHIFT 0xf
+#define VGT_DEBUG_REG21__indx_side_indx_valid_MASK 0x10000
+#define VGT_DEBUG_REG21__indx_side_indx_valid__SHIFT 0x10
+#define VGT_DEBUG_REG21__stateid_p0_q_MASK 0xe0000
+#define VGT_DEBUG_REG21__stateid_p0_q__SHIFT 0x11
+#define VGT_DEBUG_REG21__is_event_p0_q_MASK 0x100000
+#define VGT_DEBUG_REG21__is_event_p0_q__SHIFT 0x14
+#define VGT_DEBUG_REG21__lshs_dealloc_p1_MASK 0x200000
+#define VGT_DEBUG_REG21__lshs_dealloc_p1__SHIFT 0x15
+#define VGT_DEBUG_REG21__stream_id_r2_q_MASK 0x400000
+#define VGT_DEBUG_REG21__stream_id_r2_q__SHIFT 0x16
+#define VGT_DEBUG_REG21__vtx_vect_counter_q_not_0_MASK 0x800000
+#define VGT_DEBUG_REG21__vtx_vect_counter_q_not_0__SHIFT 0x17
+#define VGT_DEBUG_REG21__buff_full_p1_MASK 0x1000000
+#define VGT_DEBUG_REG21__buff_full_p1__SHIFT 0x18
+#define VGT_DEBUG_REG21__strmout_valid_p1_MASK 0x2000000
+#define VGT_DEBUG_REG21__strmout_valid_p1__SHIFT 0x19
+#define VGT_DEBUG_REG21__eotg_r2_q_MASK 0x4000000
+#define VGT_DEBUG_REG21__eotg_r2_q__SHIFT 0x1a
+#define VGT_DEBUG_REG21__null_r2_q_MASK 0x8000000
+#define VGT_DEBUG_REG21__null_r2_q__SHIFT 0x1b
+#define VGT_DEBUG_REG21__p0_dr_MASK 0x10000000
+#define VGT_DEBUG_REG21__p0_dr__SHIFT 0x1c
+#define VGT_DEBUG_REG21__p0_rtr_MASK 0x20000000
+#define VGT_DEBUG_REG21__p0_rtr__SHIFT 0x1d
+#define VGT_DEBUG_REG21__eopg_p0_q_MASK 0x40000000
+#define VGT_DEBUG_REG21__eopg_p0_q__SHIFT 0x1e
+#define VGT_DEBUG_REG21__p0_nobp_MASK 0x80000000
+#define VGT_DEBUG_REG21__p0_nobp__SHIFT 0x1f
+#define VGT_DEBUG_REG22__cm_state16_MASK 0x3
+#define VGT_DEBUG_REG22__cm_state16__SHIFT 0x0
+#define VGT_DEBUG_REG22__cm_state17_MASK 0xc
+#define VGT_DEBUG_REG22__cm_state17__SHIFT 0x2
+#define VGT_DEBUG_REG22__cm_state18_MASK 0x30
+#define VGT_DEBUG_REG22__cm_state18__SHIFT 0x4
+#define VGT_DEBUG_REG22__cm_state19_MASK 0xc0
+#define VGT_DEBUG_REG22__cm_state19__SHIFT 0x6
+#define VGT_DEBUG_REG22__cm_state20_MASK 0x300
+#define VGT_DEBUG_REG22__cm_state20__SHIFT 0x8
+#define VGT_DEBUG_REG22__cm_state21_MASK 0xc00
+#define VGT_DEBUG_REG22__cm_state21__SHIFT 0xa
+#define VGT_DEBUG_REG22__cm_state22_MASK 0x3000
+#define VGT_DEBUG_REG22__cm_state22__SHIFT 0xc
+#define VGT_DEBUG_REG22__cm_state23_MASK 0xc000
+#define VGT_DEBUG_REG22__cm_state23__SHIFT 0xe
+#define VGT_DEBUG_REG22__cm_state24_MASK 0x30000
+#define VGT_DEBUG_REG22__cm_state24__SHIFT 0x10
+#define VGT_DEBUG_REG22__cm_state25_MASK 0xc0000
+#define VGT_DEBUG_REG22__cm_state25__SHIFT 0x12
+#define VGT_DEBUG_REG22__cm_state26_MASK 0x300000
+#define VGT_DEBUG_REG22__cm_state26__SHIFT 0x14
+#define VGT_DEBUG_REG22__cm_state27_MASK 0xc00000
+#define VGT_DEBUG_REG22__cm_state27__SHIFT 0x16
+#define VGT_DEBUG_REG22__cm_state28_MASK 0x3000000
+#define VGT_DEBUG_REG22__cm_state28__SHIFT 0x18
+#define VGT_DEBUG_REG22__cm_state29_MASK 0xc000000
+#define VGT_DEBUG_REG22__cm_state29__SHIFT 0x1a
+#define VGT_DEBUG_REG22__cm_state30_MASK 0x30000000
+#define VGT_DEBUG_REG22__cm_state30__SHIFT 0x1c
+#define VGT_DEBUG_REG22__cm_state31_MASK 0xc0000000
+#define VGT_DEBUG_REG22__cm_state31__SHIFT 0x1e
+#define VGT_DEBUG_REG23__frmt_busy_MASK 0x1
+#define VGT_DEBUG_REG23__frmt_busy__SHIFT 0x0
+#define VGT_DEBUG_REG23__rcm_frmt_vert_rtr_MASK 0x2
+#define VGT_DEBUG_REG23__rcm_frmt_vert_rtr__SHIFT 0x1
+#define VGT_DEBUG_REG23__rcm_frmt_prim_rtr_MASK 0x4
+#define VGT_DEBUG_REG23__rcm_frmt_prim_rtr__SHIFT 0x2
+#define VGT_DEBUG_REG23__prim_r3_rtr_MASK 0x8
+#define VGT_DEBUG_REG23__prim_r3_rtr__SHIFT 0x3
+#define VGT_DEBUG_REG23__prim_r2_rtr_MASK 0x10
+#define VGT_DEBUG_REG23__prim_r2_rtr__SHIFT 0x4
+#define VGT_DEBUG_REG23__vert_r3_rtr_MASK 0x20
+#define VGT_DEBUG_REG23__vert_r3_rtr__SHIFT 0x5
+#define VGT_DEBUG_REG23__vert_r2_rtr_MASK 0x40
+#define VGT_DEBUG_REG23__vert_r2_rtr__SHIFT 0x6
+#define VGT_DEBUG_REG23__vert_r1_rtr_MASK 0x80
+#define VGT_DEBUG_REG23__vert_r1_rtr__SHIFT 0x7
+#define VGT_DEBUG_REG23__vert_r0_rtr_MASK 0x100
+#define VGT_DEBUG_REG23__vert_r0_rtr__SHIFT 0x8
+#define VGT_DEBUG_REG23__prim_fifo_empty_MASK 0x200
+#define VGT_DEBUG_REG23__prim_fifo_empty__SHIFT 0x9
+#define VGT_DEBUG_REG23__prim_fifo_full_MASK 0x400
+#define VGT_DEBUG_REG23__prim_fifo_full__SHIFT 0xa
+#define VGT_DEBUG_REG23__vert_dr_r2_q_MASK 0x800
+#define VGT_DEBUG_REG23__vert_dr_r2_q__SHIFT 0xb
+#define VGT_DEBUG_REG23__prim_dr_r2_q_MASK 0x1000
+#define VGT_DEBUG_REG23__prim_dr_r2_q__SHIFT 0xc
+#define VGT_DEBUG_REG23__vert_dr_r1_q_MASK 0x2000
+#define VGT_DEBUG_REG23__vert_dr_r1_q__SHIFT 0xd
+#define VGT_DEBUG_REG23__vert_dr_r0_q_MASK 0x4000
+#define VGT_DEBUG_REG23__vert_dr_r0_q__SHIFT 0xe
+#define VGT_DEBUG_REG23__new_verts_r2_q_MASK 0x18000
+#define VGT_DEBUG_REG23__new_verts_r2_q__SHIFT 0xf
+#define VGT_DEBUG_REG23__verts_sent_r2_q_MASK 0x1e0000
+#define VGT_DEBUG_REG23__verts_sent_r2_q__SHIFT 0x11
+#define VGT_DEBUG_REG23__prim_state_sel_r2_q_MASK 0xe00000
+#define VGT_DEBUG_REG23__prim_state_sel_r2_q__SHIFT 0x15
+#define VGT_DEBUG_REG23__SPARE_MASK 0xff000000
+#define VGT_DEBUG_REG23__SPARE__SHIFT 0x18
+#define VGT_DEBUG_REG24__avail_es_rb_space_r0_q_23_0_MASK 0xffffff
+#define VGT_DEBUG_REG24__avail_es_rb_space_r0_q_23_0__SHIFT 0x0
+#define VGT_DEBUG_REG24__dependent_st_cut_mode_q_MASK 0x3000000
+#define VGT_DEBUG_REG24__dependent_st_cut_mode_q__SHIFT 0x18
+#define VGT_DEBUG_REG24__SPARE31_MASK 0xfc000000
+#define VGT_DEBUG_REG24__SPARE31__SHIFT 0x1a
+#define VGT_DEBUG_REG25__avail_gs_rb_space_r0_q_25_0_MASK 0x3ffffff
+#define VGT_DEBUG_REG25__avail_gs_rb_space_r0_q_25_0__SHIFT 0x0
+#define VGT_DEBUG_REG25__active_sm_r0_q_MASK 0x3c000000
+#define VGT_DEBUG_REG25__active_sm_r0_q__SHIFT 0x1a
+#define VGT_DEBUG_REG25__add_gs_rb_space_r1_q_MASK 0x40000000
+#define VGT_DEBUG_REG25__add_gs_rb_space_r1_q__SHIFT 0x1e
+#define VGT_DEBUG_REG25__add_gs_rb_space_r0_q_MASK 0x80000000
+#define VGT_DEBUG_REG25__add_gs_rb_space_r0_q__SHIFT 0x1f
+#define VGT_DEBUG_REG26__cm_state0_MASK 0x3
+#define VGT_DEBUG_REG26__cm_state0__SHIFT 0x0
+#define VGT_DEBUG_REG26__cm_state1_MASK 0xc
+#define VGT_DEBUG_REG26__cm_state1__SHIFT 0x2
+#define VGT_DEBUG_REG26__cm_state2_MASK 0x30
+#define VGT_DEBUG_REG26__cm_state2__SHIFT 0x4
+#define VGT_DEBUG_REG26__cm_state3_MASK 0xc0
+#define VGT_DEBUG_REG26__cm_state3__SHIFT 0x6
+#define VGT_DEBUG_REG26__cm_state4_MASK 0x300
+#define VGT_DEBUG_REG26__cm_state4__SHIFT 0x8
+#define VGT_DEBUG_REG26__cm_state5_MASK 0xc00
+#define VGT_DEBUG_REG26__cm_state5__SHIFT 0xa
+#define VGT_DEBUG_REG26__cm_state6_MASK 0x3000
+#define VGT_DEBUG_REG26__cm_state6__SHIFT 0xc
+#define VGT_DEBUG_REG26__cm_state7_MASK 0xc000
+#define VGT_DEBUG_REG26__cm_state7__SHIFT 0xe
+#define VGT_DEBUG_REG26__cm_state8_MASK 0x30000
+#define VGT_DEBUG_REG26__cm_state8__SHIFT 0x10
+#define VGT_DEBUG_REG26__cm_state9_MASK 0xc0000
+#define VGT_DEBUG_REG26__cm_state9__SHIFT 0x12
+#define VGT_DEBUG_REG26__cm_state10_MASK 0x300000
+#define VGT_DEBUG_REG26__cm_state10__SHIFT 0x14
+#define VGT_DEBUG_REG26__cm_state11_MASK 0xc00000
+#define VGT_DEBUG_REG26__cm_state11__SHIFT 0x16
+#define VGT_DEBUG_REG26__cm_state12_MASK 0x3000000
+#define VGT_DEBUG_REG26__cm_state12__SHIFT 0x18
+#define VGT_DEBUG_REG26__cm_state13_MASK 0xc000000
+#define VGT_DEBUG_REG26__cm_state13__SHIFT 0x1a
+#define VGT_DEBUG_REG26__cm_state14_MASK 0x30000000
+#define VGT_DEBUG_REG26__cm_state14__SHIFT 0x1c
+#define VGT_DEBUG_REG26__cm_state15_MASK 0xc0000000
+#define VGT_DEBUG_REG26__cm_state15__SHIFT 0x1e
+#define VGT_DEBUG_REG27__pipe0_dr_MASK 0x1
+#define VGT_DEBUG_REG27__pipe0_dr__SHIFT 0x0
+#define VGT_DEBUG_REG27__gsc0_dr_MASK 0x2
+#define VGT_DEBUG_REG27__gsc0_dr__SHIFT 0x1
+#define VGT_DEBUG_REG27__pipe1_dr_MASK 0x4
+#define VGT_DEBUG_REG27__pipe1_dr__SHIFT 0x2
+#define VGT_DEBUG_REG27__tm_pt_event_rtr_MASK 0x8
+#define VGT_DEBUG_REG27__tm_pt_event_rtr__SHIFT 0x3
+#define VGT_DEBUG_REG27__pipe0_rtr_MASK 0x10
+#define VGT_DEBUG_REG27__pipe0_rtr__SHIFT 0x4
+#define VGT_DEBUG_REG27__gsc0_rtr_MASK 0x20
+#define VGT_DEBUG_REG27__gsc0_rtr__SHIFT 0x5
+#define VGT_DEBUG_REG27__pipe1_rtr_MASK 0x40
+#define VGT_DEBUG_REG27__pipe1_rtr__SHIFT 0x6
+#define VGT_DEBUG_REG27__last_indx_of_prim_p1_q_MASK 0x80
+#define VGT_DEBUG_REG27__last_indx_of_prim_p1_q__SHIFT 0x7
+#define VGT_DEBUG_REG27__indices_to_send_p0_q_MASK 0x300
+#define VGT_DEBUG_REG27__indices_to_send_p0_q__SHIFT 0x8
+#define VGT_DEBUG_REG27__event_flag_p1_q_MASK 0x400
+#define VGT_DEBUG_REG27__event_flag_p1_q__SHIFT 0xa
+#define VGT_DEBUG_REG27__eop_p1_q_MASK 0x800
+#define VGT_DEBUG_REG27__eop_p1_q__SHIFT 0xb
+#define VGT_DEBUG_REG27__gs_out_prim_type_p0_q_MASK 0x3000
+#define VGT_DEBUG_REG27__gs_out_prim_type_p0_q__SHIFT 0xc
+#define VGT_DEBUG_REG27__gsc_null_primitive_p0_q_MASK 0x4000
+#define VGT_DEBUG_REG27__gsc_null_primitive_p0_q__SHIFT 0xe
+#define VGT_DEBUG_REG27__gsc_eop_p0_q_MASK 0x8000
+#define VGT_DEBUG_REG27__gsc_eop_p0_q__SHIFT 0xf
+#define VGT_DEBUG_REG27__gsc_2cycle_output_MASK 0x10000
+#define VGT_DEBUG_REG27__gsc_2cycle_output__SHIFT 0x10
+#define VGT_DEBUG_REG27__gsc_2nd_cycle_p0_q_MASK 0x20000
+#define VGT_DEBUG_REG27__gsc_2nd_cycle_p0_q__SHIFT 0x11
+#define VGT_DEBUG_REG27__last_indx_of_vsprim_MASK 0x40000
+#define VGT_DEBUG_REG27__last_indx_of_vsprim__SHIFT 0x12
+#define VGT_DEBUG_REG27__first_vsprim_of_gsprim_p0_q_MASK 0x80000
+#define VGT_DEBUG_REG27__first_vsprim_of_gsprim_p0_q__SHIFT 0x13
+#define VGT_DEBUG_REG27__gsc_indx_count_p0_q_MASK 0x7ff00000
+#define VGT_DEBUG_REG27__gsc_indx_count_p0_q__SHIFT 0x14
+#define VGT_DEBUG_REG27__last_vsprim_of_gsprim_MASK 0x80000000
+#define VGT_DEBUG_REG27__last_vsprim_of_gsprim__SHIFT 0x1f
+#define VGT_DEBUG_REG28__con_state_q_MASK 0xf
+#define VGT_DEBUG_REG28__con_state_q__SHIFT 0x0
+#define VGT_DEBUG_REG28__second_cycle_q_MASK 0x10
+#define VGT_DEBUG_REG28__second_cycle_q__SHIFT 0x4
+#define VGT_DEBUG_REG28__process_tri_middle_p0_q_MASK 0x20
+#define VGT_DEBUG_REG28__process_tri_middle_p0_q__SHIFT 0x5
+#define VGT_DEBUG_REG28__process_tri_1st_2nd_half_p0_q_MASK 0x40
+#define VGT_DEBUG_REG28__process_tri_1st_2nd_half_p0_q__SHIFT 0x6
+#define VGT_DEBUG_REG28__process_tri_center_poly_p0_q_MASK 0x80
+#define VGT_DEBUG_REG28__process_tri_center_poly_p0_q__SHIFT 0x7
+#define VGT_DEBUG_REG28__pipe0_patch_dr_MASK 0x100
+#define VGT_DEBUG_REG28__pipe0_patch_dr__SHIFT 0x8
+#define VGT_DEBUG_REG28__pipe0_edge_dr_MASK 0x200
+#define VGT_DEBUG_REG28__pipe0_edge_dr__SHIFT 0x9
+#define VGT_DEBUG_REG28__pipe1_dr_MASK 0x400
+#define VGT_DEBUG_REG28__pipe1_dr__SHIFT 0xa
+#define VGT_DEBUG_REG28__pipe0_patch_rtr_MASK 0x800
+#define VGT_DEBUG_REG28__pipe0_patch_rtr__SHIFT 0xb
+#define VGT_DEBUG_REG28__pipe0_edge_rtr_MASK 0x1000
+#define VGT_DEBUG_REG28__pipe0_edge_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG28__pipe1_rtr_MASK 0x2000
+#define VGT_DEBUG_REG28__pipe1_rtr__SHIFT 0xd
+#define VGT_DEBUG_REG28__outer_parity_p0_q_MASK 0x4000
+#define VGT_DEBUG_REG28__outer_parity_p0_q__SHIFT 0xe
+#define VGT_DEBUG_REG28__parallel_parity_p0_q_MASK 0x8000
+#define VGT_DEBUG_REG28__parallel_parity_p0_q__SHIFT 0xf
+#define VGT_DEBUG_REG28__first_ring_of_patch_p0_q_MASK 0x10000
+#define VGT_DEBUG_REG28__first_ring_of_patch_p0_q__SHIFT 0x10
+#define VGT_DEBUG_REG28__last_ring_of_patch_p0_q_MASK 0x20000
+#define VGT_DEBUG_REG28__last_ring_of_patch_p0_q__SHIFT 0x11
+#define VGT_DEBUG_REG28__last_edge_of_outer_ring_p0_q_MASK 0x40000
+#define VGT_DEBUG_REG28__last_edge_of_outer_ring_p0_q__SHIFT 0x12
+#define VGT_DEBUG_REG28__last_point_of_outer_ring_p1_MASK 0x80000
+#define VGT_DEBUG_REG28__last_point_of_outer_ring_p1__SHIFT 0x13
+#define VGT_DEBUG_REG28__last_point_of_inner_ring_p1_MASK 0x100000
+#define VGT_DEBUG_REG28__last_point_of_inner_ring_p1__SHIFT 0x14
+#define VGT_DEBUG_REG28__outer_edge_tf_eq_one_p0_q_MASK 0x200000
+#define VGT_DEBUG_REG28__outer_edge_tf_eq_one_p0_q__SHIFT 0x15
+#define VGT_DEBUG_REG28__advance_outer_point_p1_MASK 0x400000
+#define VGT_DEBUG_REG28__advance_outer_point_p1__SHIFT 0x16
+#define VGT_DEBUG_REG28__advance_inner_point_p1_MASK 0x800000
+#define VGT_DEBUG_REG28__advance_inner_point_p1__SHIFT 0x17
+#define VGT_DEBUG_REG28__next_ring_is_rect_p0_q_MASK 0x1000000
+#define VGT_DEBUG_REG28__next_ring_is_rect_p0_q__SHIFT 0x18
+#define VGT_DEBUG_REG28__pipe1_outer1_rtr_MASK 0x2000000
+#define VGT_DEBUG_REG28__pipe1_outer1_rtr__SHIFT 0x19
+#define VGT_DEBUG_REG28__pipe1_outer2_rtr_MASK 0x4000000
+#define VGT_DEBUG_REG28__pipe1_outer2_rtr__SHIFT 0x1a
+#define VGT_DEBUG_REG28__pipe1_inner1_rtr_MASK 0x8000000
+#define VGT_DEBUG_REG28__pipe1_inner1_rtr__SHIFT 0x1b
+#define VGT_DEBUG_REG28__pipe1_inner2_rtr_MASK 0x10000000
+#define VGT_DEBUG_REG28__pipe1_inner2_rtr__SHIFT 0x1c
+#define VGT_DEBUG_REG28__pipe1_patch_rtr_MASK 0x20000000
+#define VGT_DEBUG_REG28__pipe1_patch_rtr__SHIFT 0x1d
+#define VGT_DEBUG_REG28__pipe1_edge_rtr_MASK 0x40000000
+#define VGT_DEBUG_REG28__pipe1_edge_rtr__SHIFT 0x1e
+#define VGT_DEBUG_REG28__use_stored_inner_q_ring2_MASK 0x80000000
+#define VGT_DEBUG_REG28__use_stored_inner_q_ring2__SHIFT 0x1f
+#define VGT_DEBUG_REG29__con_state_q_MASK 0xf
+#define VGT_DEBUG_REG29__con_state_q__SHIFT 0x0
+#define VGT_DEBUG_REG29__second_cycle_q_MASK 0x10
+#define VGT_DEBUG_REG29__second_cycle_q__SHIFT 0x4
+#define VGT_DEBUG_REG29__process_tri_middle_p0_q_MASK 0x20
+#define VGT_DEBUG_REG29__process_tri_middle_p0_q__SHIFT 0x5
+#define VGT_DEBUG_REG29__process_tri_1st_2nd_half_p0_q_MASK 0x40
+#define VGT_DEBUG_REG29__process_tri_1st_2nd_half_p0_q__SHIFT 0x6
+#define VGT_DEBUG_REG29__process_tri_center_poly_p0_q_MASK 0x80
+#define VGT_DEBUG_REG29__process_tri_center_poly_p0_q__SHIFT 0x7
+#define VGT_DEBUG_REG29__pipe0_patch_dr_MASK 0x100
+#define VGT_DEBUG_REG29__pipe0_patch_dr__SHIFT 0x8
+#define VGT_DEBUG_REG29__pipe0_edge_dr_MASK 0x200
+#define VGT_DEBUG_REG29__pipe0_edge_dr__SHIFT 0x9
+#define VGT_DEBUG_REG29__pipe1_dr_MASK 0x400
+#define VGT_DEBUG_REG29__pipe1_dr__SHIFT 0xa
+#define VGT_DEBUG_REG29__pipe0_patch_rtr_MASK 0x800
+#define VGT_DEBUG_REG29__pipe0_patch_rtr__SHIFT 0xb
+#define VGT_DEBUG_REG29__pipe0_edge_rtr_MASK 0x1000
+#define VGT_DEBUG_REG29__pipe0_edge_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG29__pipe1_rtr_MASK 0x2000
+#define VGT_DEBUG_REG29__pipe1_rtr__SHIFT 0xd
+#define VGT_DEBUG_REG29__outer_parity_p0_q_MASK 0x4000
+#define VGT_DEBUG_REG29__outer_parity_p0_q__SHIFT 0xe
+#define VGT_DEBUG_REG29__parallel_parity_p0_q_MASK 0x8000
+#define VGT_DEBUG_REG29__parallel_parity_p0_q__SHIFT 0xf
+#define VGT_DEBUG_REG29__first_ring_of_patch_p0_q_MASK 0x10000
+#define VGT_DEBUG_REG29__first_ring_of_patch_p0_q__SHIFT 0x10
+#define VGT_DEBUG_REG29__last_ring_of_patch_p0_q_MASK 0x20000
+#define VGT_DEBUG_REG29__last_ring_of_patch_p0_q__SHIFT 0x11
+#define VGT_DEBUG_REG29__last_edge_of_outer_ring_p0_q_MASK 0x40000
+#define VGT_DEBUG_REG29__last_edge_of_outer_ring_p0_q__SHIFT 0x12
+#define VGT_DEBUG_REG29__last_point_of_outer_ring_p1_MASK 0x80000
+#define VGT_DEBUG_REG29__last_point_of_outer_ring_p1__SHIFT 0x13
+#define VGT_DEBUG_REG29__last_point_of_inner_ring_p1_MASK 0x100000
+#define VGT_DEBUG_REG29__last_point_of_inner_ring_p1__SHIFT 0x14
+#define VGT_DEBUG_REG29__outer_edge_tf_eq_one_p0_q_MASK 0x200000
+#define VGT_DEBUG_REG29__outer_edge_tf_eq_one_p0_q__SHIFT 0x15
+#define VGT_DEBUG_REG29__advance_outer_point_p1_MASK 0x400000
+#define VGT_DEBUG_REG29__advance_outer_point_p1__SHIFT 0x16
+#define VGT_DEBUG_REG29__advance_inner_point_p1_MASK 0x800000
+#define VGT_DEBUG_REG29__advance_inner_point_p1__SHIFT 0x17
+#define VGT_DEBUG_REG29__next_ring_is_rect_p0_q_MASK 0x1000000
+#define VGT_DEBUG_REG29__next_ring_is_rect_p0_q__SHIFT 0x18
+#define VGT_DEBUG_REG29__pipe1_outer1_rtr_MASK 0x2000000
+#define VGT_DEBUG_REG29__pipe1_outer1_rtr__SHIFT 0x19
+#define VGT_DEBUG_REG29__pipe1_outer2_rtr_MASK 0x4000000
+#define VGT_DEBUG_REG29__pipe1_outer2_rtr__SHIFT 0x1a
+#define VGT_DEBUG_REG29__pipe1_inner1_rtr_MASK 0x8000000
+#define VGT_DEBUG_REG29__pipe1_inner1_rtr__SHIFT 0x1b
+#define VGT_DEBUG_REG29__pipe1_inner2_rtr_MASK 0x10000000
+#define VGT_DEBUG_REG29__pipe1_inner2_rtr__SHIFT 0x1c
+#define VGT_DEBUG_REG29__pipe1_patch_rtr_MASK 0x20000000
+#define VGT_DEBUG_REG29__pipe1_patch_rtr__SHIFT 0x1d
+#define VGT_DEBUG_REG29__pipe1_edge_rtr_MASK 0x40000000
+#define VGT_DEBUG_REG29__pipe1_edge_rtr__SHIFT 0x1e
+#define VGT_DEBUG_REG29__use_stored_inner_q_ring3_MASK 0x80000000
+#define VGT_DEBUG_REG29__use_stored_inner_q_ring3__SHIFT 0x1f
+#define VGT_DEBUG_REG31__pipe0_dr_MASK 0x1
+#define VGT_DEBUG_REG31__pipe0_dr__SHIFT 0x0
+#define VGT_DEBUG_REG31__pipe0_rtr_MASK 0x2
+#define VGT_DEBUG_REG31__pipe0_rtr__SHIFT 0x1
+#define VGT_DEBUG_REG31__pipe1_outer_dr_MASK 0x4
+#define VGT_DEBUG_REG31__pipe1_outer_dr__SHIFT 0x2
+#define VGT_DEBUG_REG31__pipe1_inner_dr_MASK 0x8
+#define VGT_DEBUG_REG31__pipe1_inner_dr__SHIFT 0x3
+#define VGT_DEBUG_REG31__pipe2_outer_dr_MASK 0x10
+#define VGT_DEBUG_REG31__pipe2_outer_dr__SHIFT 0x4
+#define VGT_DEBUG_REG31__pipe2_inner_dr_MASK 0x20
+#define VGT_DEBUG_REG31__pipe2_inner_dr__SHIFT 0x5
+#define VGT_DEBUG_REG31__pipe3_outer_dr_MASK 0x40
+#define VGT_DEBUG_REG31__pipe3_outer_dr__SHIFT 0x6
+#define VGT_DEBUG_REG31__pipe3_inner_dr_MASK 0x80
+#define VGT_DEBUG_REG31__pipe3_inner_dr__SHIFT 0x7
+#define VGT_DEBUG_REG31__pipe4_outer_dr_MASK 0x100
+#define VGT_DEBUG_REG31__pipe4_outer_dr__SHIFT 0x8
+#define VGT_DEBUG_REG31__pipe4_inner_dr_MASK 0x200
+#define VGT_DEBUG_REG31__pipe4_inner_dr__SHIFT 0x9
+#define VGT_DEBUG_REG31__pipe5_outer_dr_MASK 0x400
+#define VGT_DEBUG_REG31__pipe5_outer_dr__SHIFT 0xa
+#define VGT_DEBUG_REG31__pipe5_inner_dr_MASK 0x800
+#define VGT_DEBUG_REG31__pipe5_inner_dr__SHIFT 0xb
+#define VGT_DEBUG_REG31__pipe2_outer_rtr_MASK 0x1000
+#define VGT_DEBUG_REG31__pipe2_outer_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG31__pipe2_inner_rtr_MASK 0x2000
+#define VGT_DEBUG_REG31__pipe2_inner_rtr__SHIFT 0xd
+#define VGT_DEBUG_REG31__pipe3_outer_rtr_MASK 0x4000
+#define VGT_DEBUG_REG31__pipe3_outer_rtr__SHIFT 0xe
+#define VGT_DEBUG_REG31__pipe3_inner_rtr_MASK 0x8000
+#define VGT_DEBUG_REG31__pipe3_inner_rtr__SHIFT 0xf
+#define VGT_DEBUG_REG31__pipe4_outer_rtr_MASK 0x10000
+#define VGT_DEBUG_REG31__pipe4_outer_rtr__SHIFT 0x10
+#define VGT_DEBUG_REG31__pipe4_inner_rtr_MASK 0x20000
+#define VGT_DEBUG_REG31__pipe4_inner_rtr__SHIFT 0x11
+#define VGT_DEBUG_REG31__pipe5_outer_rtr_MASK 0x40000
+#define VGT_DEBUG_REG31__pipe5_outer_rtr__SHIFT 0x12
+#define VGT_DEBUG_REG31__pipe5_inner_rtr_MASK 0x80000
+#define VGT_DEBUG_REG31__pipe5_inner_rtr__SHIFT 0x13
+#define VGT_DEBUG_REG31__pg_con_outer_point1_rts_MASK 0x100000
+#define VGT_DEBUG_REG31__pg_con_outer_point1_rts__SHIFT 0x14
+#define VGT_DEBUG_REG31__pg_con_outer_point2_rts_MASK 0x200000
+#define VGT_DEBUG_REG31__pg_con_outer_point2_rts__SHIFT 0x15
+#define VGT_DEBUG_REG31__pg_con_inner_point1_rts_MASK 0x400000
+#define VGT_DEBUG_REG31__pg_con_inner_point1_rts__SHIFT 0x16
+#define VGT_DEBUG_REG31__pg_con_inner_point2_rts_MASK 0x800000
+#define VGT_DEBUG_REG31__pg_con_inner_point2_rts__SHIFT 0x17
+#define VGT_DEBUG_REG31__pg_patch_fifo_empty_MASK 0x1000000
+#define VGT_DEBUG_REG31__pg_patch_fifo_empty__SHIFT 0x18
+#define VGT_DEBUG_REG31__pg_edge_fifo_empty_MASK 0x2000000
+#define VGT_DEBUG_REG31__pg_edge_fifo_empty__SHIFT 0x19
+#define VGT_DEBUG_REG31__pg_inner3_perp_fifo_empty_MASK 0x4000000
+#define VGT_DEBUG_REG31__pg_inner3_perp_fifo_empty__SHIFT 0x1a
+#define VGT_DEBUG_REG31__pg_patch_fifo_full_MASK 0x8000000
+#define VGT_DEBUG_REG31__pg_patch_fifo_full__SHIFT 0x1b
+#define VGT_DEBUG_REG31__pg_edge_fifo_full_MASK 0x10000000
+#define VGT_DEBUG_REG31__pg_edge_fifo_full__SHIFT 0x1c
+#define VGT_DEBUG_REG31__pg_inner_perp_fifo_full_MASK 0x20000000
+#define VGT_DEBUG_REG31__pg_inner_perp_fifo_full__SHIFT 0x1d
+#define VGT_DEBUG_REG31__outer_ring_done_q_MASK 0x40000000
+#define VGT_DEBUG_REG31__outer_ring_done_q__SHIFT 0x1e
+#define VGT_DEBUG_REG31__inner_ring_done_q_MASK 0x80000000
+#define VGT_DEBUG_REG31__inner_ring_done_q__SHIFT 0x1f
+#define VGT_DEBUG_REG32__first_ring_of_patch_MASK 0x1
+#define VGT_DEBUG_REG32__first_ring_of_patch__SHIFT 0x0
+#define VGT_DEBUG_REG32__last_ring_of_patch_MASK 0x2
+#define VGT_DEBUG_REG32__last_ring_of_patch__SHIFT 0x1
+#define VGT_DEBUG_REG32__last_edge_of_outer_ring_MASK 0x4
+#define VGT_DEBUG_REG32__last_edge_of_outer_ring__SHIFT 0x2
+#define VGT_DEBUG_REG32__last_point_of_outer_edge_MASK 0x8
+#define VGT_DEBUG_REG32__last_point_of_outer_edge__SHIFT 0x3
+#define VGT_DEBUG_REG32__last_edge_of_inner_ring_MASK 0x10
+#define VGT_DEBUG_REG32__last_edge_of_inner_ring__SHIFT 0x4
+#define VGT_DEBUG_REG32__last_point_of_inner_edge_MASK 0x20
+#define VGT_DEBUG_REG32__last_point_of_inner_edge__SHIFT 0x5
+#define VGT_DEBUG_REG32__last_patch_of_tg_p0_q_MASK 0x40
+#define VGT_DEBUG_REG32__last_patch_of_tg_p0_q__SHIFT 0x6
+#define VGT_DEBUG_REG32__event_null_special_p0_q_MASK 0x80
+#define VGT_DEBUG_REG32__event_null_special_p0_q__SHIFT 0x7
+#define VGT_DEBUG_REG32__event_flag_p5_q_MASK 0x100
+#define VGT_DEBUG_REG32__event_flag_p5_q__SHIFT 0x8
+#define VGT_DEBUG_REG32__first_point_of_patch_p5_q_MASK 0x200
+#define VGT_DEBUG_REG32__first_point_of_patch_p5_q__SHIFT 0x9
+#define VGT_DEBUG_REG32__first_point_of_edge_p5_q_MASK 0x400
+#define VGT_DEBUG_REG32__first_point_of_edge_p5_q__SHIFT 0xa
+#define VGT_DEBUG_REG32__last_patch_of_tg_p5_q_MASK 0x800
+#define VGT_DEBUG_REG32__last_patch_of_tg_p5_q__SHIFT 0xb
+#define VGT_DEBUG_REG32__tess_topology_p5_q_MASK 0x3000
+#define VGT_DEBUG_REG32__tess_topology_p5_q__SHIFT 0xc
+#define VGT_DEBUG_REG32__pipe5_inner3_rtr_MASK 0x4000
+#define VGT_DEBUG_REG32__pipe5_inner3_rtr__SHIFT 0xe
+#define VGT_DEBUG_REG32__pipe5_inner2_rtr_MASK 0x8000
+#define VGT_DEBUG_REG32__pipe5_inner2_rtr__SHIFT 0xf
+#define VGT_DEBUG_REG32__pg_edge_fifo3_full_MASK 0x10000
+#define VGT_DEBUG_REG32__pg_edge_fifo3_full__SHIFT 0x10
+#define VGT_DEBUG_REG32__pg_edge_fifo2_full_MASK 0x20000
+#define VGT_DEBUG_REG32__pg_edge_fifo2_full__SHIFT 0x11
+#define VGT_DEBUG_REG32__pg_inner3_point_fifo_full_MASK 0x40000
+#define VGT_DEBUG_REG32__pg_inner3_point_fifo_full__SHIFT 0x12
+#define VGT_DEBUG_REG32__pg_outer3_point_fifo_full_MASK 0x80000
+#define VGT_DEBUG_REG32__pg_outer3_point_fifo_full__SHIFT 0x13
+#define VGT_DEBUG_REG32__pg_inner2_point_fifo_full_MASK 0x100000
+#define VGT_DEBUG_REG32__pg_inner2_point_fifo_full__SHIFT 0x14
+#define VGT_DEBUG_REG32__pg_outer2_point_fifo_full_MASK 0x200000
+#define VGT_DEBUG_REG32__pg_outer2_point_fifo_full__SHIFT 0x15
+#define VGT_DEBUG_REG32__pg_inner_point_fifo_full_MASK 0x400000
+#define VGT_DEBUG_REG32__pg_inner_point_fifo_full__SHIFT 0x16
+#define VGT_DEBUG_REG32__pg_outer_point_fifo_full_MASK 0x800000
+#define VGT_DEBUG_REG32__pg_outer_point_fifo_full__SHIFT 0x17
+#define VGT_DEBUG_REG32__inner2_fifos_rtr_MASK 0x1000000
+#define VGT_DEBUG_REG32__inner2_fifos_rtr__SHIFT 0x18
+#define VGT_DEBUG_REG32__inner_fifos_rtr_MASK 0x2000000
+#define VGT_DEBUG_REG32__inner_fifos_rtr__SHIFT 0x19
+#define VGT_DEBUG_REG32__outer_fifos_rtr_MASK 0x4000000
+#define VGT_DEBUG_REG32__outer_fifos_rtr__SHIFT 0x1a
+#define VGT_DEBUG_REG32__fifos_rtr_MASK 0x8000000
+#define VGT_DEBUG_REG32__fifos_rtr__SHIFT 0x1b
+#define VGT_DEBUG_REG32__SPARE_MASK 0xf0000000
+#define VGT_DEBUG_REG32__SPARE__SHIFT 0x1c
+#define VGT_DEBUG_REG33__pipe0_patch_dr_MASK 0x1
+#define VGT_DEBUG_REG33__pipe0_patch_dr__SHIFT 0x0
+#define VGT_DEBUG_REG33__ring3_pipe1_dr_MASK 0x2
+#define VGT_DEBUG_REG33__ring3_pipe1_dr__SHIFT 0x1
+#define VGT_DEBUG_REG33__pipe1_dr_MASK 0x4
+#define VGT_DEBUG_REG33__pipe1_dr__SHIFT 0x2
+#define VGT_DEBUG_REG33__pipe2_dr_MASK 0x8
+#define VGT_DEBUG_REG33__pipe2_dr__SHIFT 0x3
+#define VGT_DEBUG_REG33__pipe0_patch_rtr_MASK 0x10
+#define VGT_DEBUG_REG33__pipe0_patch_rtr__SHIFT 0x4
+#define VGT_DEBUG_REG33__ring2_pipe1_dr_MASK 0x20
+#define VGT_DEBUG_REG33__ring2_pipe1_dr__SHIFT 0x5
+#define VGT_DEBUG_REG33__ring1_pipe1_dr_MASK 0x40
+#define VGT_DEBUG_REG33__ring1_pipe1_dr__SHIFT 0x6
+#define VGT_DEBUG_REG33__pipe2_rtr_MASK 0x80
+#define VGT_DEBUG_REG33__pipe2_rtr__SHIFT 0x7
+#define VGT_DEBUG_REG33__pipe3_dr_MASK 0x100
+#define VGT_DEBUG_REG33__pipe3_dr__SHIFT 0x8
+#define VGT_DEBUG_REG33__pipe3_rtr_MASK 0x200
+#define VGT_DEBUG_REG33__pipe3_rtr__SHIFT 0x9
+#define VGT_DEBUG_REG33__ring2_in_sync_q_MASK 0x400
+#define VGT_DEBUG_REG33__ring2_in_sync_q__SHIFT 0xa
+#define VGT_DEBUG_REG33__ring1_in_sync_q_MASK 0x800
+#define VGT_DEBUG_REG33__ring1_in_sync_q__SHIFT 0xb
+#define VGT_DEBUG_REG33__pipe1_patch_rtr_MASK 0x1000
+#define VGT_DEBUG_REG33__pipe1_patch_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG33__ring3_in_sync_q_MASK 0x2000
+#define VGT_DEBUG_REG33__ring3_in_sync_q__SHIFT 0xd
+#define VGT_DEBUG_REG33__tm_te11_event_rtr_MASK 0x4000
+#define VGT_DEBUG_REG33__tm_te11_event_rtr__SHIFT 0xe
+#define VGT_DEBUG_REG33__first_prim_of_patch_q_MASK 0x8000
+#define VGT_DEBUG_REG33__first_prim_of_patch_q__SHIFT 0xf
+#define VGT_DEBUG_REG33__con_prim_fifo_full_MASK 0x10000
+#define VGT_DEBUG_REG33__con_prim_fifo_full__SHIFT 0x10
+#define VGT_DEBUG_REG33__con_vert_fifo_full_MASK 0x20000
+#define VGT_DEBUG_REG33__con_vert_fifo_full__SHIFT 0x11
+#define VGT_DEBUG_REG33__con_prim_fifo_empty_MASK 0x40000
+#define VGT_DEBUG_REG33__con_prim_fifo_empty__SHIFT 0x12
+#define VGT_DEBUG_REG33__con_vert_fifo_empty_MASK 0x80000
+#define VGT_DEBUG_REG33__con_vert_fifo_empty__SHIFT 0x13
+#define VGT_DEBUG_REG33__last_patch_of_tg_p0_q_MASK 0x100000
+#define VGT_DEBUG_REG33__last_patch_of_tg_p0_q__SHIFT 0x14
+#define VGT_DEBUG_REG33__ring3_valid_p2_MASK 0x200000
+#define VGT_DEBUG_REG33__ring3_valid_p2__SHIFT 0x15
+#define VGT_DEBUG_REG33__ring2_valid_p2_MASK 0x400000
+#define VGT_DEBUG_REG33__ring2_valid_p2__SHIFT 0x16
+#define VGT_DEBUG_REG33__ring1_valid_p2_MASK 0x800000
+#define VGT_DEBUG_REG33__ring1_valid_p2__SHIFT 0x17
+#define VGT_DEBUG_REG33__tess_type_p0_q_MASK 0x3000000
+#define VGT_DEBUG_REG33__tess_type_p0_q__SHIFT 0x18
+#define VGT_DEBUG_REG33__tess_topology_p0_q_MASK 0xc000000
+#define VGT_DEBUG_REG33__tess_topology_p0_q__SHIFT 0x1a
+#define VGT_DEBUG_REG33__te11_out_vert_gs_en_MASK 0x10000000
+#define VGT_DEBUG_REG33__te11_out_vert_gs_en__SHIFT 0x1c
+#define VGT_DEBUG_REG33__con_ring3_busy_MASK 0x20000000
+#define VGT_DEBUG_REG33__con_ring3_busy__SHIFT 0x1d
+#define VGT_DEBUG_REG33__con_ring2_busy_MASK 0x40000000
+#define VGT_DEBUG_REG33__con_ring2_busy__SHIFT 0x1e
+#define VGT_DEBUG_REG33__con_ring1_busy_MASK 0x80000000
+#define VGT_DEBUG_REG33__con_ring1_busy__SHIFT 0x1f
+#define VGT_DEBUG_REG34__con_state_q_MASK 0xf
+#define VGT_DEBUG_REG34__con_state_q__SHIFT 0x0
+#define VGT_DEBUG_REG34__second_cycle_q_MASK 0x10
+#define VGT_DEBUG_REG34__second_cycle_q__SHIFT 0x4
+#define VGT_DEBUG_REG34__process_tri_middle_p0_q_MASK 0x20
+#define VGT_DEBUG_REG34__process_tri_middle_p0_q__SHIFT 0x5
+#define VGT_DEBUG_REG34__process_tri_1st_2nd_half_p0_q_MASK 0x40
+#define VGT_DEBUG_REG34__process_tri_1st_2nd_half_p0_q__SHIFT 0x6
+#define VGT_DEBUG_REG34__process_tri_center_poly_p0_q_MASK 0x80
+#define VGT_DEBUG_REG34__process_tri_center_poly_p0_q__SHIFT 0x7
+#define VGT_DEBUG_REG34__pipe0_patch_dr_MASK 0x100
+#define VGT_DEBUG_REG34__pipe0_patch_dr__SHIFT 0x8
+#define VGT_DEBUG_REG34__pipe0_edge_dr_MASK 0x200
+#define VGT_DEBUG_REG34__pipe0_edge_dr__SHIFT 0x9
+#define VGT_DEBUG_REG34__pipe1_dr_MASK 0x400
+#define VGT_DEBUG_REG34__pipe1_dr__SHIFT 0xa
+#define VGT_DEBUG_REG34__pipe0_patch_rtr_MASK 0x800
+#define VGT_DEBUG_REG34__pipe0_patch_rtr__SHIFT 0xb
+#define VGT_DEBUG_REG34__pipe0_edge_rtr_MASK 0x1000
+#define VGT_DEBUG_REG34__pipe0_edge_rtr__SHIFT 0xc
+#define VGT_DEBUG_REG34__pipe1_rtr_MASK 0x2000
+#define VGT_DEBUG_REG34__pipe1_rtr__SHIFT 0xd
+#define VGT_DEBUG_REG34__outer_parity_p0_q_MASK 0x4000
+#define VGT_DEBUG_REG34__outer_parity_p0_q__SHIFT 0xe
+#define VGT_DEBUG_REG34__parallel_parity_p0_q_MASK 0x8000
+#define VGT_DEBUG_REG34__parallel_parity_p0_q__SHIFT 0xf
+#define VGT_DEBUG_REG34__first_ring_of_patch_p0_q_MASK 0x10000
+#define VGT_DEBUG_REG34__first_ring_of_patch_p0_q__SHIFT 0x10
+#define VGT_DEBUG_REG34__last_ring_of_patch_p0_q_MASK 0x20000
+#define VGT_DEBUG_REG34__last_ring_of_patch_p0_q__SHIFT 0x11
+#define VGT_DEBUG_REG34__last_edge_of_outer_ring_p0_q_MASK 0x40000
+#define VGT_DEBUG_REG34__last_edge_of_outer_ring_p0_q__SHIFT 0x12
+#define VGT_DEBUG_REG34__last_point_of_outer_ring_p1_MASK 0x80000
+#define VGT_DEBUG_REG34__last_point_of_outer_ring_p1__SHIFT 0x13
+#define VGT_DEBUG_REG34__last_point_of_inner_ring_p1_MASK 0x100000
+#define VGT_DEBUG_REG34__last_point_of_inner_ring_p1__SHIFT 0x14
+#define VGT_DEBUG_REG34__outer_edge_tf_eq_one_p0_q_MASK 0x200000
+#define VGT_DEBUG_REG34__outer_edge_tf_eq_one_p0_q__SHIFT 0x15
+#define VGT_DEBUG_REG34__advance_outer_point_p1_MASK 0x400000
+#define VGT_DEBUG_REG34__advance_outer_point_p1__SHIFT 0x16
+#define VGT_DEBUG_REG34__advance_inner_point_p1_MASK 0x800000
+#define VGT_DEBUG_REG34__advance_inner_point_p1__SHIFT 0x17
+#define VGT_DEBUG_REG34__next_ring_is_rect_p0_q_MASK 0x1000000
+#define VGT_DEBUG_REG34__next_ring_is_rect_p0_q__SHIFT 0x18
+#define VGT_DEBUG_REG34__pipe1_outer1_rtr_MASK 0x2000000
+#define VGT_DEBUG_REG34__pipe1_outer1_rtr__SHIFT 0x19
+#define VGT_DEBUG_REG34__pipe1_outer2_rtr_MASK 0x4000000
+#define VGT_DEBUG_REG34__pipe1_outer2_rtr__SHIFT 0x1a
+#define VGT_DEBUG_REG34__pipe1_inner1_rtr_MASK 0x8000000
+#define VGT_DEBUG_REG34__pipe1_inner1_rtr__SHIFT 0x1b
+#define VGT_DEBUG_REG34__pipe1_inner2_rtr_MASK 0x10000000
+#define VGT_DEBUG_REG34__pipe1_inner2_rtr__SHIFT 0x1c
+#define VGT_DEBUG_REG34__pipe1_patch_rtr_MASK 0x20000000
+#define VGT_DEBUG_REG34__pipe1_patch_rtr__SHIFT 0x1d
+#define VGT_DEBUG_REG34__pipe1_edge_rtr_MASK 0x40000000
+#define VGT_DEBUG_REG34__pipe1_edge_rtr__SHIFT 0x1e
+#define VGT_DEBUG_REG34__use_stored_inner_q_ring1_MASK 0x80000000
+#define VGT_DEBUG_REG34__use_stored_inner_q_ring1__SHIFT 0x1f
+#define VGT_DEBUG_REG36__VGT_PA_clipp_eop_MASK 0xffffffff
+#define VGT_DEBUG_REG36__VGT_PA_clipp_eop__SHIFT 0x0
+#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK_MASK 0xff
+#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK__SHIFT 0x0
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa
+#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000
+#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define VGT_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0xff
+#define VGT_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define VGT_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define VGT_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define VGT_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0xff
+#define VGT_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define VGT_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define VGT_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf000000
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x18
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa
+#define IA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000
+#define IA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define IA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff
+#define IA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define IA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define IA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define IA_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0xff
+#define IA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define IA_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define IA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define IA_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0xff
+#define IA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define IA_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define IA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c
+#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define WD_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0xff
+#define WD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0
+#define WD_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000
+#define WD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c
+#define WD_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff
+#define WD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0
+#define WD_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000
+#define WD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c
+#define WD_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0xff
+#define WD_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0
+#define WD_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000
+#define WD_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c
+#define WD_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0xff
+#define WD_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0
+#define WD_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000
+#define WD_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c
+#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff
+#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0
+#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff
+#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0
+#define DIDT_IND_INDEX__DIDT_IND_INDEX_MASK 0xffffffff
+#define DIDT_IND_INDEX__DIDT_IND_INDEX__SHIFT 0x0
+#define DIDT_IND_DATA__DIDT_IND_DATA_MASK 0xffffffff
+#define DIDT_IND_DATA__DIDT_IND_DATA__SHIFT 0x0
+#define DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK 0x1
+#define DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT 0x0
+#define DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK 0x2
+#define DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT 0x1
+#define DIDT_SQ_CTRL0__PHASE_OFFSET_MASK 0xc
+#define DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT 0x2
+#define DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK 0x10
+#define DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
+#define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
+#define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
+#define DIDT_SQ_CTRL0__UNUSED_0_MASK 0xffffffc0
+#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT 0x6
+#define DIDT_SQ_CTRL1__MIN_POWER_MASK 0xffff
+#define DIDT_SQ_CTRL1__MIN_POWER__SHIFT 0x0
+#define DIDT_SQ_CTRL1__MAX_POWER_MASK 0xffff0000
+#define DIDT_SQ_CTRL1__MAX_POWER__SHIFT 0x10
+#define DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK 0x3fff
+#define DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT 0x0
+#define DIDT_SQ_CTRL2__UNUSED_0_MASK 0xc000
+#define DIDT_SQ_CTRL2__UNUSED_0__SHIFT 0xe
+#define DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000
+#define DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10
+#define DIDT_SQ_CTRL2__UNUSED_1_MASK 0x4000000
+#define DIDT_SQ_CTRL2__UNUSED_1__SHIFT 0x1a
+#define DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000
+#define DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b
+#define DIDT_SQ_CTRL2__UNUSED_2_MASK 0x80000000
+#define DIDT_SQ_CTRL2__UNUSED_2__SHIFT 0x1f
+#define DIDT_SQ_CTRL_OCP__UNUSED_0_MASK 0xffff
+#define DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT 0x0
+#define DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK 0xffff0000
+#define DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT 0x10
+#define DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK 0xff
+#define DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT 0x0
+#define DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK 0xff00
+#define DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT 0x8
+#define DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK 0xff0000
+#define DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT 0x10
+#define DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK 0xff000000
+#define DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT 0x18
+#define DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK 0xff
+#define DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT 0x0
+#define DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK 0xff00
+#define DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT 0x8
+#define DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK 0xff0000
+#define DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT 0x10
+#define DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK 0xff000000
+#define DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT 0x18
+#define DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK 0xff
+#define DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT 0x0
+#define DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK 0xff00
+#define DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT 0x8
+#define DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK 0xff0000
+#define DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT 0x10
+#define DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK 0xff000000
+#define DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT 0x18
+#define DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK 0x1
+#define DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT 0x0
+#define DIDT_DB_CTRL0__USE_REF_CLOCK_MASK 0x2
+#define DIDT_DB_CTRL0__USE_REF_CLOCK__SHIFT 0x1
+#define DIDT_DB_CTRL0__PHASE_OFFSET_MASK 0xc
+#define DIDT_DB_CTRL0__PHASE_OFFSET__SHIFT 0x2
+#define DIDT_DB_CTRL0__DIDT_CTRL_RST_MASK 0x10
+#define DIDT_DB_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
+#define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
+#define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
+#define DIDT_DB_CTRL0__UNUSED_0_MASK 0xffffffc0
+#define DIDT_DB_CTRL0__UNUSED_0__SHIFT 0x6
+#define DIDT_DB_CTRL1__MIN_POWER_MASK 0xffff
+#define DIDT_DB_CTRL1__MIN_POWER__SHIFT 0x0
+#define DIDT_DB_CTRL1__MAX_POWER_MASK 0xffff0000
+#define DIDT_DB_CTRL1__MAX_POWER__SHIFT 0x10
+#define DIDT_DB_CTRL2__MAX_POWER_DELTA_MASK 0x3fff
+#define DIDT_DB_CTRL2__MAX_POWER_DELTA__SHIFT 0x0
+#define DIDT_DB_CTRL2__UNUSED_0_MASK 0xc000
+#define DIDT_DB_CTRL2__UNUSED_0__SHIFT 0xe
+#define DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000
+#define DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10
+#define DIDT_DB_CTRL2__UNUSED_1_MASK 0x4000000
+#define DIDT_DB_CTRL2__UNUSED_1__SHIFT 0x1a
+#define DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000
+#define DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b
+#define DIDT_DB_CTRL2__UNUSED_2_MASK 0x80000000
+#define DIDT_DB_CTRL2__UNUSED_2__SHIFT 0x1f
+#define DIDT_DB_CTRL_OCP__UNUSED_0_MASK 0xffff
+#define DIDT_DB_CTRL_OCP__UNUSED_0__SHIFT 0x0
+#define DIDT_DB_CTRL_OCP__OCP_MAX_POWER_MASK 0xffff0000
+#define DIDT_DB_CTRL_OCP__OCP_MAX_POWER__SHIFT 0x10
+#define DIDT_DB_WEIGHT0_3__WEIGHT0_MASK 0xff
+#define DIDT_DB_WEIGHT0_3__WEIGHT0__SHIFT 0x0
+#define DIDT_DB_WEIGHT0_3__WEIGHT1_MASK 0xff00
+#define DIDT_DB_WEIGHT0_3__WEIGHT1__SHIFT 0x8
+#define DIDT_DB_WEIGHT0_3__WEIGHT2_MASK 0xff0000
+#define DIDT_DB_WEIGHT0_3__WEIGHT2__SHIFT 0x10
+#define DIDT_DB_WEIGHT0_3__WEIGHT3_MASK 0xff000000
+#define DIDT_DB_WEIGHT0_3__WEIGHT3__SHIFT 0x18
+#define DIDT_DB_WEIGHT4_7__WEIGHT4_MASK 0xff
+#define DIDT_DB_WEIGHT4_7__WEIGHT4__SHIFT 0x0
+#define DIDT_DB_WEIGHT4_7__WEIGHT5_MASK 0xff00
+#define DIDT_DB_WEIGHT4_7__WEIGHT5__SHIFT 0x8
+#define DIDT_DB_WEIGHT4_7__WEIGHT6_MASK 0xff0000
+#define DIDT_DB_WEIGHT4_7__WEIGHT6__SHIFT 0x10
+#define DIDT_DB_WEIGHT4_7__WEIGHT7_MASK 0xff000000
+#define DIDT_DB_WEIGHT4_7__WEIGHT7__SHIFT 0x18
+#define DIDT_DB_WEIGHT8_11__WEIGHT8_MASK 0xff
+#define DIDT_DB_WEIGHT8_11__WEIGHT8__SHIFT 0x0
+#define DIDT_DB_WEIGHT8_11__WEIGHT9_MASK 0xff00
+#define DIDT_DB_WEIGHT8_11__WEIGHT9__SHIFT 0x8
+#define DIDT_DB_WEIGHT8_11__WEIGHT10_MASK 0xff0000
+#define DIDT_DB_WEIGHT8_11__WEIGHT10__SHIFT 0x10
+#define DIDT_DB_WEIGHT8_11__WEIGHT11_MASK 0xff000000
+#define DIDT_DB_WEIGHT8_11__WEIGHT11__SHIFT 0x18
+#define DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK 0x1
+#define DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT 0x0
+#define DIDT_TD_CTRL0__USE_REF_CLOCK_MASK 0x2
+#define DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT 0x1
+#define DIDT_TD_CTRL0__PHASE_OFFSET_MASK 0xc
+#define DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT 0x2
+#define DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK 0x10
+#define DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
+#define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
+#define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
+#define DIDT_TD_CTRL0__UNUSED_0_MASK 0xffffffc0
+#define DIDT_TD_CTRL0__UNUSED_0__SHIFT 0x6
+#define DIDT_TD_CTRL1__MIN_POWER_MASK 0xffff
+#define DIDT_TD_CTRL1__MIN_POWER__SHIFT 0x0
+#define DIDT_TD_CTRL1__MAX_POWER_MASK 0xffff0000
+#define DIDT_TD_CTRL1__MAX_POWER__SHIFT 0x10
+#define DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK 0x3fff
+#define DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT 0x0
+#define DIDT_TD_CTRL2__UNUSED_0_MASK 0xc000
+#define DIDT_TD_CTRL2__UNUSED_0__SHIFT 0xe
+#define DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000
+#define DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10
+#define DIDT_TD_CTRL2__UNUSED_1_MASK 0x4000000
+#define DIDT_TD_CTRL2__UNUSED_1__SHIFT 0x1a
+#define DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000
+#define DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b
+#define DIDT_TD_CTRL2__UNUSED_2_MASK 0x80000000
+#define DIDT_TD_CTRL2__UNUSED_2__SHIFT 0x1f
+#define DIDT_TD_CTRL_OCP__UNUSED_0_MASK 0xffff
+#define DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT 0x0
+#define DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK 0xffff0000
+#define DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT 0x10
+#define DIDT_TD_WEIGHT0_3__WEIGHT0_MASK 0xff
+#define DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT 0x0
+#define DIDT_TD_WEIGHT0_3__WEIGHT1_MASK 0xff00
+#define DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT 0x8
+#define DIDT_TD_WEIGHT0_3__WEIGHT2_MASK 0xff0000
+#define DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT 0x10
+#define DIDT_TD_WEIGHT0_3__WEIGHT3_MASK 0xff000000
+#define DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT 0x18
+#define DIDT_TD_WEIGHT4_7__WEIGHT4_MASK 0xff
+#define DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT 0x0
+#define DIDT_TD_WEIGHT4_7__WEIGHT5_MASK 0xff00
+#define DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT 0x8
+#define DIDT_TD_WEIGHT4_7__WEIGHT6_MASK 0xff0000
+#define DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT 0x10
+#define DIDT_TD_WEIGHT4_7__WEIGHT7_MASK 0xff000000
+#define DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT 0x18
+#define DIDT_TD_WEIGHT8_11__WEIGHT8_MASK 0xff
+#define DIDT_TD_WEIGHT8_11__WEIGHT8__SHIFT 0x0
+#define DIDT_TD_WEIGHT8_11__WEIGHT9_MASK 0xff00
+#define DIDT_TD_WEIGHT8_11__WEIGHT9__SHIFT 0x8
+#define DIDT_TD_WEIGHT8_11__WEIGHT10_MASK 0xff0000
+#define DIDT_TD_WEIGHT8_11__WEIGHT10__SHIFT 0x10
+#define DIDT_TD_WEIGHT8_11__WEIGHT11_MASK 0xff000000
+#define DIDT_TD_WEIGHT8_11__WEIGHT11__SHIFT 0x18
+#define DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK 0x1
+#define DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT 0x0
+#define DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK 0x2
+#define DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT 0x1
+#define DIDT_TCP_CTRL0__PHASE_OFFSET_MASK 0xc
+#define DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT 0x2
+#define DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK 0x10
+#define DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
+#define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
+#define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
+#define DIDT_TCP_CTRL0__UNUSED_0_MASK 0xffffffc0
+#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT 0x6
+#define DIDT_TCP_CTRL1__MIN_POWER_MASK 0xffff
+#define DIDT_TCP_CTRL1__MIN_POWER__SHIFT 0x0
+#define DIDT_TCP_CTRL1__MAX_POWER_MASK 0xffff0000
+#define DIDT_TCP_CTRL1__MAX_POWER__SHIFT 0x10
+#define DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK 0x3fff
+#define DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT 0x0
+#define DIDT_TCP_CTRL2__UNUSED_0_MASK 0xc000
+#define DIDT_TCP_CTRL2__UNUSED_0__SHIFT 0xe
+#define DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000
+#define DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10
+#define DIDT_TCP_CTRL2__UNUSED_1_MASK 0x4000000
+#define DIDT_TCP_CTRL2__UNUSED_1__SHIFT 0x1a
+#define DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000
+#define DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b
+#define DIDT_TCP_CTRL2__UNUSED_2_MASK 0x80000000
+#define DIDT_TCP_CTRL2__UNUSED_2__SHIFT 0x1f
+#define DIDT_TCP_CTRL_OCP__UNUSED_0_MASK 0xffff
+#define DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT 0x0
+#define DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK 0xffff0000
+#define DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT 0x10
+#define DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK 0xff
+#define DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT 0x0
+#define DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK 0xff00
+#define DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT 0x8
+#define DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK 0xff0000
+#define DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT 0x10
+#define DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK 0xff000000
+#define DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT 0x18
+#define DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK 0xff
+#define DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT 0x0
+#define DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK 0xff00
+#define DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT 0x8
+#define DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK 0xff0000
+#define DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT 0x10
+#define DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK 0xff000000
+#define DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT 0x18
+#define DIDT_TCP_WEIGHT8_11__WEIGHT8_MASK 0xff
+#define DIDT_TCP_WEIGHT8_11__WEIGHT8__SHIFT 0x0
+#define DIDT_TCP_WEIGHT8_11__WEIGHT9_MASK 0xff00
+#define DIDT_TCP_WEIGHT8_11__WEIGHT9__SHIFT 0x8
+#define DIDT_TCP_WEIGHT8_11__WEIGHT10_MASK 0xff0000
+#define DIDT_TCP_WEIGHT8_11__WEIGHT10__SHIFT 0x10
+#define DIDT_TCP_WEIGHT8_11__WEIGHT11_MASK 0xff000000
+#define DIDT_TCP_WEIGHT8_11__WEIGHT11__SHIFT 0x18
+#define DIDT_DBR_CTRL0__DIDT_CTRL_EN_MASK 0x1
+#define DIDT_DBR_CTRL0__DIDT_CTRL_EN__SHIFT 0x0
+#define DIDT_DBR_CTRL0__USE_REF_CLOCK_MASK 0x2
+#define DIDT_DBR_CTRL0__USE_REF_CLOCK__SHIFT 0x1
+#define DIDT_DBR_CTRL0__PHASE_OFFSET_MASK 0xc
+#define DIDT_DBR_CTRL0__PHASE_OFFSET__SHIFT 0x2
+#define DIDT_DBR_CTRL0__DIDT_CTRL_RST_MASK 0x10
+#define DIDT_DBR_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
+#define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
+#define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
+#define DIDT_DBR_CTRL0__UNUSED_0_MASK 0xffffffc0
+#define DIDT_DBR_CTRL0__UNUSED_0__SHIFT 0x6
+#define DIDT_DBR_CTRL1__MIN_POWER_MASK 0xffff
+#define DIDT_DBR_CTRL1__MIN_POWER__SHIFT 0x0
+#define DIDT_DBR_CTRL1__MAX_POWER_MASK 0xffff0000
+#define DIDT_DBR_CTRL1__MAX_POWER__SHIFT 0x10
+#define DIDT_DBR_CTRL2__MAX_POWER_DELTA_MASK 0x3fff
+#define DIDT_DBR_CTRL2__MAX_POWER_DELTA__SHIFT 0x0
+#define DIDT_DBR_CTRL2__UNUSED_0_MASK 0xc000
+#define DIDT_DBR_CTRL2__UNUSED_0__SHIFT 0xe
+#define DIDT_DBR_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000
+#define DIDT_DBR_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10
+#define DIDT_DBR_CTRL2__UNUSED_1_MASK 0x4000000
+#define DIDT_DBR_CTRL2__UNUSED_1__SHIFT 0x1a
+#define DIDT_DBR_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000
+#define DIDT_DBR_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b
+#define DIDT_DBR_CTRL2__UNUSED_2_MASK 0x80000000
+#define DIDT_DBR_CTRL2__UNUSED_2__SHIFT 0x1f
+#define DIDT_DBR_CTRL_OCP__UNUSED_0_MASK 0xffff
+#define DIDT_DBR_CTRL_OCP__UNUSED_0__SHIFT 0x0
+#define DIDT_DBR_CTRL_OCP__OCP_MAX_POWER_MASK 0xffff0000
+#define DIDT_DBR_CTRL_OCP__OCP_MAX_POWER__SHIFT 0x10
+#define DIDT_DBR_WEIGHT0_3__WEIGHT0_MASK 0xff
+#define DIDT_DBR_WEIGHT0_3__WEIGHT0__SHIFT 0x0
+#define DIDT_DBR_WEIGHT0_3__WEIGHT1_MASK 0xff00
+#define DIDT_DBR_WEIGHT0_3__WEIGHT1__SHIFT 0x8
+#define DIDT_DBR_WEIGHT0_3__WEIGHT2_MASK 0xff0000
+#define DIDT_DBR_WEIGHT0_3__WEIGHT2__SHIFT 0x10
+#define DIDT_DBR_WEIGHT0_3__WEIGHT3_MASK 0xff000000
+#define DIDT_DBR_WEIGHT0_3__WEIGHT3__SHIFT 0x18
+#define DIDT_DBR_WEIGHT4_7__WEIGHT4_MASK 0xff
+#define DIDT_DBR_WEIGHT4_7__WEIGHT4__SHIFT 0x0
+#define DIDT_DBR_WEIGHT4_7__WEIGHT5_MASK 0xff00
+#define DIDT_DBR_WEIGHT4_7__WEIGHT5__SHIFT 0x8
+#define DIDT_DBR_WEIGHT4_7__WEIGHT6_MASK 0xff0000
+#define DIDT_DBR_WEIGHT4_7__WEIGHT6__SHIFT 0x10
+#define DIDT_DBR_WEIGHT4_7__WEIGHT7_MASK 0xff000000
+#define DIDT_DBR_WEIGHT4_7__WEIGHT7__SHIFT 0x18
+#define DIDT_DBR_WEIGHT8_11__WEIGHT8_MASK 0xff
+#define DIDT_DBR_WEIGHT8_11__WEIGHT8__SHIFT 0x0
+#define DIDT_DBR_WEIGHT8_11__WEIGHT9_MASK 0xff00
+#define DIDT_DBR_WEIGHT8_11__WEIGHT9__SHIFT 0x8
+#define DIDT_DBR_WEIGHT8_11__WEIGHT10_MASK 0xff0000
+#define DIDT_DBR_WEIGHT8_11__WEIGHT10__SHIFT 0x10
+#define DIDT_DBR_WEIGHT8_11__WEIGHT11_MASK 0xff000000
+#define DIDT_DBR_WEIGHT8_11__WEIGHT11__SHIFT 0x18
+
+#endif /* GFX_8_1_SH_MASK_H */
ULONG ulMCUcodeRomStartAddr;
ULONG ulMCUcodeLength;
USHORT usMcRegInitTableOffset; // offset of ATOM_REG_INIT_SETTING array for MC core register settings.
- USHORT usReserved; // offset of ATOM_INIT_REG_BLOCK for MC SEQ/PHY regsiter setting
+ USHORT usReserved; // offset of ATOM_INIT_REG_BLOCK for MC SEQ/PHY register setting
}ATOM_MC_INIT_PARAM_TABLE_V2_1;
#include "cgs_common.h"
-/**
- * cgs_import_gpu_mem() - Import dmabuf handle
- * @cgs_device: opaque device handle
- * @dmabuf_fd: DMABuf file descriptor
- * @handle: memory handle (output)
- *
- * Must be called in the process context that dmabuf_fd belongs to.
- *
- * Return: 0 on success, -errno otherwise
- */
-typedef int (*cgs_import_gpu_mem_t)(void *cgs_device, int dmabuf_fd,
- cgs_handle_t *handle);
-
/**
* cgs_irq_source_set_func() - Callback for enabling/disabling interrupt sources
* @private_data: private data provided to cgs_add_irq_source
typedef int (*cgs_irq_put_t)(void *cgs_device, unsigned src_id, unsigned type);
struct cgs_os_ops {
- cgs_import_gpu_mem_t import_gpu_mem;
-
/* IRQ handling */
cgs_add_irq_source_t add_irq_source;
cgs_irq_get_t irq_get;
cgs_irq_put_t irq_put;
};
-#define cgs_import_gpu_mem(dev,dmabuf_fd,handle) \
- CGS_OS_CALL(import_gpu_mem,dev,dmabuf_fd,handle)
#define cgs_add_irq_source(dev,src_id,num_types,set,handler,private_data) \
CGS_OS_CALL(add_irq_source,dev,src_id,num_types,set,handler, \
private_data)
--- /dev/null
+#if !defined(_GPU_SCHED_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _GPU_SCHED_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#include <drm/drmP.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM gpu_sched
+#define TRACE_INCLUDE_FILE gpu_sched_trace
+
+TRACE_EVENT(amd_sched_job,
+ TP_PROTO(struct amd_sched_job *sched_job),
+ TP_ARGS(sched_job),
+ TP_STRUCT__entry(
+ __field(struct amd_sched_entity *, entity)
+ __field(const char *, name)
+ __field(u32, job_count)
+ __field(int, hw_job_count)
+ ),
+
+ TP_fast_assign(
+ __entry->entity = sched_job->s_entity;
+ __entry->name = sched_job->sched->name;
+ __entry->job_count = kfifo_len(
+ &sched_job->s_entity->job_queue) / sizeof(sched_job);
+ __entry->hw_job_count = atomic_read(
+ &sched_job->sched->hw_rq_count);
+ ),
+ TP_printk("entity=%p, ring=%s, job count:%u, hw job count:%d",
+ __entry->entity, __entry->name, __entry->job_count,
+ __entry->hw_job_count)
+);
+#endif
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#include <trace/define_trace.h>
#include <drm/drmP.h>
#include "gpu_scheduler.h"
+#define CREATE_TRACE_POINTS
+#include "gpu_sched_trace.h"
+
static struct amd_sched_job *
amd_sched_entity_pop_job(struct amd_sched_entity *entity);
static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
amd_sched_rq_select_job(struct amd_sched_rq *rq)
{
struct amd_sched_entity *entity;
- struct amd_sched_job *job;
+ struct amd_sched_job *sched_job;
spin_lock(&rq->lock);
entity = rq->current_entity;
if (entity) {
list_for_each_entry_continue(entity, &rq->entities, list) {
- job = amd_sched_entity_pop_job(entity);
- if (job) {
+ sched_job = amd_sched_entity_pop_job(entity);
+ if (sched_job) {
rq->current_entity = entity;
spin_unlock(&rq->lock);
- return job;
+ return sched_job;
}
}
}
list_for_each_entry(entity, &rq->entities, list) {
- job = amd_sched_entity_pop_job(entity);
- if (job) {
+ sched_job = amd_sched_entity_pop_job(entity);
+ if (sched_job) {
rq->current_entity = entity;
spin_unlock(&rq->lock);
- return job;
+ return sched_job;
}
if (entity == rq->current_entity)
struct amd_sched_rq *rq,
uint32_t jobs)
{
+ int r;
+
if (!(sched && entity && rq))
return -EINVAL;
memset(entity, 0, sizeof(struct amd_sched_entity));
- entity->belongto_rq = rq;
- entity->scheduler = sched;
- entity->fence_context = fence_context_alloc(1);
- if(kfifo_alloc(&entity->job_queue,
- jobs * sizeof(void *),
- GFP_KERNEL))
- return -EINVAL;
+ INIT_LIST_HEAD(&entity->list);
+ entity->rq = rq;
+ entity->sched = sched;
spin_lock_init(&entity->queue_lock);
+ r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);
+ if (r)
+ return r;
+
atomic_set(&entity->fence_seq, 0);
+ entity->fence_context = fence_context_alloc(1);
/* Add the entity to the run queue */
amd_sched_rq_add_entity(rq, entity);
+
return 0;
}
static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
struct amd_sched_entity *entity)
{
- return entity->scheduler == sched &&
- entity->belongto_rq != NULL;
+ return entity->sched == sched &&
+ entity->rq != NULL;
}
/**
void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
struct amd_sched_entity *entity)
{
- struct amd_sched_rq *rq = entity->belongto_rq;
+ struct amd_sched_rq *rq = entity->rq;
if (!amd_sched_entity_is_initialized(sched, entity))
return;
container_of(cb, struct amd_sched_entity, cb);
entity->dependency = NULL;
fence_put(f);
- amd_sched_wakeup(entity->scheduler);
+ amd_sched_wakeup(entity->sched);
}
static struct amd_sched_job *
amd_sched_entity_pop_job(struct amd_sched_entity *entity)
{
- struct amd_gpu_scheduler *sched = entity->scheduler;
- struct amd_sched_job *job;
+ struct amd_gpu_scheduler *sched = entity->sched;
+ struct amd_sched_job *sched_job;
if (ACCESS_ONCE(entity->dependency))
return NULL;
- if (!kfifo_out_peek(&entity->job_queue, &job, sizeof(job)))
+ if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
return NULL;
- while ((entity->dependency = sched->ops->dependency(job))) {
+ while ((entity->dependency = sched->ops->dependency(sched_job))) {
+
+ if (entity->dependency->context == entity->fence_context) {
+ /* We can ignore fences from ourself */
+ fence_put(entity->dependency);
+ continue;
+ }
if (fence_add_callback(entity->dependency, &entity->cb,
amd_sched_entity_wakeup))
return NULL;
}
- return job;
+ return sched_job;
}
/**
* Helper to submit a job to the job queue
*
- * @job The pointer to job required to submit
+ * @sched_job The pointer to job required to submit
*
* Returns true if we could submit the job.
*/
-static bool amd_sched_entity_in(struct amd_sched_job *job)
+static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
{
- struct amd_sched_entity *entity = job->s_entity;
+ struct amd_sched_entity *entity = sched_job->s_entity;
bool added, first = false;
spin_lock(&entity->queue_lock);
- added = kfifo_in(&entity->job_queue, &job, sizeof(job)) == sizeof(job);
+ added = kfifo_in(&entity->job_queue, &sched_job,
+ sizeof(sched_job)) == sizeof(sched_job);
- if (added && kfifo_len(&entity->job_queue) == sizeof(job))
+ if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))
first = true;
spin_unlock(&entity->queue_lock);
/* first job wakes up scheduler */
if (first)
- amd_sched_wakeup(job->sched);
+ amd_sched_wakeup(sched_job->sched);
return added;
}
/**
* Submit a job to the job queue
*
- * @job The pointer to job required to submit
+ * @sched_job The pointer to job required to submit
*
* Returns 0 for success, negative error code otherwise.
*/
fence_get(&fence->base);
sched_job->s_fence = fence;
- wait_event(entity->scheduler->job_scheduled,
+ wait_event(entity->sched->job_scheduled,
amd_sched_entity_in(sched_job));
-
+ trace_amd_sched_job(sched_job);
return 0;
}
static struct amd_sched_job *
amd_sched_select_job(struct amd_gpu_scheduler *sched)
{
- struct amd_sched_job *job;
+ struct amd_sched_job *sched_job;
if (!amd_sched_ready(sched))
return NULL;
/* Kernel run queue has higher priority than normal run queue*/
- job = amd_sched_rq_select_job(&sched->kernel_rq);
- if (job == NULL)
- job = amd_sched_rq_select_job(&sched->sched_rq);
+ sched_job = amd_sched_rq_select_job(&sched->kernel_rq);
+ if (sched_job == NULL)
+ sched_job = amd_sched_rq_select_job(&sched->sched_rq);
- return job;
+ return sched_job;
}
static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
{
- struct amd_sched_job *sched_job =
- container_of(cb, struct amd_sched_job, cb);
- struct amd_gpu_scheduler *sched;
+ struct amd_sched_fence *s_fence =
+ container_of(cb, struct amd_sched_fence, cb);
+ struct amd_gpu_scheduler *sched = s_fence->sched;
+ unsigned long flags;
- sched = sched_job->sched;
- amd_sched_fence_signal(sched_job->s_fence);
atomic_dec(&sched->hw_rq_count);
- fence_put(&sched_job->s_fence->base);
- sched->ops->process_job(sched_job);
+ amd_sched_fence_signal(s_fence);
+ if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
+ cancel_delayed_work(&s_fence->dwork);
+ spin_lock_irqsave(&sched->fence_list_lock, flags);
+ list_del_init(&s_fence->list);
+ spin_unlock_irqrestore(&sched->fence_list_lock, flags);
+ }
+ fence_put(&s_fence->base);
wake_up_interruptible(&sched->wake_up_worker);
}
+static void amd_sched_fence_work_func(struct work_struct *work)
+{
+ struct amd_sched_fence *s_fence =
+ container_of(work, struct amd_sched_fence, dwork.work);
+ struct amd_gpu_scheduler *sched = s_fence->sched;
+ struct amd_sched_fence *entity, *tmp;
+ unsigned long flags;
+
+ DRM_ERROR("[%s] scheduler is timeout!\n", sched->name);
+
+ /* Clean all pending fences */
+ spin_lock_irqsave(&sched->fence_list_lock, flags);
+ list_for_each_entry_safe(entity, tmp, &sched->fence_list, list) {
+ DRM_ERROR(" fence no %d\n", entity->base.seqno);
+ cancel_delayed_work(&entity->dwork);
+ list_del_init(&entity->list);
+ fence_put(&entity->base);
+ }
+ spin_unlock_irqrestore(&sched->fence_list_lock, flags);
+}
+
static int amd_sched_main(void *param)
{
struct sched_param sparam = {.sched_priority = 1};
struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
int r, count;
+ spin_lock_init(&sched->fence_list_lock);
+ INIT_LIST_HEAD(&sched->fence_list);
sched_setscheduler(current, SCHED_FIFO, &sparam);
while (!kthread_should_stop()) {
struct amd_sched_entity *entity;
- struct amd_sched_job *job;
+ struct amd_sched_fence *s_fence;
+ struct amd_sched_job *sched_job;
struct fence *fence;
+ unsigned long flags;
wait_event_interruptible(sched->wake_up_worker,
kthread_should_stop() ||
- (job = amd_sched_select_job(sched)));
+ (sched_job = amd_sched_select_job(sched)));
- if (!job)
+ if (!sched_job)
continue;
- entity = job->s_entity;
+ entity = sched_job->s_entity;
+ s_fence = sched_job->s_fence;
+
+ if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
+ INIT_DELAYED_WORK(&s_fence->dwork, amd_sched_fence_work_func);
+ schedule_delayed_work(&s_fence->dwork, sched->timeout);
+ spin_lock_irqsave(&sched->fence_list_lock, flags);
+ list_add_tail(&s_fence->list, &sched->fence_list);
+ spin_unlock_irqrestore(&sched->fence_list_lock, flags);
+ }
+
atomic_inc(&sched->hw_rq_count);
- fence = sched->ops->run_job(job);
+ fence = sched->ops->run_job(sched_job);
if (fence) {
- r = fence_add_callback(fence, &job->cb,
+ r = fence_add_callback(fence, &s_fence->cb,
amd_sched_process_job);
if (r == -ENOENT)
- amd_sched_process_job(fence, &job->cb);
+ amd_sched_process_job(fence, &s_fence->cb);
else if (r)
DRM_ERROR("fence add callback failed (%d)\n", r);
fence_put(fence);
+ } else {
+ DRM_ERROR("Failed to run job!\n");
+ amd_sched_process_job(NULL, &s_fence->cb);
}
- count = kfifo_out(&entity->job_queue, &job, sizeof(job));
- WARN_ON(count != sizeof(job));
+ count = kfifo_out(&entity->job_queue, &sched_job,
+ sizeof(sched_job));
+ WARN_ON(count != sizeof(sched_job));
wake_up(&sched->job_scheduled);
}
return 0;
}
/**
- * Create a gpu scheduler
+ * Init a gpu scheduler instance
*
+ * @sched The pointer to the scheduler
* @ops The backend operations for this scheduler.
- * @ring The the ring id for the scheduler.
* @hw_submissions Number of hw submissions to do.
+ * @name Name used for debugging
*
- * Return the pointer to scheduler for success, otherwise return NULL
+ * Return 0 on success, otherwise error code.
*/
-struct amd_gpu_scheduler *amd_sched_create(struct amd_sched_backend_ops *ops,
- unsigned ring, unsigned hw_submission,
- void *priv)
+int amd_sched_init(struct amd_gpu_scheduler *sched,
+ struct amd_sched_backend_ops *ops,
+ unsigned hw_submission, long timeout, const char *name)
{
- struct amd_gpu_scheduler *sched;
-
- sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
- if (!sched)
- return NULL;
-
sched->ops = ops;
- sched->ring_id = ring;
sched->hw_submission_limit = hw_submission;
- sched->priv = priv;
- snprintf(sched->name, sizeof(sched->name), "amdgpu[%d]", ring);
+ sched->name = name;
+ sched->timeout = timeout;
amd_sched_rq_init(&sched->sched_rq);
amd_sched_rq_init(&sched->kernel_rq);
init_waitqueue_head(&sched->wake_up_worker);
init_waitqueue_head(&sched->job_scheduled);
atomic_set(&sched->hw_rq_count, 0);
+
/* Each scheduler will run on a seperate kernel thread */
sched->thread = kthread_run(amd_sched_main, sched, sched->name);
if (IS_ERR(sched->thread)) {
- DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
- kfree(sched);
- return NULL;
+ DRM_ERROR("Failed to create scheduler for %s.\n", name);
+ return PTR_ERR(sched->thread);
}
- return sched;
+ return 0;
}
/**
* Destroy a gpu scheduler
*
* @sched The pointer to the scheduler
- *
- * return 0 if succeed. -1 if failed.
*/
-int amd_sched_destroy(struct amd_gpu_scheduler *sched)
+void amd_sched_fini(struct amd_gpu_scheduler *sched)
{
- kthread_stop(sched->thread);
- kfree(sched);
- return 0;
+ if (sched->thread)
+ kthread_stop(sched->thread);
}
*/
struct amd_sched_entity {
struct list_head list;
- struct amd_sched_rq *belongto_rq;
- atomic_t fence_seq;
- /* the job_queue maintains the jobs submitted by clients */
- struct kfifo job_queue;
+ struct amd_sched_rq *rq;
+ struct amd_gpu_scheduler *sched;
+
spinlock_t queue_lock;
- struct amd_gpu_scheduler *scheduler;
+ struct kfifo job_queue;
+
+ atomic_t fence_seq;
uint64_t fence_context;
+
struct fence *dependency;
struct fence_cb cb;
};
struct amd_sched_fence {
struct fence base;
- struct amd_gpu_scheduler *scheduler;
+ struct fence_cb cb;
+ struct amd_gpu_scheduler *sched;
spinlock_t lock;
void *owner;
+ struct delayed_work dwork;
+ struct list_head list;
};
struct amd_sched_job {
- struct fence_cb cb;
struct amd_gpu_scheduler *sched;
struct amd_sched_entity *s_entity;
struct amd_sched_fence *s_fence;
* these functions should be implemented in driver side
*/
struct amd_sched_backend_ops {
- struct fence *(*dependency)(struct amd_sched_job *job);
- struct fence *(*run_job)(struct amd_sched_job *job);
- void (*process_job)(struct amd_sched_job *job);
+ struct fence *(*dependency)(struct amd_sched_job *sched_job);
+ struct fence *(*run_job)(struct amd_sched_job *sched_job);
};
/**
* One scheduler is implemented for each hardware ring
*/
struct amd_gpu_scheduler {
- struct task_struct *thread;
+ struct amd_sched_backend_ops *ops;
+ uint32_t hw_submission_limit;
+ long timeout;
+ const char *name;
struct amd_sched_rq sched_rq;
struct amd_sched_rq kernel_rq;
- atomic_t hw_rq_count;
- struct amd_sched_backend_ops *ops;
- uint32_t ring_id;
wait_queue_head_t wake_up_worker;
wait_queue_head_t job_scheduled;
- uint32_t hw_submission_limit;
- char name[20];
- void *priv;
+ atomic_t hw_rq_count;
+ struct list_head fence_list;
+ spinlock_t fence_list_lock;
+ struct task_struct *thread;
};
-struct amd_gpu_scheduler *
-amd_sched_create(struct amd_sched_backend_ops *ops,
- uint32_t ring, uint32_t hw_submission, void *priv);
-int amd_sched_destroy(struct amd_gpu_scheduler *sched);
+int amd_sched_init(struct amd_gpu_scheduler *sched,
+ struct amd_sched_backend_ops *ops,
+ uint32_t hw_submission, long timeout, const char *name);
+void amd_sched_fini(struct amd_gpu_scheduler *sched);
int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
struct amd_sched_entity *entity,
if (fence == NULL)
return NULL;
fence->owner = owner;
- fence->scheduler = s_entity->scheduler;
+ fence->sched = s_entity->sched;
spin_lock_init(&fence->lock);
seq = atomic_inc_return(&s_entity->fence_seq);
static const char *amd_sched_fence_get_timeline_name(struct fence *f)
{
struct amd_sched_fence *fence = to_amd_sched_fence(f);
- return (const char *)fence->scheduler->name;
+ return (const char *)fence->sched->name;
}
static bool amd_sched_fence_enable_signaling(struct fence *f)
This driver provides no built-in acceleration; acceleration is
performed by other IP found on the SoC. This driver provides
kernel mode setting and buffer management to userspace.
-
-config DRM_ARMADA_TDA1998X
- bool "Support TDA1998X HDMI output"
- depends on DRM_ARMADA != n
- depends on I2C && DRM_I2C_NXP_TDA998X = y
- default y
- help
- Support the TDA1998x HDMI output device found on the Solid-Run
- CuBox.
armada-y := armada_crtc.o armada_drv.o armada_fb.o armada_fbdev.o \
- armada_gem.o armada_output.o armada_overlay.o \
- armada_slave.o
+ armada_gem.o armada_overlay.o
armada-y += armada_510.o
armada-$(CONFIG_DEBUG_FS) += armada_debugfs.o
#include "armada_hw.h"
struct armada_frame_work {
+ struct armada_plane_work work;
struct drm_pending_vblank_event *event;
struct armada_regs regs[4];
struct drm_framebuffer *old_fb;
CSC_RGB_STUDIO = 2,
};
+static const uint32_t armada_primary_formats[] = {
+ DRM_FORMAT_UYVY,
+ DRM_FORMAT_YUYV,
+ DRM_FORMAT_VYUY,
+ DRM_FORMAT_YVYU,
+ DRM_FORMAT_ARGB8888,
+ DRM_FORMAT_ABGR8888,
+ DRM_FORMAT_XRGB8888,
+ DRM_FORMAT_XBGR8888,
+ DRM_FORMAT_RGB888,
+ DRM_FORMAT_BGR888,
+ DRM_FORMAT_ARGB1555,
+ DRM_FORMAT_ABGR1555,
+ DRM_FORMAT_RGB565,
+ DRM_FORMAT_BGR565,
+};
+
/*
* A note about interlacing. Let's consider HDMI 1920x1080i.
* The timing parameters we have from X are:
return i;
}
-static int armada_drm_crtc_queue_frame_work(struct armada_crtc *dcrtc,
- struct armada_frame_work *work)
+static void armada_drm_plane_work_run(struct armada_crtc *dcrtc,
+ struct armada_plane *plane)
+{
+ struct armada_plane_work *work = xchg(&plane->work, NULL);
+
+ /* Handle any pending frame work. */
+ if (work) {
+ work->fn(dcrtc, plane, work);
+ drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+ }
+
+ wake_up(&plane->frame_wait);
+}
+
+int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
+ struct armada_plane *plane, struct armada_plane_work *work)
{
- struct drm_device *dev = dcrtc->crtc.dev;
- unsigned long flags;
int ret;
- ret = drm_vblank_get(dev, dcrtc->num);
+ ret = drm_vblank_get(dcrtc->crtc.dev, dcrtc->num);
if (ret) {
DRM_ERROR("failed to acquire vblank counter\n");
return ret;
}
- spin_lock_irqsave(&dev->event_lock, flags);
- if (!dcrtc->frame_work)
- dcrtc->frame_work = work;
- else
- ret = -EBUSY;
- spin_unlock_irqrestore(&dev->event_lock, flags);
-
+ ret = cmpxchg(&plane->work, NULL, work) ? -EBUSY : 0;
if (ret)
- drm_vblank_put(dev, dcrtc->num);
+ drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
return ret;
}
-static void armada_drm_crtc_complete_frame_work(struct armada_crtc *dcrtc)
+int armada_drm_plane_work_wait(struct armada_plane *plane, long timeout)
{
- struct drm_device *dev = dcrtc->crtc.dev;
- struct armada_frame_work *work = dcrtc->frame_work;
+ return wait_event_timeout(plane->frame_wait, !plane->work, timeout);
+}
- dcrtc->frame_work = NULL;
+struct armada_plane_work *armada_drm_plane_work_cancel(
+ struct armada_crtc *dcrtc, struct armada_plane *plane)
+{
+ struct armada_plane_work *work = xchg(&plane->work, NULL);
- armada_drm_crtc_update_regs(dcrtc, work->regs);
+ if (work)
+ drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
- if (work->event)
- drm_send_vblank_event(dev, dcrtc->num, work->event);
+ return work;
+}
- drm_vblank_put(dev, dcrtc->num);
+static int armada_drm_crtc_queue_frame_work(struct armada_crtc *dcrtc,
+ struct armada_frame_work *work)
+{
+ struct armada_plane *plane = drm_to_armada_plane(dcrtc->crtc.primary);
+
+ return armada_drm_plane_work_queue(dcrtc, plane, &work->work);
+}
+
+static void armada_drm_crtc_complete_frame_work(struct armada_crtc *dcrtc,
+ struct armada_plane *plane, struct armada_plane_work *work)
+{
+ struct armada_frame_work *fwork = container_of(work, struct armada_frame_work, work);
+ struct drm_device *dev = dcrtc->crtc.dev;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dcrtc->irq_lock, flags);
+ armada_drm_crtc_update_regs(dcrtc, fwork->regs);
+ spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
+
+ if (fwork->event) {
+ spin_lock_irqsave(&dev->event_lock, flags);
+ drm_send_vblank_event(dev, dcrtc->num, fwork->event);
+ spin_unlock_irqrestore(&dev->event_lock, flags);
+ }
/* Finally, queue the process-half of the cleanup. */
- __armada_drm_queue_unref_work(dcrtc->crtc.dev, work->old_fb);
- kfree(work);
+ __armada_drm_queue_unref_work(dcrtc->crtc.dev, fwork->old_fb);
+ kfree(fwork);
}
static void armada_drm_crtc_finish_fb(struct armada_crtc *dcrtc,
work = kmalloc(sizeof(*work), GFP_KERNEL);
if (work) {
int i = 0;
+ work->work.fn = armada_drm_crtc_complete_frame_work;
work->event = NULL;
work->old_fb = fb;
armada_reg_queue_end(work->regs, i);
static void armada_drm_vblank_off(struct armada_crtc *dcrtc)
{
- struct drm_device *dev = dcrtc->crtc.dev;
+ struct armada_plane *plane = drm_to_armada_plane(dcrtc->crtc.primary);
/*
* Tell the DRM core that vblank IRQs aren't going to happen for
* a while. This cleans up any pending vblank events for us.
*/
drm_crtc_vblank_off(&dcrtc->crtc);
-
- /* Handle any pending flip event. */
- spin_lock_irq(&dev->event_lock);
- if (dcrtc->frame_work)
- armada_drm_crtc_complete_frame_work(dcrtc);
- spin_unlock_irq(&dev->event_lock);
+ armada_drm_plane_work_run(dcrtc, plane);
}
void armada_drm_crtc_gamma_set(struct drm_crtc *crtc, u16 r, u16 g, u16 b,
if (dcrtc->dpms != dpms) {
dcrtc->dpms = dpms;
+ if (!IS_ERR(dcrtc->clk) && !dpms_blanked(dpms))
+ WARN_ON(clk_prepare_enable(dcrtc->clk));
armada_drm_crtc_update(dcrtc);
+ if (!IS_ERR(dcrtc->clk) && dpms_blanked(dpms))
+ clk_disable_unprepare(dcrtc->clk);
if (dpms_blanked(dpms))
armada_drm_vblank_off(dcrtc);
else
/*
* If we have an overlay plane associated with this CRTC, disable
* it before the modeset to avoid its coordinates being outside
- * the new mode parameters. DRM doesn't provide help with this.
+ * the new mode parameters.
*/
plane = dcrtc->plane;
- if (plane) {
- struct drm_framebuffer *fb = plane->fb;
-
- plane->funcs->disable_plane(plane);
- plane->fb = NULL;
- plane->crtc = NULL;
- drm_framebuffer_unreference(fb);
- }
+ if (plane)
+ drm_plane_force_disable(plane);
}
/* The mode_config.mutex will be held for this call */
static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
{
- struct armada_vbl_event *e, *n;
void __iomem *base = dcrtc->base;
+ struct drm_plane *ovl_plane;
if (stat & DMA_FF_UNDERFLOW)
DRM_ERROR("video underflow on crtc %u\n", dcrtc->num);
drm_handle_vblank(dcrtc->crtc.dev, dcrtc->num);
spin_lock(&dcrtc->irq_lock);
-
- list_for_each_entry_safe(e, n, &dcrtc->vbl_list, node) {
- list_del_init(&e->node);
- drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
- e->fn(dcrtc, e->data);
+ ovl_plane = dcrtc->plane;
+ if (ovl_plane) {
+ struct armada_plane *plane = drm_to_armada_plane(ovl_plane);
+ armada_drm_plane_work_run(dcrtc, plane);
}
if (stat & GRA_FRAME_IRQ && dcrtc->interlaced) {
spin_unlock(&dcrtc->irq_lock);
if (stat & GRA_FRAME_IRQ) {
- struct drm_device *dev = dcrtc->crtc.dev;
-
- spin_lock(&dev->event_lock);
- if (dcrtc->frame_work)
- armada_drm_crtc_complete_frame_work(dcrtc);
- spin_unlock(&dev->event_lock);
-
- wake_up(&dcrtc->frame_wait);
+ struct armada_plane *plane = drm_to_armada_plane(dcrtc->crtc.primary);
+ armada_drm_plane_work_run(dcrtc, plane);
}
}
adj->crtc_vtotal, tm, bm);
/* Wait for pending flips to complete */
- wait_event(dcrtc->frame_wait, !dcrtc->frame_work);
+ armada_drm_plane_work_wait(drm_to_armada_plane(dcrtc->crtc.primary),
+ MAX_SCHEDULE_TIMEOUT);
drm_crtc_vblank_off(crtc);
writel_relaxed(val, dcrtc->base + LCD_SPU_DUMB_CTRL);
}
+ /*
+ * If we are blanked, we would have disabled the clock. Re-enable
+ * it so that compute_clock() does the right thing.
+ */
+ if (!IS_ERR(dcrtc->clk) && dpms_blanked(dcrtc->dpms))
+ WARN_ON(clk_prepare_enable(dcrtc->clk));
+
/* Now compute the divider for real */
dcrtc->variant->compute_clock(dcrtc, adj, &sclk);
armada_reg_queue_end(regs, i);
/* Wait for pending flips to complete */
- wait_event(dcrtc->frame_wait, !dcrtc->frame_work);
+ armada_drm_plane_work_wait(drm_to_armada_plane(dcrtc->crtc.primary),
+ MAX_SCHEDULE_TIMEOUT);
/* Take a reference to the new fb as we're using it */
drm_framebuffer_reference(crtc->primary->fb);
return 0;
}
+void armada_drm_crtc_plane_disable(struct armada_crtc *dcrtc,
+ struct drm_plane *plane)
+{
+ u32 sram_para1, dma_ctrl0_mask;
+
+ /*
+ * Drop our reference on any framebuffer attached to this plane.
+ * We don't need to NULL this out as drm_plane_force_disable(),
+ * and __setplane_internal() will do so for an overlay plane, and
+ * __drm_helper_disable_unused_functions() will do so for the
+ * primary plane.
+ */
+ if (plane->fb)
+ drm_framebuffer_unreference(plane->fb);
+
+ /* Power down the Y/U/V FIFOs */
+ sram_para1 = CFG_PDWN16x66 | CFG_PDWN32x66;
+
+ /* Power down most RAMs and FIFOs if this is the primary plane */
+ if (plane->type == DRM_PLANE_TYPE_PRIMARY) {
+ sram_para1 |= CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
+ CFG_PDWN32x32 | CFG_PDWN64x66;
+ dma_ctrl0_mask = CFG_GRA_ENA;
+ } else {
+ dma_ctrl0_mask = CFG_DMA_ENA;
+ }
+
+ spin_lock_irq(&dcrtc->irq_lock);
+ armada_updatel(0, dma_ctrl0_mask, dcrtc->base + LCD_SPU_DMA_CTRL0);
+ spin_unlock_irq(&dcrtc->irq_lock);
+
+ armada_updatel(sram_para1, 0, dcrtc->base + LCD_SPU_SRAM_PARA1);
+}
+
/* The mode_config.mutex will be held for this call */
static void armada_drm_crtc_disable(struct drm_crtc *crtc)
{
struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
armada_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
- armada_drm_crtc_finish_fb(dcrtc, crtc->primary->fb, true);
-
- /* Power down most RAMs and FIFOs */
- writel_relaxed(CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
- CFG_PDWN32x32 | CFG_PDWN16x66 | CFG_PDWN32x66 |
- CFG_PDWN64x66, dcrtc->base + LCD_SPU_SRAM_PARA1);
+ armada_drm_crtc_plane_disable(dcrtc, crtc->primary);
}
static const struct drm_crtc_helper_funcs armada_crtc_helper_funcs = {
{
struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
struct armada_frame_work *work;
- struct drm_device *dev = crtc->dev;
- unsigned long flags;
unsigned i;
int ret;
if (!work)
return -ENOMEM;
+ work->work.fn = armada_drm_crtc_complete_frame_work;
work->event = event;
work->old_fb = dcrtc->crtc.primary->fb;
* Finally, if the display is blanked, we won't receive an
* interrupt, so complete it now.
*/
- if (dpms_blanked(dcrtc->dpms)) {
- spin_lock_irqsave(&dev->event_lock, flags);
- if (dcrtc->frame_work)
- armada_drm_crtc_complete_frame_work(dcrtc);
- spin_unlock_irqrestore(&dev->event_lock, flags);
- }
+ if (dpms_blanked(dcrtc->dpms))
+ armada_drm_plane_work_run(dcrtc, drm_to_armada_plane(dcrtc->crtc.primary));
return 0;
}
.set_property = armada_drm_crtc_set_property,
};
+static const struct drm_plane_funcs armada_primary_plane_funcs = {
+ .update_plane = drm_primary_helper_update,
+ .disable_plane = drm_primary_helper_disable,
+ .destroy = drm_primary_helper_destroy,
+};
+
+int armada_drm_plane_init(struct armada_plane *plane)
+{
+ init_waitqueue_head(&plane->frame_wait);
+
+ return 0;
+}
+
static struct drm_prop_enum_list armada_drm_csc_yuv_enum_list[] = {
{ CSC_AUTO, "Auto" },
{ CSC_YUV_CCIR601, "CCIR601" },
return 0;
}
-int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
+static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
struct resource *res, int irq, const struct armada_variant *variant,
struct device_node *port)
{
struct armada_private *priv = drm->dev_private;
struct armada_crtc *dcrtc;
+ struct armada_plane *primary;
void __iomem *base;
int ret;
dcrtc->spu_iopad_ctrl = CFG_VSCALE_LN_EN | CFG_IOPAD_DUMB24;
spin_lock_init(&dcrtc->irq_lock);
dcrtc->irq_ena = CLEAN_SPU_IRQ_ISR;
- INIT_LIST_HEAD(&dcrtc->vbl_list);
- init_waitqueue_head(&dcrtc->frame_wait);
/* Initialize some registers which we don't otherwise set */
writel_relaxed(0x00000001, dcrtc->base + LCD_CFG_SCLK_DIV);
priv->dcrtc[dcrtc->num] = dcrtc;
dcrtc->crtc.port = port;
- drm_crtc_init(drm, &dcrtc->crtc, &armada_crtc_funcs);
+
+ primary = kzalloc(sizeof(*primary), GFP_KERNEL);
+ if (!primary)
+ return -ENOMEM;
+
+ ret = armada_drm_plane_init(primary);
+ if (ret) {
+ kfree(primary);
+ return ret;
+ }
+
+ ret = drm_universal_plane_init(drm, &primary->base, 0,
+ &armada_primary_plane_funcs,
+ armada_primary_formats,
+ ARRAY_SIZE(armada_primary_formats),
+ DRM_PLANE_TYPE_PRIMARY);
+ if (ret) {
+ kfree(primary);
+ return ret;
+ }
+
+ ret = drm_crtc_init_with_planes(drm, &dcrtc->crtc, &primary->base, NULL,
+ &armada_crtc_funcs);
+ if (ret)
+ goto err_crtc_init;
+
drm_crtc_helper_add(&dcrtc->crtc, &armada_crtc_helper_funcs);
drm_object_attach_property(&dcrtc->crtc.base, priv->csc_yuv_prop,
dcrtc->csc_rgb_mode);
return armada_overlay_plane_create(drm, 1 << dcrtc->num);
+
+err_crtc_init:
+ primary->base.funcs->destroy(&primary->base);
+ return ret;
}
static int
#define armada_reg_queue_end(_r, _i) \
armada_reg_queue_mod(_r, _i, 0, 0, ~0)
-struct armada_frame_work;
+struct armada_crtc;
+struct armada_plane;
struct armada_variant;
+struct armada_plane_work {
+ void (*fn)(struct armada_crtc *,
+ struct armada_plane *,
+ struct armada_plane_work *);
+};
+
+struct armada_plane {
+ struct drm_plane base;
+ wait_queue_head_t frame_wait;
+ struct armada_plane_work *work;
+};
+#define drm_to_armada_plane(p) container_of(p, struct armada_plane, base)
+
+int armada_drm_plane_init(struct armada_plane *plane);
+int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
+ struct armada_plane *plane, struct armada_plane_work *work);
+int armada_drm_plane_work_wait(struct armada_plane *plane, long timeout);
+struct armada_plane_work *armada_drm_plane_work_cancel(
+ struct armada_crtc *dcrtc, struct armada_plane *plane);
+
struct armada_crtc {
struct drm_crtc crtc;
const struct armada_variant *variant;
uint32_t dumb_ctrl;
uint32_t spu_iopad_ctrl;
- wait_queue_head_t frame_wait;
- struct armada_frame_work *frame_work;
-
spinlock_t irq_lock;
uint32_t irq_ena;
- struct list_head vbl_list;
};
#define drm_to_armada_crtc(c) container_of(c, struct armada_crtc, crtc)
-struct device_node;
-int armada_drm_crtc_create(struct drm_device *, struct device *,
- struct resource *, int, const struct armada_variant *,
- struct device_node *);
void armada_drm_crtc_gamma_set(struct drm_crtc *, u16, u16, u16, int);
void armada_drm_crtc_gamma_get(struct drm_crtc *, u16 *, u16 *, u16 *, int);
void armada_drm_crtc_disable_irq(struct armada_crtc *, u32);
void armada_drm_crtc_enable_irq(struct armada_crtc *, u32);
void armada_drm_crtc_update_regs(struct armada_crtc *, struct armada_regs *);
+void armada_drm_crtc_plane_disable(struct armada_crtc *dcrtc,
+ struct drm_plane *plane);
+
extern struct platform_driver armada_lcd_platform_driver;
#endif
return ALIGN(pitch, 128);
}
-struct armada_vbl_event {
- struct list_head node;
- void *data;
- void (*fn)(struct armada_crtc *, void *);
-};
-void armada_drm_vbl_event_add(struct armada_crtc *,
- struct armada_vbl_event *);
-void armada_drm_vbl_event_remove(struct armada_crtc *,
- struct armada_vbl_event *);
-#define armada_drm_vbl_event_init(_e, _f, _d) do { \
- struct armada_vbl_event *__e = _e; \
- INIT_LIST_HEAD(&__e->node); \
- __e->data = _d; \
- __e->fn = _f; \
-} while (0)
-
struct armada_private;
#include <linux/of_graph.h>
#include <drm/drmP.h>
#include <drm/drm_crtc_helper.h>
+#include <drm/drm_of.h>
#include "armada_crtc.h"
#include "armada_drm.h"
#include "armada_gem.h"
#include <drm/armada_drm.h>
#include "armada_ioctlP.h"
-#ifdef CONFIG_DRM_ARMADA_TDA1998X
-#include <drm/i2c/tda998x.h>
-#include "armada_slave.h"
-
-static struct tda998x_encoder_params params = {
- /* With 0x24, there is no translation between vp_out and int_vp
- FB LCD out Pins VIP Int Vp
- R:23:16 R:7:0 VPC7:0 7:0 7:0[R]
- G:15:8 G:15:8 VPB7:0 23:16 23:16[G]
- B:7:0 B:23:16 VPA7:0 15:8 15:8[B]
- */
- .swap_a = 2,
- .swap_b = 3,
- .swap_c = 4,
- .swap_d = 5,
- .swap_e = 0,
- .swap_f = 1,
- .audio_cfg = BIT(2),
- .audio_frame[1] = 1,
- .audio_format = AFMT_SPDIF,
- .audio_sample_rate = 44100,
-};
-
-static const struct armada_drm_slave_config tda19988_config = {
- .i2c_adapter_id = 0,
- .crtcs = 1 << 0, /* Only LCD0 at the moment */
- .polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT,
- .interlace_allowed = true,
- .info = {
- .type = "tda998x",
- .addr = 0x70,
- .platform_data = ¶ms,
- },
-};
-#endif
-
-static bool is_componentized(struct device *dev)
-{
- return dev->of_node || dev->platform_data;
-}
-
static void armada_drm_unref_work(struct work_struct *work)
{
struct armada_private *priv =
static int armada_drm_load(struct drm_device *dev, unsigned long flags)
{
- const struct platform_device_id *id;
- const struct armada_variant *variant;
struct armada_private *priv;
- struct resource *res[ARRAY_SIZE(priv->dcrtc)];
struct resource *mem = NULL;
- int ret, n, i;
-
- memset(res, 0, sizeof(res));
+ int ret, n;
- for (n = i = 0; ; n++) {
+ for (n = 0; ; n++) {
struct resource *r = platform_get_resource(dev->platformdev,
IORESOURCE_MEM, n);
if (!r)
/* Resources above 64K are graphics memory */
if (resource_size(r) > SZ_64K)
mem = r;
- else if (i < ARRAY_SIZE(priv->dcrtc))
- res[i++] = r;
else
return -EINVAL;
}
platform_set_drvdata(dev->platformdev, dev);
dev->dev_private = priv;
- /* Get the implementation specific driver data. */
- id = platform_get_device_id(dev->platformdev);
- if (!id)
- return -ENXIO;
-
- variant = (const struct armada_variant *)id->driver_data;
-
INIT_WORK(&priv->fb_unref_work, armada_drm_unref_work);
INIT_KFIFO(priv->fb_unref);
dev->mode_config.funcs = &armada_drm_mode_config_funcs;
drm_mm_init(&priv->linear, mem->start, resource_size(mem));
- /* Create all LCD controllers */
- for (n = 0; n < ARRAY_SIZE(priv->dcrtc); n++) {
- int irq;
-
- if (!res[n])
- break;
-
- irq = platform_get_irq(dev->platformdev, n);
- if (irq < 0)
- goto err_kms;
-
- ret = armada_drm_crtc_create(dev, dev->dev, res[n], irq,
- variant, NULL);
- if (ret)
- goto err_kms;
- }
-
- if (is_componentized(dev->dev)) {
- ret = component_bind_all(dev->dev, dev);
- if (ret)
- goto err_kms;
- } else {
-#ifdef CONFIG_DRM_ARMADA_TDA1998X
- ret = armada_drm_connector_slave_create(dev, &tda19988_config);
- if (ret)
- goto err_kms;
-#endif
- }
+ ret = component_bind_all(dev->dev, dev);
+ if (ret)
+ goto err_kms;
ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
if (ret)
return 0;
err_comp:
- if (is_componentized(dev->dev))
- component_unbind_all(dev->dev, dev);
+ component_unbind_all(dev->dev, dev);
err_kms:
drm_mode_config_cleanup(dev);
drm_mm_takedown(&priv->linear);
drm_kms_helper_poll_fini(dev);
armada_fbdev_fini(dev);
- if (is_componentized(dev->dev))
- component_unbind_all(dev->dev, dev);
+ component_unbind_all(dev->dev, dev);
drm_mode_config_cleanup(dev);
drm_mm_takedown(&priv->linear);
return 0;
}
-void armada_drm_vbl_event_add(struct armada_crtc *dcrtc,
- struct armada_vbl_event *evt)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&dcrtc->irq_lock, flags);
- if (list_empty(&evt->node)) {
- list_add_tail(&evt->node, &dcrtc->vbl_list);
-
- drm_vblank_get(dcrtc->crtc.dev, dcrtc->num);
- }
- spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
-}
-
-void armada_drm_vbl_event_remove(struct armada_crtc *dcrtc,
- struct armada_vbl_event *evt)
-{
- if (!list_empty(&evt->node)) {
- list_del_init(&evt->node);
- drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
- }
-}
-
/* These are called under the vbl_lock. */
-static int armada_drm_enable_vblank(struct drm_device *dev, int crtc)
+static int armada_drm_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct armada_private *priv = dev->dev_private;
- armada_drm_crtc_enable_irq(priv->dcrtc[crtc], VSYNC_IRQ_ENA);
+ armada_drm_crtc_enable_irq(priv->dcrtc[pipe], VSYNC_IRQ_ENA);
return 0;
}
-static void armada_drm_disable_vblank(struct drm_device *dev, int crtc)
+static void armada_drm_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct armada_private *priv = dev->dev_private;
- armada_drm_crtc_disable_irq(priv->dcrtc[crtc], VSYNC_IRQ_ENA);
+ armada_drm_crtc_disable_irq(priv->dcrtc[pipe], VSYNC_IRQ_ENA);
}
static struct drm_ioctl_desc armada_ioctls[] = {
- DRM_IOCTL_DEF_DRV(ARMADA_GEM_CREATE, armada_gem_create_ioctl,
- DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(ARMADA_GEM_MMAP, armada_gem_mmap_ioctl,
- DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(ARMADA_GEM_PWRITE, armada_gem_pwrite_ioctl,
- DRM_UNLOCKED),
+ DRM_IOCTL_DEF_DRV(ARMADA_GEM_CREATE, armada_gem_create_ioctl,0),
+ DRM_IOCTL_DEF_DRV(ARMADA_GEM_MMAP, armada_gem_mmap_ioctl, 0),
+ DRM_IOCTL_DEF_DRV(ARMADA_GEM_PWRITE, armada_gem_pwrite_ioctl, 0),
};
static void armada_drm_lastclose(struct drm_device *dev)
.lastclose = armada_drm_lastclose,
.unload = armada_drm_unload,
.set_busid = drm_platform_set_busid,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = armada_drm_enable_vblank,
.disable_vblank = armada_drm_disable_vblank,
#ifdef CONFIG_DEBUG_FS
}
}
-static int armada_drm_find_components(struct device *dev,
- struct component_match **match)
-{
- struct device_node *port;
- int i;
-
- if (dev->of_node) {
- struct device_node *np = dev->of_node;
-
- for (i = 0; ; i++) {
- port = of_parse_phandle(np, "ports", i);
- if (!port)
- break;
-
- component_match_add(dev, match, compare_of, port);
- of_node_put(port);
- }
+static const struct component_master_ops armada_master_ops = {
+ .bind = armada_drm_bind,
+ .unbind = armada_drm_unbind,
+};
- if (i == 0) {
- dev_err(dev, "missing 'ports' property\n");
- return -ENODEV;
- }
+static int armada_drm_probe(struct platform_device *pdev)
+{
+ struct component_match *match = NULL;
+ struct device *dev = &pdev->dev;
+ int ret;
- for (i = 0; ; i++) {
- port = of_parse_phandle(np, "ports", i);
- if (!port)
- break;
+ ret = drm_of_component_probe(dev, compare_dev_name, &armada_master_ops);
+ if (ret != -EINVAL)
+ return ret;
- armada_add_endpoints(dev, match, port);
- of_node_put(port);
- }
- } else if (dev->platform_data) {
+ if (dev->platform_data) {
char **devices = dev->platform_data;
+ struct device_node *port;
struct device *d;
+ int i;
for (i = 0; devices[i]; i++)
- component_match_add(dev, match, compare_dev_name,
+ component_match_add(dev, &match, compare_dev_name,
devices[i]);
if (i == 0) {
for (i = 0; devices[i]; i++) {
d = bus_find_device_by_name(&platform_bus_type, NULL,
- devices[i]);
+ devices[i]);
if (d && d->of_node) {
for_each_child_of_node(d->of_node, port)
- armada_add_endpoints(dev, match, port);
+ armada_add_endpoints(dev, &match, port);
}
put_device(d);
}
}
- return 0;
-}
-
-static const struct component_master_ops armada_master_ops = {
- .bind = armada_drm_bind,
- .unbind = armada_drm_unbind,
-};
-
-static int armada_drm_probe(struct platform_device *pdev)
-{
- if (is_componentized(&pdev->dev)) {
- struct component_match *match = NULL;
- int ret;
-
- ret = armada_drm_find_components(&pdev->dev, &match);
- if (ret < 0)
- return ret;
-
- return component_master_add_with_match(&pdev->dev,
- &armada_master_ops, match);
- } else {
- return drm_platform_init(&armada_drm_driver, pdev);
- }
+ return component_master_add_with_match(&pdev->dev, &armada_master_ops,
+ match);
}
static int armada_drm_remove(struct platform_device *pdev)
{
- if (is_componentized(&pdev->dev))
- component_master_del(&pdev->dev, &armada_master_ops);
- else
- drm_put_dev(platform_get_drvdata(pdev));
+ component_master_del(&pdev->dev, &armada_master_ops);
return 0;
}
static const struct platform_device_id armada_drm_platform_ids[] = {
{
.name = "armada-drm",
- .driver_data = (unsigned long)&armada510_ops,
}, {
.name = "armada-510-drm",
- .driver_data = (unsigned long)&armada510_ops,
},
{ },
};
+++ /dev/null
-/*
- * Copyright (C) 2012 Russell King
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <drm/drmP.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_edid.h>
-#include <drm/drm_encoder_slave.h>
-#include "armada_output.h"
-#include "armada_drm.h"
-
-struct armada_connector {
- struct drm_connector conn;
- const struct armada_output_type *type;
-};
-
-#define drm_to_armada_conn(c) container_of(c, struct armada_connector, conn)
-
-struct drm_encoder *armada_drm_connector_encoder(struct drm_connector *conn)
-{
- struct drm_encoder *enc = conn->encoder;
-
- return enc ? enc : drm_encoder_find(conn->dev, conn->encoder_ids[0]);
-}
-
-static enum drm_connector_status armada_drm_connector_detect(
- struct drm_connector *conn, bool force)
-{
- struct armada_connector *dconn = drm_to_armada_conn(conn);
- enum drm_connector_status status = connector_status_disconnected;
-
- if (dconn->type->detect) {
- status = dconn->type->detect(conn, force);
- } else {
- struct drm_encoder *enc = armada_drm_connector_encoder(conn);
-
- if (enc)
- status = encoder_helper_funcs(enc)->detect(enc, conn);
- }
-
- return status;
-}
-
-static void armada_drm_connector_destroy(struct drm_connector *conn)
-{
- struct armada_connector *dconn = drm_to_armada_conn(conn);
-
- drm_connector_unregister(conn);
- drm_connector_cleanup(conn);
- kfree(dconn);
-}
-
-static int armada_drm_connector_set_property(struct drm_connector *conn,
- struct drm_property *property, uint64_t value)
-{
- struct armada_connector *dconn = drm_to_armada_conn(conn);
-
- if (!dconn->type->set_property)
- return -EINVAL;
-
- return dconn->type->set_property(conn, property, value);
-}
-
-static const struct drm_connector_funcs armada_drm_conn_funcs = {
- .dpms = drm_helper_connector_dpms,
- .fill_modes = drm_helper_probe_single_connector_modes,
- .detect = armada_drm_connector_detect,
- .destroy = armada_drm_connector_destroy,
- .set_property = armada_drm_connector_set_property,
-};
-
-/* Shouldn't this be a generic helper function? */
-int armada_drm_slave_encoder_mode_valid(struct drm_connector *conn,
- struct drm_display_mode *mode)
-{
- struct drm_encoder *encoder = armada_drm_connector_encoder(conn);
- int valid = MODE_BAD;
-
- if (encoder) {
- struct drm_encoder_slave *slave = to_encoder_slave(encoder);
-
- valid = slave->slave_funcs->mode_valid(encoder, mode);
- }
- return valid;
-}
-
-int armada_drm_slave_encoder_set_property(struct drm_connector *conn,
- struct drm_property *property, uint64_t value)
-{
- struct drm_encoder *encoder = armada_drm_connector_encoder(conn);
- int rc = -EINVAL;
-
- if (encoder) {
- struct drm_encoder_slave *slave = to_encoder_slave(encoder);
-
- rc = slave->slave_funcs->set_property(encoder, conn, property,
- value);
- }
- return rc;
-}
-
-int armada_output_create(struct drm_device *dev,
- const struct armada_output_type *type, const void *data)
-{
- struct armada_connector *dconn;
- int ret;
-
- dconn = kzalloc(sizeof(*dconn), GFP_KERNEL);
- if (!dconn)
- return -ENOMEM;
-
- dconn->type = type;
-
- ret = drm_connector_init(dev, &dconn->conn, &armada_drm_conn_funcs,
- type->connector_type);
- if (ret) {
- DRM_ERROR("unable to init connector\n");
- goto err_destroy_dconn;
- }
-
- ret = type->create(&dconn->conn, data);
- if (ret)
- goto err_conn;
-
- ret = drm_connector_register(&dconn->conn);
- if (ret)
- goto err_sysfs;
-
- return 0;
-
- err_sysfs:
- if (dconn->conn.encoder)
- dconn->conn.encoder->funcs->destroy(dconn->conn.encoder);
- err_conn:
- drm_connector_cleanup(&dconn->conn);
- err_destroy_dconn:
- kfree(dconn);
- return ret;
-}
+++ /dev/null
-/*
- * Copyright (C) 2012 Russell King
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#ifndef ARMADA_CONNETOR_H
-#define ARMADA_CONNETOR_H
-
-#define encoder_helper_funcs(encoder) \
- ((const struct drm_encoder_helper_funcs *)encoder->helper_private)
-
-struct armada_output_type {
- int connector_type;
- enum drm_connector_status (*detect)(struct drm_connector *, bool);
- int (*create)(struct drm_connector *, const void *);
- int (*set_property)(struct drm_connector *, struct drm_property *,
- uint64_t);
-};
-
-struct drm_encoder *armada_drm_connector_encoder(struct drm_connector *conn);
-
-int armada_drm_slave_encoder_mode_valid(struct drm_connector *conn,
- struct drm_display_mode *mode);
-
-int armada_drm_slave_encoder_set_property(struct drm_connector *conn,
- struct drm_property *property, uint64_t value);
-
-int armada_output_create(struct drm_device *dev,
- const struct armada_output_type *type, const void *data);
-
-#endif
#include <drm/armada_drm.h>
#include "armada_ioctlP.h"
-struct armada_plane_properties {
+struct armada_ovl_plane_properties {
uint32_t colorkey_yr;
uint32_t colorkey_ug;
uint32_t colorkey_vb;
uint32_t colorkey_mode;
};
-struct armada_plane {
- struct drm_plane base;
- spinlock_t lock;
+struct armada_ovl_plane {
+ struct armada_plane base;
struct drm_framebuffer *old_fb;
uint32_t src_hw;
uint32_t dst_hw;
uint32_t dst_yx;
uint32_t ctrl0;
struct {
- struct armada_vbl_event update;
+ struct armada_plane_work work;
struct armada_regs regs[13];
- wait_queue_head_t wait;
} vbl;
- struct armada_plane_properties prop;
+ struct armada_ovl_plane_properties prop;
};
-#define drm_to_armada_plane(p) container_of(p, struct armada_plane, base)
+#define drm_to_armada_ovl_plane(p) \
+ container_of(p, struct armada_ovl_plane, base.base)
static void
-armada_ovl_update_attr(struct armada_plane_properties *prop,
+armada_ovl_update_attr(struct armada_ovl_plane_properties *prop,
struct armada_crtc *dcrtc)
{
writel_relaxed(prop->colorkey_yr, dcrtc->base + LCD_SPU_COLORKEY_Y);
spin_unlock_irq(&dcrtc->irq_lock);
}
-/* === Plane support === */
-static void armada_plane_vbl(struct armada_crtc *dcrtc, void *data)
+static void armada_ovl_retire_fb(struct armada_ovl_plane *dplane,
+ struct drm_framebuffer *fb)
{
- struct armada_plane *dplane = data;
- struct drm_framebuffer *fb;
+ struct drm_framebuffer *old_fb;
- armada_drm_crtc_update_regs(dcrtc, dplane->vbl.regs);
+ old_fb = xchg(&dplane->old_fb, fb);
- spin_lock(&dplane->lock);
- fb = dplane->old_fb;
- dplane->old_fb = NULL;
- spin_unlock(&dplane->lock);
+ if (old_fb)
+ armada_drm_queue_unref_work(dplane->base.base.dev, old_fb);
+}
- if (fb)
- armada_drm_queue_unref_work(dcrtc->crtc.dev, fb);
+/* === Plane support === */
+static void armada_ovl_plane_work(struct armada_crtc *dcrtc,
+ struct armada_plane *plane, struct armada_plane_work *work)
+{
+ struct armada_ovl_plane *dplane = container_of(plane, struct armada_ovl_plane, base);
- wake_up(&dplane->vbl.wait);
+ armada_drm_crtc_update_regs(dcrtc, dplane->vbl.regs);
+ armada_ovl_retire_fb(dplane, NULL);
}
static int
-armada_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
+armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
struct drm_framebuffer *fb,
int crtc_x, int crtc_y, unsigned crtc_w, unsigned crtc_h,
uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h)
{
- struct armada_plane *dplane = drm_to_armada_plane(plane);
+ struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
struct drm_rect src = {
.x1 = src_x,
dcrtc->base + LCD_SPU_SRAM_PARA1);
}
- wait_event_timeout(dplane->vbl.wait,
- list_empty(&dplane->vbl.update.node),
- HZ/25);
+ if (armada_drm_plane_work_wait(&dplane->base, HZ / 25) == 0)
+ armada_drm_plane_work_cancel(dcrtc, &dplane->base);
if (plane->fb != fb) {
struct armada_gem_object *obj = drm_fb_obj(fb);
*/
drm_framebuffer_reference(fb);
- if (plane->fb) {
- struct drm_framebuffer *older_fb;
-
- spin_lock_irq(&dplane->lock);
- older_fb = dplane->old_fb;
- dplane->old_fb = plane->fb;
- spin_unlock_irq(&dplane->lock);
- if (older_fb)
- armada_drm_queue_unref_work(dcrtc->crtc.dev,
- older_fb);
- }
+ if (plane->fb)
+ armada_ovl_retire_fb(dplane, plane->fb);
src_y = src.y1 >> 16;
src_x = src.x1 >> 16;
}
if (idx) {
armada_reg_queue_end(dplane->vbl.regs, idx);
- armada_drm_vbl_event_add(dcrtc, &dplane->vbl.update);
+ armada_drm_plane_work_queue(dcrtc, &dplane->base,
+ &dplane->vbl.work);
}
return 0;
}
-static int armada_plane_disable(struct drm_plane *plane)
+static int armada_ovl_plane_disable(struct drm_plane *plane)
{
- struct armada_plane *dplane = drm_to_armada_plane(plane);
+ struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
struct drm_framebuffer *fb;
struct armada_crtc *dcrtc;
- if (!dplane->base.crtc)
+ if (!dplane->base.base.crtc)
return 0;
- dcrtc = drm_to_armada_crtc(dplane->base.crtc);
- dcrtc->plane = NULL;
-
- spin_lock_irq(&dcrtc->irq_lock);
- armada_drm_vbl_event_remove(dcrtc, &dplane->vbl.update);
- armada_updatel(0, CFG_DMA_ENA, dcrtc->base + LCD_SPU_DMA_CTRL0);
- dplane->ctrl0 = 0;
- spin_unlock_irq(&dcrtc->irq_lock);
+ dcrtc = drm_to_armada_crtc(dplane->base.base.crtc);
- /* Power down the Y/U/V FIFOs */
- armada_updatel(CFG_PDWN16x66 | CFG_PDWN32x66, 0,
- dcrtc->base + LCD_SPU_SRAM_PARA1);
+ armada_drm_plane_work_cancel(dcrtc, &dplane->base);
+ armada_drm_crtc_plane_disable(dcrtc, plane);
- if (plane->fb)
- drm_framebuffer_unreference(plane->fb);
+ dcrtc->plane = NULL;
+ dplane->ctrl0 = 0;
- spin_lock_irq(&dplane->lock);
- fb = dplane->old_fb;
- dplane->old_fb = NULL;
- spin_unlock_irq(&dplane->lock);
+ fb = xchg(&dplane->old_fb, NULL);
if (fb)
drm_framebuffer_unreference(fb);
return 0;
}
-static void armada_plane_destroy(struct drm_plane *plane)
+static void armada_ovl_plane_destroy(struct drm_plane *plane)
{
- struct armada_plane *dplane = drm_to_armada_plane(plane);
+ struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
drm_plane_cleanup(plane);
kfree(dplane);
}
-static int armada_plane_set_property(struct drm_plane *plane,
+static int armada_ovl_plane_set_property(struct drm_plane *plane,
struct drm_property *property, uint64_t val)
{
struct armada_private *priv = plane->dev->dev_private;
- struct armada_plane *dplane = drm_to_armada_plane(plane);
+ struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
bool update_attr = false;
if (property == priv->colorkey_prop) {
update_attr = true;
}
- if (update_attr && dplane->base.crtc)
+ if (update_attr && dplane->base.base.crtc)
armada_ovl_update_attr(&dplane->prop,
- drm_to_armada_crtc(dplane->base.crtc));
+ drm_to_armada_crtc(dplane->base.base.crtc));
return 0;
}
-static const struct drm_plane_funcs armada_plane_funcs = {
- .update_plane = armada_plane_update,
- .disable_plane = armada_plane_disable,
- .destroy = armada_plane_destroy,
- .set_property = armada_plane_set_property,
+static const struct drm_plane_funcs armada_ovl_plane_funcs = {
+ .update_plane = armada_ovl_plane_update,
+ .disable_plane = armada_ovl_plane_disable,
+ .destroy = armada_ovl_plane_destroy,
+ .set_property = armada_ovl_plane_set_property,
};
-static const uint32_t armada_formats[] = {
+static const uint32_t armada_ovl_formats[] = {
DRM_FORMAT_UYVY,
DRM_FORMAT_YUYV,
DRM_FORMAT_YUV420,
{
struct armada_private *priv = dev->dev_private;
struct drm_mode_object *mobj;
- struct armada_plane *dplane;
+ struct armada_ovl_plane *dplane;
int ret;
ret = armada_overlay_create_properties(dev);
if (!dplane)
return -ENOMEM;
- spin_lock_init(&dplane->lock);
- init_waitqueue_head(&dplane->vbl.wait);
- armada_drm_vbl_event_init(&dplane->vbl.update, armada_plane_vbl,
- dplane);
+ ret = armada_drm_plane_init(&dplane->base);
+ if (ret) {
+ kfree(dplane);
+ return ret;
+ }
+
+ dplane->vbl.work.fn = armada_ovl_plane_work;
- drm_plane_init(dev, &dplane->base, crtcs, &armada_plane_funcs,
- armada_formats, ARRAY_SIZE(armada_formats), false);
+ ret = drm_universal_plane_init(dev, &dplane->base.base, crtcs,
+ &armada_ovl_plane_funcs,
+ armada_ovl_formats,
+ ARRAY_SIZE(armada_ovl_formats),
+ DRM_PLANE_TYPE_OVERLAY);
+ if (ret) {
+ kfree(dplane);
+ return ret;
+ }
dplane->prop.colorkey_yr = 0xfefefe00;
dplane->prop.colorkey_ug = 0x01010100;
dplane->prop.contrast = 0x4000;
dplane->prop.saturation = 0x4000;
- mobj = &dplane->base.base;
+ mobj = &dplane->base.base.base;
drm_object_attach_property(mobj, priv->colorkey_prop,
0x0101fe);
drm_object_attach_property(mobj, priv->colorkey_min_prop,
+++ /dev/null
-/*
- * Copyright (C) 2012 Russell King
- * Rewritten from the dovefb driver, and Armada510 manuals.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <drm/drmP.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_edid.h>
-#include <drm/drm_encoder_slave.h>
-#include "armada_drm.h"
-#include "armada_output.h"
-#include "armada_slave.h"
-
-static int armada_drm_slave_get_modes(struct drm_connector *conn)
-{
- struct drm_encoder *enc = armada_drm_connector_encoder(conn);
- int count = 0;
-
- if (enc) {
- struct drm_encoder_slave *slave = to_encoder_slave(enc);
-
- count = slave->slave_funcs->get_modes(enc, conn);
- }
-
- return count;
-}
-
-static void armada_drm_slave_destroy(struct drm_encoder *enc)
-{
- struct drm_encoder_slave *slave = to_encoder_slave(enc);
- struct i2c_client *client = drm_i2c_encoder_get_client(enc);
-
- if (slave->slave_funcs)
- slave->slave_funcs->destroy(enc);
- if (client)
- i2c_put_adapter(client->adapter);
-
- drm_encoder_cleanup(&slave->base);
- kfree(slave);
-}
-
-static const struct drm_encoder_funcs armada_drm_slave_encoder_funcs = {
- .destroy = armada_drm_slave_destroy,
-};
-
-static const struct drm_connector_helper_funcs armada_drm_slave_helper_funcs = {
- .get_modes = armada_drm_slave_get_modes,
- .mode_valid = armada_drm_slave_encoder_mode_valid,
- .best_encoder = armada_drm_connector_encoder,
-};
-
-static const struct drm_encoder_helper_funcs drm_slave_encoder_helpers = {
- .dpms = drm_i2c_encoder_dpms,
- .save = drm_i2c_encoder_save,
- .restore = drm_i2c_encoder_restore,
- .mode_fixup = drm_i2c_encoder_mode_fixup,
- .prepare = drm_i2c_encoder_prepare,
- .commit = drm_i2c_encoder_commit,
- .mode_set = drm_i2c_encoder_mode_set,
- .detect = drm_i2c_encoder_detect,
-};
-
-static int
-armada_drm_conn_slave_create(struct drm_connector *conn, const void *data)
-{
- const struct armada_drm_slave_config *config = data;
- struct drm_encoder_slave *slave;
- struct i2c_adapter *adap;
- int ret;
-
- conn->interlace_allowed = config->interlace_allowed;
- conn->doublescan_allowed = config->doublescan_allowed;
- conn->polled = config->polled;
-
- drm_connector_helper_add(conn, &armada_drm_slave_helper_funcs);
-
- slave = kzalloc(sizeof(*slave), GFP_KERNEL);
- if (!slave)
- return -ENOMEM;
-
- slave->base.possible_crtcs = config->crtcs;
-
- adap = i2c_get_adapter(config->i2c_adapter_id);
- if (!adap) {
- kfree(slave);
- return -EPROBE_DEFER;
- }
-
- ret = drm_encoder_init(conn->dev, &slave->base,
- &armada_drm_slave_encoder_funcs,
- DRM_MODE_ENCODER_TMDS);
- if (ret) {
- DRM_ERROR("unable to init encoder\n");
- i2c_put_adapter(adap);
- kfree(slave);
- return ret;
- }
-
- ret = drm_i2c_encoder_init(conn->dev, slave, adap, &config->info);
- i2c_put_adapter(adap);
- if (ret) {
- DRM_ERROR("unable to init encoder slave\n");
- armada_drm_slave_destroy(&slave->base);
- return ret;
- }
-
- drm_encoder_helper_add(&slave->base, &drm_slave_encoder_helpers);
-
- ret = slave->slave_funcs->create_resources(&slave->base, conn);
- if (ret) {
- armada_drm_slave_destroy(&slave->base);
- return ret;
- }
-
- ret = drm_mode_connector_attach_encoder(conn, &slave->base);
- if (ret) {
- armada_drm_slave_destroy(&slave->base);
- return ret;
- }
-
- conn->encoder = &slave->base;
-
- return ret;
-}
-
-static const struct armada_output_type armada_drm_conn_slave = {
- .connector_type = DRM_MODE_CONNECTOR_HDMIA,
- .create = armada_drm_conn_slave_create,
- .set_property = armada_drm_slave_encoder_set_property,
-};
-
-int armada_drm_connector_slave_create(struct drm_device *dev,
- const struct armada_drm_slave_config *config)
-{
- return armada_output_create(dev, &armada_drm_conn_slave, config);
-}
+++ /dev/null
-/*
- * Copyright (C) 2012 Russell King
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#ifndef ARMADA_SLAVE_H
-#define ARMADA_SLAVE_H
-
-#include <linux/i2c.h>
-#include <drm/drmP.h>
-
-struct armada_drm_slave_config {
- int i2c_adapter_id;
- uint32_t crtcs;
- uint8_t polled;
- bool interlace_allowed;
- bool doublescan_allowed;
- struct i2c_board_info info;
-};
-
-int armada_drm_connector_slave_create(struct drm_device *dev,
- const struct armada_drm_slave_config *);
-
-#endif
regmap_read(dc->hlcdc->regmap, ATMEL_HLCDC_ISR, &isr);
}
-static int atmel_hlcdc_dc_enable_vblank(struct drm_device *dev, int crtc)
+static int atmel_hlcdc_dc_enable_vblank(struct drm_device *dev,
+ unsigned int pipe)
{
struct atmel_hlcdc_dc *dc = dev->dev_private;
return 0;
}
-static void atmel_hlcdc_dc_disable_vblank(struct drm_device *dev, int crtc)
+static void atmel_hlcdc_dc_disable_vblank(struct drm_device *dev,
+ unsigned int pipe)
{
struct atmel_hlcdc_dc *dc = dev->dev_private;
.irq_preinstall = atmel_hlcdc_dc_irq_uninstall,
.irq_postinstall = atmel_hlcdc_dc_irq_postinstall,
.irq_uninstall = atmel_hlcdc_dc_irq_uninstall,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = atmel_hlcdc_dc_enable_vblank,
.disable_vblank = atmel_hlcdc_dc_disable_vblank,
.gem_free_object = drm_gem_cma_free_object,
if (!state->bpp[i])
return -EINVAL;
- switch (state->base.rotation & 0xf) {
+ switch (state->base.rotation & DRM_ROTATE_MASK) {
case BIT(DRM_ROTATE_90):
offset = ((y_offset + state->src_y + patched_src_w - 1) /
ydiv) * fb->pitches[i];
}
static int atmel_hlcdc_plane_prepare_fb(struct drm_plane *p,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p);
+ if (!new_state->fb)
+ return 0;
+
return atmel_hlcdc_layer_update_start(&plane->layer);
}
tristate
select DRM_KMS_HELPER
+config DRM_DW_HDMI_AHB_AUDIO
+ tristate "Synopsis Designware AHB Audio interface"
+ depends on DRM_DW_HDMI && SND
+ select SND_PCM
+ select SND_PCM_ELD
+ select SND_PCM_IEC958
+ help
+ Support the AHB Audio interface which is part of the Synopsis
+ Designware HDMI block. This is used in conjunction with
+ the i.MX6 HDMI driver.
+
+
config DRM_NXP_PTN3460
tristate "NXP PTN3460 DP/LVDS bridge"
depends on OF
ccflags-y := -Iinclude/drm
obj-$(CONFIG_DRM_DW_HDMI) += dw_hdmi.o
+obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw_hdmi-ahb-audio.o
obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
--- /dev/null
+/*
+ * DesignWare HDMI audio driver
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written and tested against the Designware HDMI Tx found in iMX6.
+ */
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <drm/bridge/dw_hdmi.h>
+#include <drm/drm_edid.h>
+
+#include <sound/asoundef.h>
+#include <sound/core.h>
+#include <sound/initval.h>
+#include <sound/pcm.h>
+#include <sound/pcm_drm_eld.h>
+#include <sound/pcm_iec958.h>
+
+#include "dw_hdmi-audio.h"
+
+#define DRIVER_NAME "dw-hdmi-ahb-audio"
+
+/* Provide some bits rather than bit offsets */
+enum {
+ HDMI_AHB_DMA_CONF0_SW_FIFO_RST = BIT(7),
+ HDMI_AHB_DMA_CONF0_EN_HLOCK = BIT(3),
+ HDMI_AHB_DMA_START_START = BIT(0),
+ HDMI_AHB_DMA_STOP_STOP = BIT(0),
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_ERROR = BIT(5),
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_LOST = BIT(4),
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_RETRY = BIT(3),
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_DONE = BIT(2),
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFFULL = BIT(1),
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFEMPTY = BIT(0),
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_ALL =
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_ERROR |
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_LOST |
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_RETRY |
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_DONE |
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFFULL |
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFEMPTY,
+ HDMI_IH_AHBDMAAUD_STAT0_ERROR = BIT(5),
+ HDMI_IH_AHBDMAAUD_STAT0_LOST = BIT(4),
+ HDMI_IH_AHBDMAAUD_STAT0_RETRY = BIT(3),
+ HDMI_IH_AHBDMAAUD_STAT0_DONE = BIT(2),
+ HDMI_IH_AHBDMAAUD_STAT0_BUFFFULL = BIT(1),
+ HDMI_IH_AHBDMAAUD_STAT0_BUFFEMPTY = BIT(0),
+ HDMI_IH_AHBDMAAUD_STAT0_ALL =
+ HDMI_IH_AHBDMAAUD_STAT0_ERROR |
+ HDMI_IH_AHBDMAAUD_STAT0_LOST |
+ HDMI_IH_AHBDMAAUD_STAT0_RETRY |
+ HDMI_IH_AHBDMAAUD_STAT0_DONE |
+ HDMI_IH_AHBDMAAUD_STAT0_BUFFFULL |
+ HDMI_IH_AHBDMAAUD_STAT0_BUFFEMPTY,
+ HDMI_AHB_DMA_CONF0_INCR16 = 2 << 1,
+ HDMI_AHB_DMA_CONF0_INCR8 = 1 << 1,
+ HDMI_AHB_DMA_CONF0_INCR4 = 0,
+ HDMI_AHB_DMA_CONF0_BURST_MODE = BIT(0),
+ HDMI_AHB_DMA_MASK_DONE = BIT(7),
+
+ HDMI_REVISION_ID = 0x0001,
+ HDMI_IH_AHBDMAAUD_STAT0 = 0x0109,
+ HDMI_IH_MUTE_AHBDMAAUD_STAT0 = 0x0189,
+ HDMI_FC_AUDICONF2 = 0x1027,
+ HDMI_FC_AUDSCONF = 0x1063,
+ HDMI_FC_AUDSCONF_LAYOUT1 = 1 << 0,
+ HDMI_FC_AUDSCONF_LAYOUT0 = 0 << 0,
+ HDMI_AHB_DMA_CONF0 = 0x3600,
+ HDMI_AHB_DMA_START = 0x3601,
+ HDMI_AHB_DMA_STOP = 0x3602,
+ HDMI_AHB_DMA_THRSLD = 0x3603,
+ HDMI_AHB_DMA_STRADDR0 = 0x3604,
+ HDMI_AHB_DMA_STPADDR0 = 0x3608,
+ HDMI_AHB_DMA_MASK = 0x3614,
+ HDMI_AHB_DMA_POL = 0x3615,
+ HDMI_AHB_DMA_CONF1 = 0x3616,
+ HDMI_AHB_DMA_BUFFPOL = 0x361a,
+};
+
+struct dw_hdmi_channel_conf {
+ u8 conf1;
+ u8 ca;
+};
+
+/*
+ * The default mapping of ALSA channels to HDMI channels and speaker
+ * allocation bits. Note that we can't do channel remapping here -
+ * channels must be in the same order.
+ *
+ * Mappings for alsa-lib pcm/surround*.conf files:
+ *
+ * Front Sur4.0 Sur4.1 Sur5.0 Sur5.1 Sur7.1
+ * Channels 2 4 6 6 6 8
+ *
+ * Our mapping from ALSA channel to CEA686D speaker name and HDMI channel:
+ *
+ * Number of ALSA channels
+ * ALSA Channel 2 3 4 5 6 7 8
+ * 0 FL:0 = = = = = =
+ * 1 FR:1 = = = = = =
+ * 2 FC:3 RL:4 LFE:2 = = =
+ * 3 RR:5 RL:4 FC:3 = =
+ * 4 RR:5 RL:4 = =
+ * 5 RR:5 = =
+ * 6 RC:6 =
+ * 7 RLC/FRC RLC/FRC
+ */
+static struct dw_hdmi_channel_conf default_hdmi_channel_config[7] = {
+ { 0x03, 0x00 }, /* FL,FR */
+ { 0x0b, 0x02 }, /* FL,FR,FC */
+ { 0x33, 0x08 }, /* FL,FR,RL,RR */
+ { 0x37, 0x09 }, /* FL,FR,LFE,RL,RR */
+ { 0x3f, 0x0b }, /* FL,FR,LFE,FC,RL,RR */
+ { 0x7f, 0x0f }, /* FL,FR,LFE,FC,RL,RR,RC */
+ { 0xff, 0x13 }, /* FL,FR,LFE,FC,RL,RR,[FR]RC,[FR]LC */
+};
+
+struct snd_dw_hdmi {
+ struct snd_card *card;
+ struct snd_pcm *pcm;
+ spinlock_t lock;
+ struct dw_hdmi_audio_data data;
+ struct snd_pcm_substream *substream;
+ void (*reformat)(struct snd_dw_hdmi *, size_t, size_t);
+ void *buf_src;
+ void *buf_dst;
+ dma_addr_t buf_addr;
+ unsigned buf_offset;
+ unsigned buf_period;
+ unsigned buf_size;
+ unsigned channels;
+ u8 revision;
+ u8 iec_offset;
+ u8 cs[192][8];
+};
+
+static void dw_hdmi_writel(u32 val, void __iomem *ptr)
+{
+ writeb_relaxed(val, ptr);
+ writeb_relaxed(val >> 8, ptr + 1);
+ writeb_relaxed(val >> 16, ptr + 2);
+ writeb_relaxed(val >> 24, ptr + 3);
+}
+
+/*
+ * Convert to hardware format: The userspace buffer contains IEC958 samples,
+ * with the PCUV bits in bits 31..28 and audio samples in bits 27..4. We
+ * need these to be in bits 27..24, with the IEC B bit in bit 28, and audio
+ * samples in 23..0.
+ *
+ * Default preamble in bits 3..0: 8 = block start, 4 = even 2 = odd
+ *
+ * Ideally, we could do with having the data properly formatted in userspace.
+ */
+static void dw_hdmi_reformat_iec958(struct snd_dw_hdmi *dw,
+ size_t offset, size_t bytes)
+{
+ u32 *src = dw->buf_src + offset;
+ u32 *dst = dw->buf_dst + offset;
+ u32 *end = dw->buf_src + offset + bytes;
+
+ do {
+ u32 b, sample = *src++;
+
+ b = (sample & 8) << (28 - 3);
+
+ sample >>= 4;
+
+ *dst++ = sample | b;
+ } while (src < end);
+}
+
+static u32 parity(u32 sample)
+{
+ sample ^= sample >> 16;
+ sample ^= sample >> 8;
+ sample ^= sample >> 4;
+ sample ^= sample >> 2;
+ sample ^= sample >> 1;
+ return (sample & 1) << 27;
+}
+
+static void dw_hdmi_reformat_s24(struct snd_dw_hdmi *dw,
+ size_t offset, size_t bytes)
+{
+ u32 *src = dw->buf_src + offset;
+ u32 *dst = dw->buf_dst + offset;
+ u32 *end = dw->buf_src + offset + bytes;
+
+ do {
+ unsigned i;
+ u8 *cs;
+
+ cs = dw->cs[dw->iec_offset++];
+ if (dw->iec_offset >= 192)
+ dw->iec_offset = 0;
+
+ i = dw->channels;
+ do {
+ u32 sample = *src++;
+
+ sample &= ~0xff000000;
+ sample |= *cs++ << 24;
+ sample |= parity(sample & ~0xf8000000);
+
+ *dst++ = sample;
+ } while (--i);
+ } while (src < end);
+}
+
+static void dw_hdmi_create_cs(struct snd_dw_hdmi *dw,
+ struct snd_pcm_runtime *runtime)
+{
+ u8 cs[4];
+ unsigned ch, i, j;
+
+ snd_pcm_create_iec958_consumer(runtime, cs, sizeof(cs));
+
+ memset(dw->cs, 0, sizeof(dw->cs));
+
+ for (ch = 0; ch < 8; ch++) {
+ cs[2] &= ~IEC958_AES2_CON_CHANNEL;
+ cs[2] |= (ch + 1) << 4;
+
+ for (i = 0; i < ARRAY_SIZE(cs); i++) {
+ unsigned c = cs[i];
+
+ for (j = 0; j < 8; j++, c >>= 1)
+ dw->cs[i * 8 + j][ch] = (c & 1) << 2;
+ }
+ }
+ dw->cs[0][0] |= BIT(4);
+}
+
+static void dw_hdmi_start_dma(struct snd_dw_hdmi *dw)
+{
+ void __iomem *base = dw->data.base;
+ unsigned offset = dw->buf_offset;
+ unsigned period = dw->buf_period;
+ u32 start, stop;
+
+ dw->reformat(dw, offset, period);
+
+ /* Clear all irqs before enabling irqs and starting DMA */
+ writeb_relaxed(HDMI_IH_AHBDMAAUD_STAT0_ALL,
+ base + HDMI_IH_AHBDMAAUD_STAT0);
+
+ start = dw->buf_addr + offset;
+ stop = start + period - 1;
+
+ /* Setup the hardware start/stop addresses */
+ dw_hdmi_writel(start, base + HDMI_AHB_DMA_STRADDR0);
+ dw_hdmi_writel(stop, base + HDMI_AHB_DMA_STPADDR0);
+
+ writeb_relaxed((u8)~HDMI_AHB_DMA_MASK_DONE, base + HDMI_AHB_DMA_MASK);
+ writeb(HDMI_AHB_DMA_START_START, base + HDMI_AHB_DMA_START);
+
+ offset += period;
+ if (offset >= dw->buf_size)
+ offset = 0;
+ dw->buf_offset = offset;
+}
+
+static void dw_hdmi_stop_dma(struct snd_dw_hdmi *dw)
+{
+ /* Disable interrupts before disabling DMA */
+ writeb_relaxed(~0, dw->data.base + HDMI_AHB_DMA_MASK);
+ writeb_relaxed(HDMI_AHB_DMA_STOP_STOP, dw->data.base + HDMI_AHB_DMA_STOP);
+}
+
+static irqreturn_t snd_dw_hdmi_irq(int irq, void *data)
+{
+ struct snd_dw_hdmi *dw = data;
+ struct snd_pcm_substream *substream;
+ unsigned stat;
+
+ stat = readb_relaxed(dw->data.base + HDMI_IH_AHBDMAAUD_STAT0);
+ if (!stat)
+ return IRQ_NONE;
+
+ writeb_relaxed(stat, dw->data.base + HDMI_IH_AHBDMAAUD_STAT0);
+
+ substream = dw->substream;
+ if (stat & HDMI_IH_AHBDMAAUD_STAT0_DONE && substream) {
+ snd_pcm_period_elapsed(substream);
+
+ spin_lock(&dw->lock);
+ if (dw->substream)
+ dw_hdmi_start_dma(dw);
+ spin_unlock(&dw->lock);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static struct snd_pcm_hardware dw_hdmi_hw = {
+ .info = SNDRV_PCM_INFO_INTERLEAVED |
+ SNDRV_PCM_INFO_BLOCK_TRANSFER |
+ SNDRV_PCM_INFO_MMAP |
+ SNDRV_PCM_INFO_MMAP_VALID,
+ .formats = SNDRV_PCM_FMTBIT_IEC958_SUBFRAME_LE |
+ SNDRV_PCM_FMTBIT_S24_LE,
+ .rates = SNDRV_PCM_RATE_32000 |
+ SNDRV_PCM_RATE_44100 |
+ SNDRV_PCM_RATE_48000 |
+ SNDRV_PCM_RATE_88200 |
+ SNDRV_PCM_RATE_96000 |
+ SNDRV_PCM_RATE_176400 |
+ SNDRV_PCM_RATE_192000,
+ .channels_min = 2,
+ .channels_max = 8,
+ .buffer_bytes_max = 1024 * 1024,
+ .period_bytes_min = 256,
+ .period_bytes_max = 8192, /* ERR004323: must limit to 8k */
+ .periods_min = 2,
+ .periods_max = 16,
+ .fifo_size = 0,
+};
+
+static int dw_hdmi_open(struct snd_pcm_substream *substream)
+{
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct snd_dw_hdmi *dw = substream->private_data;
+ void __iomem *base = dw->data.base;
+ int ret;
+
+ runtime->hw = dw_hdmi_hw;
+
+ ret = snd_pcm_hw_constraint_eld(runtime, dw->data.eld);
+ if (ret < 0)
+ return ret;
+
+ ret = snd_pcm_limit_hw_rates(runtime);
+ if (ret < 0)
+ return ret;
+
+ ret = snd_pcm_hw_constraint_integer(runtime,
+ SNDRV_PCM_HW_PARAM_PERIODS);
+ if (ret < 0)
+ return ret;
+
+ /* Limit the buffer size to the size of the preallocated buffer */
+ ret = snd_pcm_hw_constraint_minmax(runtime,
+ SNDRV_PCM_HW_PARAM_BUFFER_SIZE,
+ 0, substream->dma_buffer.bytes);
+ if (ret < 0)
+ return ret;
+
+ /* Clear FIFO */
+ writeb_relaxed(HDMI_AHB_DMA_CONF0_SW_FIFO_RST,
+ base + HDMI_AHB_DMA_CONF0);
+
+ /* Configure interrupt polarities */
+ writeb_relaxed(~0, base + HDMI_AHB_DMA_POL);
+ writeb_relaxed(~0, base + HDMI_AHB_DMA_BUFFPOL);
+
+ /* Keep interrupts masked, and clear any pending */
+ writeb_relaxed(~0, base + HDMI_AHB_DMA_MASK);
+ writeb_relaxed(~0, base + HDMI_IH_AHBDMAAUD_STAT0);
+
+ ret = request_irq(dw->data.irq, snd_dw_hdmi_irq, IRQF_SHARED,
+ "dw-hdmi-audio", dw);
+ if (ret)
+ return ret;
+
+ /* Un-mute done interrupt */
+ writeb_relaxed(HDMI_IH_MUTE_AHBDMAAUD_STAT0_ALL &
+ ~HDMI_IH_MUTE_AHBDMAAUD_STAT0_DONE,
+ base + HDMI_IH_MUTE_AHBDMAAUD_STAT0);
+
+ return 0;
+}
+
+static int dw_hdmi_close(struct snd_pcm_substream *substream)
+{
+ struct snd_dw_hdmi *dw = substream->private_data;
+
+ /* Mute all interrupts */
+ writeb_relaxed(HDMI_IH_MUTE_AHBDMAAUD_STAT0_ALL,
+ dw->data.base + HDMI_IH_MUTE_AHBDMAAUD_STAT0);
+
+ free_irq(dw->data.irq, dw);
+
+ return 0;
+}
+
+static int dw_hdmi_hw_free(struct snd_pcm_substream *substream)
+{
+ return snd_pcm_lib_free_vmalloc_buffer(substream);
+}
+
+static int dw_hdmi_hw_params(struct snd_pcm_substream *substream,
+ struct snd_pcm_hw_params *params)
+{
+ /* Allocate the PCM runtime buffer, which is exposed to userspace. */
+ return snd_pcm_lib_alloc_vmalloc_buffer(substream,
+ params_buffer_bytes(params));
+}
+
+static int dw_hdmi_prepare(struct snd_pcm_substream *substream)
+{
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct snd_dw_hdmi *dw = substream->private_data;
+ u8 threshold, conf0, conf1, layout, ca;
+
+ /* Setup as per 3.0.5 FSL 4.1.0 BSP */
+ switch (dw->revision) {
+ case 0x0a:
+ conf0 = HDMI_AHB_DMA_CONF0_BURST_MODE |
+ HDMI_AHB_DMA_CONF0_INCR4;
+ if (runtime->channels == 2)
+ threshold = 126;
+ else
+ threshold = 124;
+ break;
+ case 0x1a:
+ conf0 = HDMI_AHB_DMA_CONF0_BURST_MODE |
+ HDMI_AHB_DMA_CONF0_INCR8;
+ threshold = 128;
+ break;
+ default:
+ /* NOTREACHED */
+ return -EINVAL;
+ }
+
+ dw_hdmi_set_sample_rate(dw->data.hdmi, runtime->rate);
+
+ /* Minimum number of bytes in the fifo. */
+ runtime->hw.fifo_size = threshold * 32;
+
+ conf0 |= HDMI_AHB_DMA_CONF0_EN_HLOCK;
+ conf1 = default_hdmi_channel_config[runtime->channels - 2].conf1;
+ ca = default_hdmi_channel_config[runtime->channels - 2].ca;
+
+ /*
+ * For >2 channel PCM audio, we need to select layout 1
+ * and set an appropriate channel map.
+ */
+ if (runtime->channels > 2)
+ layout = HDMI_FC_AUDSCONF_LAYOUT1;
+ else
+ layout = HDMI_FC_AUDSCONF_LAYOUT0;
+
+ writeb_relaxed(threshold, dw->data.base + HDMI_AHB_DMA_THRSLD);
+ writeb_relaxed(conf0, dw->data.base + HDMI_AHB_DMA_CONF0);
+ writeb_relaxed(conf1, dw->data.base + HDMI_AHB_DMA_CONF1);
+ writeb_relaxed(layout, dw->data.base + HDMI_FC_AUDSCONF);
+ writeb_relaxed(ca, dw->data.base + HDMI_FC_AUDICONF2);
+
+ switch (runtime->format) {
+ case SNDRV_PCM_FORMAT_IEC958_SUBFRAME_LE:
+ dw->reformat = dw_hdmi_reformat_iec958;
+ break;
+ case SNDRV_PCM_FORMAT_S24_LE:
+ dw_hdmi_create_cs(dw, runtime);
+ dw->reformat = dw_hdmi_reformat_s24;
+ break;
+ }
+ dw->iec_offset = 0;
+ dw->channels = runtime->channels;
+ dw->buf_src = runtime->dma_area;
+ dw->buf_dst = substream->dma_buffer.area;
+ dw->buf_addr = substream->dma_buffer.addr;
+ dw->buf_period = snd_pcm_lib_period_bytes(substream);
+ dw->buf_size = snd_pcm_lib_buffer_bytes(substream);
+
+ return 0;
+}
+
+static int dw_hdmi_trigger(struct snd_pcm_substream *substream, int cmd)
+{
+ struct snd_dw_hdmi *dw = substream->private_data;
+ unsigned long flags;
+ int ret = 0;
+
+ switch (cmd) {
+ case SNDRV_PCM_TRIGGER_START:
+ spin_lock_irqsave(&dw->lock, flags);
+ dw->buf_offset = 0;
+ dw->substream = substream;
+ dw_hdmi_start_dma(dw);
+ dw_hdmi_audio_enable(dw->data.hdmi);
+ spin_unlock_irqrestore(&dw->lock, flags);
+ substream->runtime->delay = substream->runtime->period_size;
+ break;
+
+ case SNDRV_PCM_TRIGGER_STOP:
+ spin_lock_irqsave(&dw->lock, flags);
+ dw->substream = NULL;
+ dw_hdmi_stop_dma(dw);
+ dw_hdmi_audio_disable(dw->data.hdmi);
+ spin_unlock_irqrestore(&dw->lock, flags);
+ break;
+
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ return ret;
+}
+
+static snd_pcm_uframes_t dw_hdmi_pointer(struct snd_pcm_substream *substream)
+{
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct snd_dw_hdmi *dw = substream->private_data;
+
+ /*
+ * We are unable to report the exact hardware position as
+ * reading the 32-bit DMA position using 8-bit reads is racy.
+ */
+ return bytes_to_frames(runtime, dw->buf_offset);
+}
+
+static struct snd_pcm_ops snd_dw_hdmi_ops = {
+ .open = dw_hdmi_open,
+ .close = dw_hdmi_close,
+ .ioctl = snd_pcm_lib_ioctl,
+ .hw_params = dw_hdmi_hw_params,
+ .hw_free = dw_hdmi_hw_free,
+ .prepare = dw_hdmi_prepare,
+ .trigger = dw_hdmi_trigger,
+ .pointer = dw_hdmi_pointer,
+ .page = snd_pcm_lib_get_vmalloc_page,
+};
+
+static int snd_dw_hdmi_probe(struct platform_device *pdev)
+{
+ const struct dw_hdmi_audio_data *data = pdev->dev.platform_data;
+ struct device *dev = pdev->dev.parent;
+ struct snd_dw_hdmi *dw;
+ struct snd_card *card;
+ struct snd_pcm *pcm;
+ unsigned revision;
+ int ret;
+
+ writeb_relaxed(HDMI_IH_MUTE_AHBDMAAUD_STAT0_ALL,
+ data->base + HDMI_IH_MUTE_AHBDMAAUD_STAT0);
+ revision = readb_relaxed(data->base + HDMI_REVISION_ID);
+ if (revision != 0x0a && revision != 0x1a) {
+ dev_err(dev, "dw-hdmi-audio: unknown revision 0x%02x\n",
+ revision);
+ return -ENXIO;
+ }
+
+ ret = snd_card_new(dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1,
+ THIS_MODULE, sizeof(struct snd_dw_hdmi), &card);
+ if (ret < 0)
+ return ret;
+
+ strlcpy(card->driver, DRIVER_NAME, sizeof(card->driver));
+ strlcpy(card->shortname, "DW-HDMI", sizeof(card->shortname));
+ snprintf(card->longname, sizeof(card->longname),
+ "%s rev 0x%02x, irq %d", card->shortname, revision,
+ data->irq);
+
+ dw = card->private_data;
+ dw->card = card;
+ dw->data = *data;
+ dw->revision = revision;
+
+ spin_lock_init(&dw->lock);
+
+ ret = snd_pcm_new(card, "DW HDMI", 0, 1, 0, &pcm);
+ if (ret < 0)
+ goto err;
+
+ dw->pcm = pcm;
+ pcm->private_data = dw;
+ strlcpy(pcm->name, DRIVER_NAME, sizeof(pcm->name));
+ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &snd_dw_hdmi_ops);
+
+ /*
+ * To support 8-channel 96kHz audio reliably, we need 512k
+ * to satisfy alsa with our restricted period (ERR004323).
+ */
+ snd_pcm_lib_preallocate_pages_for_all(pcm, SNDRV_DMA_TYPE_DEV,
+ dev, 128 * 1024, 1024 * 1024);
+
+ ret = snd_card_register(card);
+ if (ret < 0)
+ goto err;
+
+ platform_set_drvdata(pdev, dw);
+
+ return 0;
+
+err:
+ snd_card_free(card);
+ return ret;
+}
+
+static int snd_dw_hdmi_remove(struct platform_device *pdev)
+{
+ struct snd_dw_hdmi *dw = platform_get_drvdata(pdev);
+
+ snd_card_free(dw->card);
+
+ return 0;
+}
+
+#if defined(CONFIG_PM_SLEEP) && defined(IS_NOT_BROKEN)
+/*
+ * This code is fine, but requires implementation in the dw_hdmi_trigger()
+ * method which is currently missing as I have no way to test this.
+ */
+static int snd_dw_hdmi_suspend(struct device *dev)
+{
+ struct snd_dw_hdmi *dw = dev_get_drvdata(dev);
+
+ snd_power_change_state(dw->card, SNDRV_CTL_POWER_D3cold);
+ snd_pcm_suspend_all(dw->pcm);
+
+ return 0;
+}
+
+static int snd_dw_hdmi_resume(struct device *dev)
+{
+ struct snd_dw_hdmi *dw = dev_get_drvdata(dev);
+
+ snd_power_change_state(dw->card, SNDRV_CTL_POWER_D0);
+
+ return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(snd_dw_hdmi_pm, snd_dw_hdmi_suspend,
+ snd_dw_hdmi_resume);
+#define PM_OPS &snd_dw_hdmi_pm
+#else
+#define PM_OPS NULL
+#endif
+
+static struct platform_driver snd_dw_hdmi_driver = {
+ .probe = snd_dw_hdmi_probe,
+ .remove = snd_dw_hdmi_remove,
+ .driver = {
+ .name = DRIVER_NAME,
+ .owner = THIS_MODULE,
+ .pm = PM_OPS,
+ },
+};
+
+module_platform_driver(snd_dw_hdmi_driver);
+
+MODULE_AUTHOR("Russell King <rmk+kernel@arm.linux.org.uk>");
+MODULE_DESCRIPTION("Synopsis Designware HDMI AHB ALSA interface");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:" DRIVER_NAME);
--- /dev/null
+#ifndef DW_HDMI_AUDIO_H
+#define DW_HDMI_AUDIO_H
+
+struct dw_hdmi;
+
+struct dw_hdmi_audio_data {
+ phys_addr_t phys;
+ void __iomem *base;
+ int irq;
+ struct dw_hdmi *hdmi;
+ u8 *eld;
+};
+
+#endif
#include <drm/bridge/dw_hdmi.h>
#include "dw_hdmi.h"
+#include "dw_hdmi-audio.h"
#define HDMI_EDID_LEN 512
struct drm_encoder *encoder;
struct drm_bridge *bridge;
+ struct platform_device *audio;
enum dw_hdmi_devtype dev_type;
struct device *dev;
struct clk *isfr_clk;
bool sink_has_audio;
struct mutex mutex; /* for state below and previous_mode */
+ enum drm_connector_force force; /* mutex-protected force state */
bool disabled; /* DRM has disabled our bridge */
+ bool bridge_is_on; /* indicates the bridge is on */
+ bool rxsense; /* rxsense state */
+ u8 phy_mask; /* desired phy int mask settings */
spinlock_t audio_lock;
struct mutex audio_mutex;
unsigned int audio_cts;
unsigned int audio_n;
bool audio_enable;
- int ratio;
void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
u8 (*read)(struct dw_hdmi *hdmi, int offset);
};
+#define HDMI_IH_PHY_STAT0_RX_SENSE \
+ (HDMI_IH_PHY_STAT0_RX_SENSE0 | HDMI_IH_PHY_STAT0_RX_SENSE1 | \
+ HDMI_IH_PHY_STAT0_RX_SENSE2 | HDMI_IH_PHY_STAT0_RX_SENSE3)
+
+#define HDMI_PHY_RX_SENSE \
+ (HDMI_PHY_RX_SENSE0 | HDMI_PHY_RX_SENSE1 | \
+ HDMI_PHY_RX_SENSE2 | HDMI_PHY_RX_SENSE3)
+
static void dw_hdmi_writel(struct dw_hdmi *hdmi, u8 val, int offset)
{
writel(val, hdmi->regs + (offset << 2));
hdmi_writeb(hdmi, n & 0xff, HDMI_AUD_N1);
}
-static unsigned int hdmi_compute_n(unsigned int freq, unsigned long pixel_clk,
- unsigned int ratio)
+static unsigned int hdmi_compute_n(unsigned int freq, unsigned long pixel_clk)
{
unsigned int n = (128 * freq) / 1000;
+ unsigned int mult = 1;
+
+ while (freq > 48000) {
+ mult *= 2;
+ freq /= 2;
+ }
switch (freq) {
case 32000:
- if (pixel_clk == 25170000)
- n = (ratio == 150) ? 9152 : 4576;
- else if (pixel_clk == 27020000)
- n = (ratio == 150) ? 8192 : 4096;
- else if (pixel_clk == 74170000 || pixel_clk == 148350000)
+ if (pixel_clk == 25175000)
+ n = 4576;
+ else if (pixel_clk == 27027000)
+ n = 4096;
+ else if (pixel_clk == 74176000 || pixel_clk == 148352000)
n = 11648;
else
n = 4096;
+ n *= mult;
break;
case 44100:
- if (pixel_clk == 25170000)
+ if (pixel_clk == 25175000)
n = 7007;
- else if (pixel_clk == 74170000)
+ else if (pixel_clk == 74176000)
n = 17836;
- else if (pixel_clk == 148350000)
- n = (ratio == 150) ? 17836 : 8918;
+ else if (pixel_clk == 148352000)
+ n = 8918;
else
n = 6272;
+ n *= mult;
break;
case 48000:
- if (pixel_clk == 25170000)
- n = (ratio == 150) ? 9152 : 6864;
- else if (pixel_clk == 27020000)
- n = (ratio == 150) ? 8192 : 6144;
- else if (pixel_clk == 74170000)
+ if (pixel_clk == 25175000)
+ n = 6864;
+ else if (pixel_clk == 27027000)
+ n = 6144;
+ else if (pixel_clk == 74176000)
n = 11648;
- else if (pixel_clk == 148350000)
- n = (ratio == 150) ? 11648 : 5824;
+ else if (pixel_clk == 148352000)
+ n = 5824;
else
n = 6144;
- break;
-
- case 88200:
- n = hdmi_compute_n(44100, pixel_clk, ratio) * 2;
- break;
-
- case 96000:
- n = hdmi_compute_n(48000, pixel_clk, ratio) * 2;
- break;
-
- case 176400:
- n = hdmi_compute_n(44100, pixel_clk, ratio) * 4;
- break;
-
- case 192000:
- n = hdmi_compute_n(48000, pixel_clk, ratio) * 4;
+ n *= mult;
break;
default:
return n;
}
-static unsigned int hdmi_compute_cts(unsigned int freq, unsigned long pixel_clk,
- unsigned int ratio)
-{
- unsigned int cts = 0;
-
- pr_debug("%s: freq: %d pixel_clk: %ld ratio: %d\n", __func__, freq,
- pixel_clk, ratio);
-
- switch (freq) {
- case 32000:
- if (pixel_clk == 297000000) {
- cts = 222750;
- break;
- }
- case 48000:
- case 96000:
- case 192000:
- switch (pixel_clk) {
- case 25200000:
- case 27000000:
- case 54000000:
- case 74250000:
- case 148500000:
- cts = pixel_clk / 1000;
- break;
- case 297000000:
- cts = 247500;
- break;
- /*
- * All other TMDS clocks are not supported by
- * DWC_hdmi_tx. The TMDS clocks divided or
- * multiplied by 1,001 coefficients are not
- * supported.
- */
- default:
- break;
- }
- break;
- case 44100:
- case 88200:
- case 176400:
- switch (pixel_clk) {
- case 25200000:
- cts = 28000;
- break;
- case 27000000:
- cts = 30000;
- break;
- case 54000000:
- cts = 60000;
- break;
- case 74250000:
- cts = 82500;
- break;
- case 148500000:
- cts = 165000;
- break;
- case 297000000:
- cts = 247500;
- break;
- default:
- break;
- }
- break;
- default:
- break;
- }
- if (ratio == 100)
- return cts;
- return (cts * ratio) / 100;
-}
-
static void hdmi_set_clk_regenerator(struct dw_hdmi *hdmi,
- unsigned long pixel_clk, unsigned int sample_rate, unsigned int ratio)
+ unsigned long pixel_clk, unsigned int sample_rate)
{
+ unsigned long ftdms = pixel_clk;
unsigned int n, cts;
+ u64 tmp;
- n = hdmi_compute_n(sample_rate, pixel_clk, ratio);
- cts = hdmi_compute_cts(sample_rate, pixel_clk, ratio);
- if (!cts) {
- dev_err(hdmi->dev,
- "%s: pixel clock/sample rate not supported: %luMHz / %ukHz\n",
- __func__, pixel_clk, sample_rate);
- }
+ n = hdmi_compute_n(sample_rate, pixel_clk);
+
+ /*
+ * Compute the CTS value from the N value. Note that CTS and N
+ * can be up to 20 bits in total, so we need 64-bit math. Also
+ * note that our TDMS clock is not fully accurate; it is accurate
+ * to kHz. This can introduce an unnecessary remainder in the
+ * calculation below, so we don't try to warn about that.
+ */
+ tmp = (u64)ftdms * n;
+ do_div(tmp, 128 * sample_rate);
+ cts = tmp;
- dev_dbg(hdmi->dev, "%s: samplerate=%ukHz ratio=%d pixelclk=%luMHz N=%d cts=%d\n",
- __func__, sample_rate, ratio, pixel_clk, n, cts);
+ dev_dbg(hdmi->dev, "%s: fs=%uHz ftdms=%lu.%03luMHz N=%d cts=%d\n",
+ __func__, sample_rate, ftdms / 1000000, (ftdms / 1000) % 1000,
+ n, cts);
spin_lock_irq(&hdmi->audio_lock);
hdmi->audio_n = n;
static void hdmi_init_clk_regenerator(struct dw_hdmi *hdmi)
{
mutex_lock(&hdmi->audio_mutex);
- hdmi_set_clk_regenerator(hdmi, 74250000, hdmi->sample_rate,
- hdmi->ratio);
+ hdmi_set_clk_regenerator(hdmi, 74250000, hdmi->sample_rate);
mutex_unlock(&hdmi->audio_mutex);
}
{
mutex_lock(&hdmi->audio_mutex);
hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mpixelclock,
- hdmi->sample_rate, hdmi->ratio);
+ hdmi->sample_rate);
mutex_unlock(&hdmi->audio_mutex);
}
mutex_lock(&hdmi->audio_mutex);
hdmi->sample_rate = rate;
hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mpixelclock,
- hdmi->sample_rate, hdmi->ratio);
+ hdmi->sample_rate);
mutex_unlock(&hdmi->audio_mutex);
}
EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_rate);
u8 inv_val;
struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode;
int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len;
+ unsigned int vdisplay;
vmode->mpixelclock = mode->clock * 1000;
hdmi_writeb(hdmi, inv_val, HDMI_FC_INVIDCONF);
+ vdisplay = mode->vdisplay;
+ vblank = mode->vtotal - mode->vdisplay;
+ v_de_vs = mode->vsync_start - mode->vdisplay;
+ vsync_len = mode->vsync_end - mode->vsync_start;
+
+ /*
+ * When we're setting an interlaced mode, we need
+ * to adjust the vertical timing to suit.
+ */
+ if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+ vdisplay /= 2;
+ vblank /= 2;
+ v_de_vs /= 2;
+ vsync_len /= 2;
+ }
+
/* Set up horizontal active pixel width */
hdmi_writeb(hdmi, mode->hdisplay >> 8, HDMI_FC_INHACTV1);
hdmi_writeb(hdmi, mode->hdisplay, HDMI_FC_INHACTV0);
/* Set up vertical active lines */
- hdmi_writeb(hdmi, mode->vdisplay >> 8, HDMI_FC_INVACTV1);
- hdmi_writeb(hdmi, mode->vdisplay, HDMI_FC_INVACTV0);
+ hdmi_writeb(hdmi, vdisplay >> 8, HDMI_FC_INVACTV1);
+ hdmi_writeb(hdmi, vdisplay, HDMI_FC_INVACTV0);
/* Set up horizontal blanking pixel region width */
hblank = mode->htotal - mode->hdisplay;
hdmi_writeb(hdmi, hblank, HDMI_FC_INHBLANK0);
/* Set up vertical blanking pixel region width */
- vblank = mode->vtotal - mode->vdisplay;
hdmi_writeb(hdmi, vblank, HDMI_FC_INVBLANK);
/* Set up HSYNC active edge delay width (in pixel clks) */
hdmi_writeb(hdmi, h_de_hs, HDMI_FC_HSYNCINDELAY0);
/* Set up VSYNC active edge delay (in lines) */
- v_de_vs = mode->vsync_start - mode->vdisplay;
hdmi_writeb(hdmi, v_de_vs, HDMI_FC_VSYNCINDELAY);
/* Set up HSYNC active pulse width (in pixel clks) */
hdmi_writeb(hdmi, hsync_len, HDMI_FC_HSYNCINWIDTH0);
/* Set up VSYNC active edge delay (in lines) */
- vsync_len = mode->vsync_end - mode->vsync_start;
hdmi_writeb(hdmi, vsync_len, HDMI_FC_VSYNCINWIDTH);
}
HDMI_PHY_I2CM_CTLINT_ADDR);
/* enable cable hot plug irq */
- hdmi_writeb(hdmi, (u8)~HDMI_PHY_HPD, HDMI_PHY_MASK0);
+ hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0);
/* Clear Hotplug interrupts */
- hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD, HDMI_IH_PHY_STAT0);
+ hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE,
+ HDMI_IH_PHY_STAT0);
return 0;
}
static void dw_hdmi_poweron(struct dw_hdmi *hdmi)
{
+ hdmi->bridge_is_on = true;
dw_hdmi_setup(hdmi, &hdmi->previous_mode);
}
static void dw_hdmi_poweroff(struct dw_hdmi *hdmi)
{
dw_hdmi_phy_disable(hdmi);
+ hdmi->bridge_is_on = false;
+}
+
+static void dw_hdmi_update_power(struct dw_hdmi *hdmi)
+{
+ int force = hdmi->force;
+
+ if (hdmi->disabled) {
+ force = DRM_FORCE_OFF;
+ } else if (force == DRM_FORCE_UNSPECIFIED) {
+ if (hdmi->rxsense)
+ force = DRM_FORCE_ON;
+ else
+ force = DRM_FORCE_OFF;
+ }
+
+ if (force == DRM_FORCE_OFF) {
+ if (hdmi->bridge_is_on)
+ dw_hdmi_poweroff(hdmi);
+ } else {
+ if (!hdmi->bridge_is_on)
+ dw_hdmi_poweron(hdmi);
+ }
+}
+
+/*
+ * Adjust the detection of RXSENSE according to whether we have a forced
+ * connection mode enabled, or whether we have been disabled. There is
+ * no point processing RXSENSE interrupts if we have a forced connection
+ * state, or DRM has us disabled.
+ *
+ * We also disable rxsense interrupts when we think we're disconnected
+ * to avoid floating TDMS signals giving false rxsense interrupts.
+ *
+ * Note: we still need to listen for HPD interrupts even when DRM has us
+ * disabled so that we can detect a connect event.
+ */
+static void dw_hdmi_update_phy_mask(struct dw_hdmi *hdmi)
+{
+ u8 old_mask = hdmi->phy_mask;
+
+ if (hdmi->force || hdmi->disabled || !hdmi->rxsense)
+ hdmi->phy_mask |= HDMI_PHY_RX_SENSE;
+ else
+ hdmi->phy_mask &= ~HDMI_PHY_RX_SENSE;
+
+ if (old_mask != hdmi->phy_mask)
+ hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0);
}
static void dw_hdmi_bridge_mode_set(struct drm_bridge *bridge,
mutex_lock(&hdmi->mutex);
hdmi->disabled = true;
- dw_hdmi_poweroff(hdmi);
+ dw_hdmi_update_power(hdmi);
+ dw_hdmi_update_phy_mask(hdmi);
mutex_unlock(&hdmi->mutex);
}
struct dw_hdmi *hdmi = bridge->driver_private;
mutex_lock(&hdmi->mutex);
- dw_hdmi_poweron(hdmi);
hdmi->disabled = false;
+ dw_hdmi_update_power(hdmi);
+ dw_hdmi_update_phy_mask(hdmi);
mutex_unlock(&hdmi->mutex);
}
struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
connector);
+ mutex_lock(&hdmi->mutex);
+ hdmi->force = DRM_FORCE_UNSPECIFIED;
+ dw_hdmi_update_power(hdmi);
+ dw_hdmi_update_phy_mask(hdmi);
+ mutex_unlock(&hdmi->mutex);
+
return hdmi_readb(hdmi, HDMI_PHY_STAT0) & HDMI_PHY_HPD ?
connector_status_connected : connector_status_disconnected;
}
hdmi->sink_has_audio = drm_detect_monitor_audio(edid);
drm_mode_connector_update_edid_property(connector, edid);
ret = drm_add_edid_modes(connector, edid);
+ /* Store the ELD */
+ drm_edid_to_eld(connector, edid);
kfree(edid);
} else {
dev_dbg(hdmi->dev, "failed to get edid\n");
drm_connector_cleanup(connector);
}
+static void dw_hdmi_connector_force(struct drm_connector *connector)
+{
+ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
+ connector);
+
+ mutex_lock(&hdmi->mutex);
+ hdmi->force = connector->force;
+ dw_hdmi_update_power(hdmi);
+ dw_hdmi_update_phy_mask(hdmi);
+ mutex_unlock(&hdmi->mutex);
+}
+
static struct drm_connector_funcs dw_hdmi_connector_funcs = {
.dpms = drm_helper_connector_dpms,
.fill_modes = drm_helper_probe_single_connector_modes,
.detect = dw_hdmi_connector_detect,
.destroy = dw_hdmi_connector_destroy,
+ .force = dw_hdmi_connector_force,
};
static struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = {
static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
{
struct dw_hdmi *hdmi = dev_id;
- u8 intr_stat;
- u8 phy_int_pol;
+ u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat;
intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0);
-
phy_int_pol = hdmi_readb(hdmi, HDMI_PHY_POL0);
+ phy_stat = hdmi_readb(hdmi, HDMI_PHY_STAT0);
+
+ phy_pol_mask = 0;
+ if (intr_stat & HDMI_IH_PHY_STAT0_HPD)
+ phy_pol_mask |= HDMI_PHY_HPD;
+ if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE0)
+ phy_pol_mask |= HDMI_PHY_RX_SENSE0;
+ if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE1)
+ phy_pol_mask |= HDMI_PHY_RX_SENSE1;
+ if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE2)
+ phy_pol_mask |= HDMI_PHY_RX_SENSE2;
+ if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE3)
+ phy_pol_mask |= HDMI_PHY_RX_SENSE3;
+
+ if (phy_pol_mask)
+ hdmi_modb(hdmi, ~phy_int_pol, phy_pol_mask, HDMI_PHY_POL0);
- if (intr_stat & HDMI_IH_PHY_STAT0_HPD) {
- hdmi_modb(hdmi, ~phy_int_pol, HDMI_PHY_HPD, HDMI_PHY_POL0);
+ /*
+ * RX sense tells us whether the TDMS transmitters are detecting
+ * load - in other words, there's something listening on the
+ * other end of the link. Use this to decide whether we should
+ * power on the phy as HPD may be toggled by the sink to merely
+ * ask the source to re-read the EDID.
+ */
+ if (intr_stat &
+ (HDMI_IH_PHY_STAT0_RX_SENSE | HDMI_IH_PHY_STAT0_HPD)) {
mutex_lock(&hdmi->mutex);
- if (phy_int_pol & HDMI_PHY_HPD) {
- dev_dbg(hdmi->dev, "EVENT=plugin\n");
-
- if (!hdmi->disabled)
- dw_hdmi_poweron(hdmi);
- } else {
- dev_dbg(hdmi->dev, "EVENT=plugout\n");
-
- if (!hdmi->disabled)
- dw_hdmi_poweroff(hdmi);
+ if (!hdmi->disabled && !hdmi->force) {
+ /*
+ * If the RX sense status indicates we're disconnected,
+ * clear the software rxsense status.
+ */
+ if (!(phy_stat & HDMI_PHY_RX_SENSE))
+ hdmi->rxsense = false;
+
+ /*
+ * Only set the software rxsense status when both
+ * rxsense and hpd indicates we're connected.
+ * This avoids what seems to be bad behaviour in
+ * at least iMX6S versions of the phy.
+ */
+ if (phy_stat & HDMI_PHY_HPD)
+ hdmi->rxsense = true;
+
+ dw_hdmi_update_power(hdmi);
+ dw_hdmi_update_phy_mask(hdmi);
}
mutex_unlock(&hdmi->mutex);
+ }
+
+ if (intr_stat & HDMI_IH_PHY_STAT0_HPD) {
+ dev_dbg(hdmi->dev, "EVENT=%s\n",
+ phy_int_pol & HDMI_PHY_HPD ? "plugin" : "plugout");
drm_helper_hpd_irq_event(hdmi->bridge->dev);
}
hdmi_writeb(hdmi, intr_stat, HDMI_IH_PHY_STAT0);
- hdmi_writeb(hdmi, ~HDMI_IH_PHY_STAT0_HPD, HDMI_IH_MUTE_PHY_STAT0);
+ hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE),
+ HDMI_IH_MUTE_PHY_STAT0);
return IRQ_HANDLED;
}
{
struct drm_device *drm = data;
struct device_node *np = dev->of_node;
+ struct platform_device_info pdevinfo;
struct device_node *ddc_node;
+ struct dw_hdmi_audio_data audio;
struct dw_hdmi *hdmi;
int ret;
u32 val = 1;
if (!hdmi)
return -ENOMEM;
+ hdmi->connector.interlace_allowed = 1;
+
hdmi->plat_data = plat_data;
hdmi->dev = dev;
hdmi->dev_type = plat_data->dev_type;
hdmi->sample_rate = 48000;
- hdmi->ratio = 100;
hdmi->encoder = encoder;
hdmi->disabled = true;
+ hdmi->rxsense = true;
+ hdmi->phy_mask = (u8)~(HDMI_PHY_HPD | HDMI_PHY_RX_SENSE);
mutex_init(&hdmi->mutex);
mutex_init(&hdmi->audio_mutex);
* Configure registers related to HDMI interrupt
* generation before registering IRQ.
*/
- hdmi_writeb(hdmi, HDMI_PHY_HPD, HDMI_PHY_POL0);
+ hdmi_writeb(hdmi, HDMI_PHY_HPD | HDMI_PHY_RX_SENSE, HDMI_PHY_POL0);
/* Clear Hotplug interrupts */
- hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD, HDMI_IH_PHY_STAT0);
+ hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE,
+ HDMI_IH_PHY_STAT0);
ret = dw_hdmi_fb_registered(hdmi);
if (ret)
goto err_iahb;
/* Unmute interrupts */
- hdmi_writeb(hdmi, ~HDMI_IH_PHY_STAT0_HPD, HDMI_IH_MUTE_PHY_STAT0);
+ hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE),
+ HDMI_IH_MUTE_PHY_STAT0);
+
+ memset(&pdevinfo, 0, sizeof(pdevinfo));
+ pdevinfo.parent = dev;
+ pdevinfo.id = PLATFORM_DEVID_AUTO;
+
+ if (hdmi_readb(hdmi, HDMI_CONFIG1_ID) & HDMI_CONFIG1_AHB) {
+ audio.phys = iores->start;
+ audio.base = hdmi->regs;
+ audio.irq = irq;
+ audio.hdmi = hdmi;
+ audio.eld = hdmi->connector.eld;
+
+ pdevinfo.name = "dw-hdmi-ahb-audio";
+ pdevinfo.data = &audio;
+ pdevinfo.size_data = sizeof(audio);
+ pdevinfo.dma_mask = DMA_BIT_MASK(32);
+ hdmi->audio = platform_device_register_full(&pdevinfo);
+ }
dev_set_drvdata(dev, hdmi);
{
struct dw_hdmi *hdmi = dev_get_drvdata(dev);
+ if (hdmi->audio && !IS_ERR(hdmi->audio))
+ platform_device_unregister(hdmi->audio);
+
/* Disable all interrupts */
hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0);
#define HDMI_I2CM_FS_SCL_LCNT_0_ADDR 0x7E12
enum {
+/* CONFIG1_ID field values */
+ HDMI_CONFIG1_AHB = 0x01,
+
/* IH_FC_INT2 field values */
HDMI_IH_FC_INT2_OVERFLOW_MASK = 0x03,
HDMI_IH_FC_INT2_LOW_PRIORITY_OVERFLOW = 0x02,
#include <linux/slab.h>
#include "drm_legacy.h"
-#if __OS_HAS_AGP
-
#include <asm/agp.h>
/**
return mem;
}
EXPORT_SYMBOL(drm_agp_bind_pages);
-
-#endif /* __OS_HAS_AGP */
* consistent behavior you must call this function rather than the
* driver hook directly.
*/
-int drm_atomic_crtc_get_property(struct drm_crtc *crtc,
+static int
+drm_atomic_crtc_get_property(struct drm_crtc *crtc,
const struct drm_crtc_state *state,
struct drm_property *property, uint64_t *val)
{
return 0;
}
+static bool
+plane_switching_crtc(struct drm_atomic_state *state,
+ struct drm_plane *plane,
+ struct drm_plane_state *plane_state)
+{
+ if (!plane->state->crtc || !plane_state->crtc)
+ return false;
+
+ if (plane->state->crtc == plane_state->crtc)
+ return false;
+
+ /* This could be refined, but currently there's no helper or driver code
+ * to implement direct switching of active planes nor userspace to take
+ * advantage of more direct plane switching without the intermediate
+ * full OFF state.
+ */
+ return true;
+}
+
/**
* drm_atomic_plane_check - check plane state
* @plane: plane to check
return -ENOSPC;
}
+ if (plane_switching_crtc(state->state, plane, state)) {
+ DRM_DEBUG_ATOMIC("[PLANE:%d] switching CRTC directly\n",
+ plane->base.id);
+ return -EINVAL;
+ }
+
return 0;
}
* add their own additional internal state.
*
* This library also provides default implementations for the check callback in
- * drm_atomic_helper_check and for the commit callback with
- * drm_atomic_helper_commit. But the individual stages and callbacks are expose
- * to allow drivers to mix and match and e.g. use the plane helpers only
+ * drm_atomic_helper_check() and for the commit callback with
+ * drm_atomic_helper_commit(). But the individual stages and callbacks are
+ * exposed to allow drivers to mix and match and e.g. use the plane helpers only
* together with a driver private modeset implementation.
*
* This library also provides implementations for all the legacy driver
- * interfaces on top of the atomic interface. See drm_atomic_helper_set_config,
- * drm_atomic_helper_disable_plane, drm_atomic_helper_disable_plane and the
+ * interfaces on top of the atomic interface. See drm_atomic_helper_set_config(),
+ * drm_atomic_helper_disable_plane(), drm_atomic_helper_disable_plane() and the
* various functions to implement set_property callbacks. New drivers must not
* implement these functions themselves but must use the provided helpers.
*/
* object. This can still fail when e.g. the framebuffer reservation fails. For
* now this doesn't implement asynchronous commits.
*
+ * Note that right now this function does not support async commits, and hence
+ * driver writers must implement their own version for now. Also note that the
+ * default ordering of how the various stages are called is to match the legacy
+ * modeset helper library closest. One peculiarity of that is that it doesn't
+ * mesh well with runtime PM at all.
+ *
+ * For drivers supporting runtime PM the recommended sequence is
+ *
+ * drm_atomic_helper_commit_modeset_disables(dev, state);
+ *
+ * drm_atomic_helper_commit_modeset_enables(dev, state);
+ *
+ * drm_atomic_helper_commit_planes(dev, state, true);
+ *
+ * See the kerneldoc entries for these three functions for more details.
+ *
* RETURNS
* Zero for success or -errno.
*/
drm_atomic_helper_commit_modeset_disables(dev, state);
- drm_atomic_helper_commit_planes(dev, state);
+ drm_atomic_helper_commit_planes(dev, state, false);
drm_atomic_helper_commit_modeset_enables(dev, state);
* work item, which allows nice concurrent updates on disjoint sets of crtcs.
*
* 3. The software state is updated synchronously with
- * drm_atomic_helper_swap_state. Doing this under the protection of all modeset
+ * drm_atomic_helper_swap_state(). Doing this under the protection of all modeset
* locks means concurrent callers never see inconsistent state. And doing this
* while it's guaranteed that no relevant async worker runs means that async
* workers do not need grab any locks. Actually they must not grab locks, for
const struct drm_plane_helper_funcs *funcs;
struct drm_plane *plane = state->planes[i];
struct drm_plane_state *plane_state = state->plane_states[i];
- struct drm_framebuffer *fb;
if (!plane)
continue;
funcs = plane->helper_private;
- fb = plane_state->fb;
-
- if (fb && funcs->prepare_fb) {
- ret = funcs->prepare_fb(plane, fb, plane_state);
+ if (funcs->prepare_fb) {
+ ret = funcs->prepare_fb(plane, plane_state);
if (ret)
goto fail;
}
const struct drm_plane_helper_funcs *funcs;
struct drm_plane *plane = state->planes[i];
struct drm_plane_state *plane_state = state->plane_states[i];
- struct drm_framebuffer *fb;
if (!plane)
continue;
funcs = plane->helper_private;
- fb = state->plane_states[i]->fb;
-
- if (fb && funcs->cleanup_fb)
- funcs->cleanup_fb(plane, fb, plane_state);
+ if (funcs->cleanup_fb)
+ funcs->cleanup_fb(plane, plane_state);
}
}
EXPORT_SYMBOL(drm_atomic_helper_prepare_planes);
+bool plane_crtc_active(struct drm_plane_state *state)
+{
+ return state->crtc && state->crtc->state->active;
+}
+
/**
* drm_atomic_helper_commit_planes - commit plane state
* @dev: DRM device
* @old_state: atomic state object with old state structures
+ * @active_only: Only commit on active CRTC if set
*
* This function commits the new plane state using the plane and atomic helper
* functions for planes and crtcs. It assumes that the atomic state has already
* Note that this function does all plane updates across all CRTCs in one step.
* If the hardware can't support this approach look at
* drm_atomic_helper_commit_planes_on_crtc() instead.
+ *
+ * Plane parameters can be updated by applications while the associated CRTC is
+ * disabled. The DRM/KMS core will store the parameters in the plane state,
+ * which will be available to the driver when the CRTC is turned on. As a result
+ * most drivers don't need to be immediately notified of plane updates for a
+ * disabled CRTC.
+ *
+ * Unless otherwise needed, drivers are advised to set the @active_only
+ * parameters to true in order not to receive plane update notifications related
+ * to a disabled CRTC. This avoids the need to manually ignore plane updates in
+ * driver code when the driver and/or hardware can't or just don't need to deal
+ * with updates on disabled CRTCs, for example when supporting runtime PM.
+ *
+ * The drm_atomic_helper_commit() default implementation only sets @active_only
+ * to false to most closely match the behaviour of the legacy helpers. This should
+ * not be copied blindly by drivers.
*/
void drm_atomic_helper_commit_planes(struct drm_device *dev,
- struct drm_atomic_state *old_state)
+ struct drm_atomic_state *old_state,
+ bool active_only)
{
struct drm_crtc *crtc;
struct drm_crtc_state *old_crtc_state;
if (!funcs || !funcs->atomic_begin)
continue;
+ if (active_only && !crtc->state->active)
+ continue;
+
funcs->atomic_begin(crtc, old_crtc_state);
}
for_each_plane_in_state(old_state, plane, old_plane_state, i) {
const struct drm_plane_helper_funcs *funcs;
+ bool disabling;
funcs = plane->helper_private;
if (!funcs)
continue;
+ disabling = drm_atomic_plane_disabling(plane, old_plane_state);
+
+ if (active_only) {
+ /*
+ * Skip planes related to inactive CRTCs. If the plane
+ * is enabled use the state of the current CRTC. If the
+ * plane is being disabled use the state of the old
+ * CRTC to avoid skipping planes being disabled on an
+ * active CRTC.
+ */
+ if (!disabling && !plane_crtc_active(plane->state))
+ continue;
+ if (disabling && !plane_crtc_active(old_plane_state))
+ continue;
+ }
+
/*
* Special-case disabling the plane if drivers support it.
*/
- if (drm_atomic_plane_disabling(plane, old_plane_state) &&
- funcs->atomic_disable)
+ if (disabling && funcs->atomic_disable)
funcs->atomic_disable(plane, old_plane_state);
- else if (plane->state->crtc ||
- drm_atomic_plane_disabling(plane, old_plane_state))
+ else if (plane->state->crtc || disabling)
funcs->atomic_update(plane, old_plane_state);
}
if (!funcs || !funcs->atomic_flush)
continue;
+ if (active_only && !crtc->state->active)
+ continue;
+
funcs->atomic_flush(crtc, old_crtc_state);
}
}
for_each_plane_in_state(old_state, plane, plane_state, i) {
const struct drm_plane_helper_funcs *funcs;
- struct drm_framebuffer *old_fb;
funcs = plane->helper_private;
- old_fb = plane_state->fb;
-
- if (old_fb && funcs->cleanup_fb)
- funcs->cleanup_fb(plane, old_fb, plane_state);
+ if (funcs->cleanup_fb)
+ funcs->cleanup_fb(plane, plane_state);
}
}
EXPORT_SYMBOL(drm_atomic_helper_cleanup_planes);
*
* 4. Actually commit the hardware state.
*
- * 5. Call drm_atomic_helper_cleanup_planes with @state, which since step 3
+ * 5. Call drm_atomic_helper_cleanup_planes() with @state, which since step 3
* contains the old state. Also do any other cleanup required with that state.
*/
void drm_atomic_helper_swap_state(struct drm_device *dev,
goto fail;
}
- ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
+ ret = __drm_atomic_helper_disable_plane(plane, plane_state);
if (ret != 0)
goto fail;
- drm_atomic_set_fb_for_plane(plane_state, NULL);
- plane_state->crtc_x = 0;
- plane_state->crtc_y = 0;
- plane_state->crtc_h = 0;
- plane_state->crtc_w = 0;
- plane_state->src_x = 0;
- plane_state->src_y = 0;
- plane_state->src_h = 0;
- plane_state->src_w = 0;
-
- if (plane == plane->crtc->cursor)
- state->legacy_cursor_update = true;
ret = drm_atomic_commit(state);
if (ret != 0)
}
EXPORT_SYMBOL(drm_atomic_helper_disable_plane);
+/* just used from fb-helper and atomic-helper: */
+int __drm_atomic_helper_disable_plane(struct drm_plane *plane,
+ struct drm_plane_state *plane_state)
+{
+ int ret;
+
+ ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
+ if (ret != 0)
+ return ret;
+
+ drm_atomic_set_fb_for_plane(plane_state, NULL);
+ plane_state->crtc_x = 0;
+ plane_state->crtc_y = 0;
+ plane_state->crtc_h = 0;
+ plane_state->crtc_w = 0;
+ plane_state->src_x = 0;
+ plane_state->src_y = 0;
+ plane_state->src_h = 0;
+ plane_state->src_w = 0;
+
+ if (plane->crtc && (plane == plane->crtc->cursor))
+ plane_state->state->legacy_cursor_update = true;
+
+ return 0;
+}
+
static int update_output_state(struct drm_atomic_state *state,
struct drm_mode_set *set)
{
{
struct drm_atomic_state *state;
struct drm_crtc *crtc = set->crtc;
- struct drm_crtc_state *crtc_state;
- struct drm_plane_state *primary_state;
int ret = 0;
state = drm_atomic_state_alloc(crtc->dev);
state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
retry:
- crtc_state = drm_atomic_get_crtc_state(state, crtc);
- if (IS_ERR(crtc_state)) {
- ret = PTR_ERR(crtc_state);
+ ret = __drm_atomic_helper_set_config(set, state);
+ if (ret != 0)
goto fail;
- }
- primary_state = drm_atomic_get_plane_state(state, crtc->primary);
- if (IS_ERR(primary_state)) {
- ret = PTR_ERR(primary_state);
+ ret = drm_atomic_commit(state);
+ if (ret != 0)
goto fail;
- }
+
+ /* Driver takes ownership of state on successful commit. */
+ return 0;
+fail:
+ if (ret == -EDEADLK)
+ goto backoff;
+
+ drm_atomic_state_free(state);
+
+ return ret;
+backoff:
+ drm_atomic_state_clear(state);
+ drm_atomic_legacy_backoff(state);
+
+ /*
+ * Someone might have exchanged the framebuffer while we dropped locks
+ * in the backoff code. We need to fix up the fb refcount tracking the
+ * core does for us.
+ */
+ crtc->primary->old_fb = crtc->primary->fb;
+
+ goto retry;
+}
+EXPORT_SYMBOL(drm_atomic_helper_set_config);
+
+/* just used from fb-helper and atomic-helper: */
+int __drm_atomic_helper_set_config(struct drm_mode_set *set,
+ struct drm_atomic_state *state)
+{
+ struct drm_crtc_state *crtc_state;
+ struct drm_plane_state *primary_state;
+ struct drm_crtc *crtc = set->crtc;
+ int ret;
+
+ crtc_state = drm_atomic_get_crtc_state(state, crtc);
+ if (IS_ERR(crtc_state))
+ return PTR_ERR(crtc_state);
+
+ primary_state = drm_atomic_get_plane_state(state, crtc->primary);
+ if (IS_ERR(primary_state))
+ return PTR_ERR(primary_state);
if (!set->mode) {
WARN_ON(set->fb);
ret = drm_atomic_set_mode_for_crtc(crtc_state, NULL);
if (ret != 0)
- goto fail;
+ return ret;
crtc_state->active = false;
ret = drm_atomic_set_crtc_for_plane(primary_state, NULL);
if (ret != 0)
- goto fail;
+ return ret;
drm_atomic_set_fb_for_plane(primary_state, NULL);
ret = drm_atomic_set_mode_for_crtc(crtc_state, set->mode);
if (ret != 0)
- goto fail;
+ return ret;
crtc_state->active = true;
ret = drm_atomic_set_crtc_for_plane(primary_state, crtc);
if (ret != 0)
- goto fail;
+ return ret;
+
drm_atomic_set_fb_for_plane(primary_state, set->fb);
primary_state->crtc_x = 0;
primary_state->crtc_y = 0;
primary_state->crtc_w = set->mode->hdisplay;
primary_state->src_x = set->x << 16;
primary_state->src_y = set->y << 16;
- primary_state->src_h = set->mode->vdisplay << 16;
- primary_state->src_w = set->mode->hdisplay << 16;
+ if (primary_state->rotation & (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270))) {
+ primary_state->src_h = set->mode->hdisplay << 16;
+ primary_state->src_w = set->mode->vdisplay << 16;
+ } else {
+ primary_state->src_h = set->mode->vdisplay << 16;
+ primary_state->src_w = set->mode->hdisplay << 16;
+ }
commit:
ret = update_output_state(state, set);
if (ret)
- goto fail;
-
- ret = drm_atomic_commit(state);
- if (ret != 0)
- goto fail;
+ return ret;
- /* Driver takes ownership of state on successful commit. */
return 0;
-fail:
- if (ret == -EDEADLK)
- goto backoff;
-
- drm_atomic_state_free(state);
-
- return ret;
-backoff:
- drm_atomic_state_clear(state);
- drm_atomic_legacy_backoff(state);
-
- /*
- * Someone might have exchanged the framebuffer while we dropped locks
- * in the backoff code. We need to fix up the fb refcount tracking the
- * core does for us.
- */
- crtc->primary->old_fb = crtc->primary->fb;
-
- goto retry;
}
-EXPORT_SYMBOL(drm_atomic_helper_set_config);
/**
* drm_atomic_helper_crtc_set_property - helper for crtc properties
}
EXPORT_SYMBOL(drm_atomic_helper_connector_duplicate_state);
+/**
+ * drm_atomic_helper_duplicate_state - duplicate an atomic state object
+ * @dev: DRM device
+ * @ctx: lock acquisition context
+ *
+ * Makes a copy of the current atomic state by looping over all objects and
+ * duplicating their respective states.
+ *
+ * Note that this treats atomic state as persistent between save and restore.
+ * Drivers must make sure that this is possible and won't result in confusion
+ * or erroneous behaviour.
+ *
+ * Note that if callers haven't already acquired all modeset locks this might
+ * return -EDEADLK, which must be handled by calling drm_modeset_backoff().
+ *
+ * Returns:
+ * A pointer to the copy of the atomic state object on success or an
+ * ERR_PTR()-encoded error code on failure.
+ */
+struct drm_atomic_state *
+drm_atomic_helper_duplicate_state(struct drm_device *dev,
+ struct drm_modeset_acquire_ctx *ctx)
+{
+ struct drm_atomic_state *state;
+ struct drm_connector *conn;
+ struct drm_plane *plane;
+ struct drm_crtc *crtc;
+ int err = 0;
+
+ state = drm_atomic_state_alloc(dev);
+ if (!state)
+ return ERR_PTR(-ENOMEM);
+
+ state->acquire_ctx = ctx;
+
+ drm_for_each_crtc(crtc, dev) {
+ struct drm_crtc_state *crtc_state;
+
+ crtc_state = drm_atomic_get_crtc_state(state, crtc);
+ if (IS_ERR(crtc_state)) {
+ err = PTR_ERR(crtc_state);
+ goto free;
+ }
+ }
+
+ drm_for_each_plane(plane, dev) {
+ struct drm_plane_state *plane_state;
+
+ plane_state = drm_atomic_get_plane_state(state, plane);
+ if (IS_ERR(plane_state)) {
+ err = PTR_ERR(plane_state);
+ goto free;
+ }
+ }
+
+ drm_for_each_connector(conn, dev) {
+ struct drm_connector_state *conn_state;
+
+ conn_state = drm_atomic_get_connector_state(state, conn);
+ if (IS_ERR(conn_state)) {
+ err = PTR_ERR(conn_state);
+ goto free;
+ }
+ }
+
+ /* clear the acquire context so that it isn't accidentally reused */
+ state->acquire_ctx = NULL;
+
+free:
+ if (err < 0) {
+ drm_atomic_state_free(state);
+ state = ERR_PTR(err);
+ }
+
+ return state;
+}
+EXPORT_SYMBOL(drm_atomic_helper_duplicate_state);
+
/**
* __drm_atomic_helper_connector_destroy_state - release connector state
* @connector: connector object
}
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
/**
* Add AGP buffers for DMA transfers.
*
return 0;
}
EXPORT_SYMBOL(drm_legacy_addbufs_agp);
-#endif /* __OS_HAS_AGP */
+#endif /* CONFIG_AGP */
int drm_legacy_addbufs_pci(struct drm_device *dev,
struct drm_buf_desc *request)
if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA))
return -EINVAL;
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (request->flags & _DRM_AGP_BUFFER)
ret = drm_legacy_addbufs_agp(dev, request);
else
* reference counted modeset objects like framebuffers.
*
* Returns:
- * New unique (relative to other objects in @dev) integer identifier for the
- * object.
+ * Zero on success, error code on failure.
*/
int drm_mode_object_get(struct drm_device *dev,
struct drm_mode_object *obj, uint32_t obj_type)
out:
mutex_unlock(&dev->mode_config.fb_lock);
- return 0;
+ return ret;
}
EXPORT_SYMBOL(drm_framebuffer_init);
*/
void drm_framebuffer_unregister_private(struct drm_framebuffer *fb)
{
- struct drm_device *dev = fb->dev;
+ struct drm_device *dev;
+
+ if (!fb)
+ return;
+
+ dev = fb->dev;
mutex_lock(&dev->mode_config.fb_lock);
/* Mark fb as reaped and drop idr ref. */
*/
void drm_framebuffer_remove(struct drm_framebuffer *fb)
{
- struct drm_device *dev = fb->dev;
+ struct drm_device *dev;
struct drm_crtc *crtc;
struct drm_plane *plane;
struct drm_mode_set set;
int ret;
+ if (!fb)
+ return;
+
+ dev = fb->dev;
+
WARN_ON(!list_empty(&fb->filp_head));
/*
crtc->dev = dev;
crtc->funcs = funcs;
- crtc->invert_dimensions = false;
drm_modeset_lock_init(&crtc->mutex);
ret = drm_mode_object_get(dev, &crtc->base, DRM_MODE_OBJECT_CRTC);
*/
int drm_mode_create_tv_properties(struct drm_device *dev,
unsigned int num_modes,
- char *modes[])
+ const char * const modes[])
{
struct drm_property *tv_selector;
struct drm_property *tv_subconnector;
"select subconnector",
drm_tv_select_enum_list,
ARRAY_SIZE(drm_tv_select_enum_list));
+ if (!tv_selector)
+ goto nomem;
+
dev->mode_config.tv_select_subconnector_property = tv_selector;
tv_subconnector =
"subconnector",
drm_tv_subconnector_enum_list,
ARRAY_SIZE(drm_tv_subconnector_enum_list));
+ if (!tv_subconnector)
+ goto nomem;
dev->mode_config.tv_subconnector_property = tv_subconnector;
/*
*/
dev->mode_config.tv_left_margin_property =
drm_property_create_range(dev, 0, "left margin", 0, 100);
+ if (!dev->mode_config.tv_left_margin_property)
+ goto nomem;
dev->mode_config.tv_right_margin_property =
drm_property_create_range(dev, 0, "right margin", 0, 100);
+ if (!dev->mode_config.tv_right_margin_property)
+ goto nomem;
dev->mode_config.tv_top_margin_property =
drm_property_create_range(dev, 0, "top margin", 0, 100);
+ if (!dev->mode_config.tv_top_margin_property)
+ goto nomem;
dev->mode_config.tv_bottom_margin_property =
drm_property_create_range(dev, 0, "bottom margin", 0, 100);
+ if (!dev->mode_config.tv_bottom_margin_property)
+ goto nomem;
dev->mode_config.tv_mode_property =
drm_property_create(dev, DRM_MODE_PROP_ENUM,
"mode", num_modes);
+ if (!dev->mode_config.tv_mode_property)
+ goto nomem;
+
for (i = 0; i < num_modes; i++)
drm_property_add_enum(dev->mode_config.tv_mode_property, i,
i, modes[i]);
dev->mode_config.tv_brightness_property =
drm_property_create_range(dev, 0, "brightness", 0, 100);
+ if (!dev->mode_config.tv_brightness_property)
+ goto nomem;
dev->mode_config.tv_contrast_property =
drm_property_create_range(dev, 0, "contrast", 0, 100);
+ if (!dev->mode_config.tv_contrast_property)
+ goto nomem;
dev->mode_config.tv_flicker_reduction_property =
drm_property_create_range(dev, 0, "flicker reduction", 0, 100);
+ if (!dev->mode_config.tv_flicker_reduction_property)
+ goto nomem;
dev->mode_config.tv_overscan_property =
drm_property_create_range(dev, 0, "overscan", 0, 100);
+ if (!dev->mode_config.tv_overscan_property)
+ goto nomem;
dev->mode_config.tv_saturation_property =
drm_property_create_range(dev, 0, "saturation", 0, 100);
+ if (!dev->mode_config.tv_saturation_property)
+ goto nomem;
dev->mode_config.tv_hue_property =
drm_property_create_range(dev, 0, "hue", 0, 100);
+ if (!dev->mode_config.tv_hue_property)
+ goto nomem;
return 0;
+nomem:
+ return -ENOMEM;
}
EXPORT_SYMBOL(drm_mode_create_tv_properties);
return -EINVAL;
}
+static int check_src_coords(uint32_t src_x, uint32_t src_y,
+ uint32_t src_w, uint32_t src_h,
+ const struct drm_framebuffer *fb)
+{
+ unsigned int fb_width, fb_height;
+
+ fb_width = fb->width << 16;
+ fb_height = fb->height << 16;
+
+ /* Make sure source coordinates are inside the fb. */
+ if (src_w > fb_width ||
+ src_x > fb_width - src_w ||
+ src_h > fb_height ||
+ src_y > fb_height - src_h) {
+ DRM_DEBUG_KMS("Invalid source coordinates "
+ "%u.%06ux%u.%06u+%u.%06u+%u.%06u\n",
+ src_w >> 16, ((src_w & 0xffff) * 15625) >> 10,
+ src_h >> 16, ((src_h & 0xffff) * 15625) >> 10,
+ src_x >> 16, ((src_x & 0xffff) * 15625) >> 10,
+ src_y >> 16, ((src_y & 0xffff) * 15625) >> 10);
+ return -ENOSPC;
+ }
+
+ return 0;
+}
+
/*
* setplane_internal - setplane handler for internal callers
*
uint32_t src_w, uint32_t src_h)
{
int ret = 0;
- unsigned int fb_width, fb_height;
/* No fb means shut it down */
if (!fb) {
crtc_y > INT_MAX - (int32_t) crtc_h) {
DRM_DEBUG_KMS("Invalid CRTC coordinates %ux%u+%d+%d\n",
crtc_w, crtc_h, crtc_x, crtc_y);
- return -ERANGE;
+ ret = -ERANGE;
+ goto out;
}
-
- fb_width = fb->width << 16;
- fb_height = fb->height << 16;
-
- /* Make sure source coordinates are inside the fb. */
- if (src_w > fb_width ||
- src_x > fb_width - src_w ||
- src_h > fb_height ||
- src_y > fb_height - src_h) {
- DRM_DEBUG_KMS("Invalid source coordinates "
- "%u.%06ux%u.%06u+%u.%06u+%u.%06u\n",
- src_w >> 16, ((src_w & 0xffff) * 15625) >> 10,
- src_h >> 16, ((src_h & 0xffff) * 15625) >> 10,
- src_x >> 16, ((src_x & 0xffff) * 15625) >> 10,
- src_y >> 16, ((src_y & 0xffff) * 15625) >> 10);
- ret = -ENOSPC;
+ ret = check_src_coords(src_x, src_y, src_w, src_h, fb);
+ if (ret)
goto out;
- }
plane->old_fb = plane->fb;
ret = plane->funcs->update_plane(plane, crtc, fb,
drm_crtc_get_hv_timing(mode, &hdisplay, &vdisplay);
- if (crtc->invert_dimensions)
+ if (crtc->state &&
+ crtc->primary->state->rotation & (BIT(DRM_ROTATE_90) |
+ BIT(DRM_ROTATE_270)))
swap(hdisplay, vdisplay);
- if (hdisplay > fb->width ||
- vdisplay > fb->height ||
- x > fb->width - hdisplay ||
- y > fb->height - vdisplay) {
- DRM_DEBUG_KMS("Invalid fb size %ux%u for CRTC viewport %ux%u+%d+%d%s.\n",
- fb->width, fb->height, hdisplay, vdisplay, x, y,
- crtc->invert_dimensions ? " (inverted)" : "");
- return -ENOSPC;
- }
-
- return 0;
+ return check_src_coords(x << 16, y << 16,
+ hdisplay << 16, vdisplay << 16, fb);
}
EXPORT_SYMBOL(drm_crtc_check_viewport);
if (!found)
goto fail_lookup;
- /* Mark fb as reaped, we still have a ref from fpriv->fbs. */
- __drm_framebuffer_unregister(dev, fb);
-
list_del_init(&fb->filp_head);
mutex_unlock(&dev->mode_config.fb_lock);
mutex_unlock(&file_priv->fbs_lock);
- drm_framebuffer_remove(fb);
+ drm_framebuffer_unreference(fb);
return 0;
*/
void drm_fb_release(struct drm_file *priv)
{
- struct drm_device *dev = priv->minor->dev;
struct drm_framebuffer *fb, *tfb;
/*
* at it any more.
*/
list_for_each_entry_safe(fb, tfb, &priv->fbs, filp_head) {
-
- mutex_lock(&dev->mode_config.fb_lock);
- /* Mark fb as reaped, we still have a ref from fpriv->fbs. */
- __drm_framebuffer_unregister(dev, fb);
- mutex_unlock(&dev->mode_config.fb_lock);
-
list_del_init(&fb->filp_head);
- /* This will also drop the fpriv->fbs reference. */
- drm_framebuffer_remove(fb);
+ /* This drops the fpriv->fbs reference. */
+ drm_framebuffer_unreference(fb);
}
}
goto out;
}
- ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y, &crtc->mode, fb);
+ if (crtc->state) {
+ const struct drm_plane_state *state = crtc->primary->state;
+
+ ret = check_src_coords(state->src_x, state->src_y,
+ state->src_w, state->src_h, fb);
+ } else {
+ ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y, &crtc->mode, fb);
+ }
if (ret)
goto out;
{
if (rotation & ~supported_rotations) {
rotation ^= BIT(DRM_REFLECT_X) | BIT(DRM_REFLECT_Y);
- rotation = (rotation & ~0xf) | BIT((ffs(rotation & 0xf) + 1) % 4);
+ rotation = (rotation & DRM_REFLECT_MASK) |
+ BIT((ffs(rotation & DRM_ROTATE_MASK) + 1) % 4);
}
return rotation;
*/
WARN_ON(!list_empty(&dev->mode_config.fb_list));
list_for_each_entry_safe(fb, fbt, &dev->mode_config.fb_list, head) {
- drm_framebuffer_remove(fb);
+ drm_framebuffer_free(&fb->refcount);
}
list_for_each_entry_safe(plane, plt, &dev->mode_config.plane_list,
I2C_FUNC_10BIT_ADDR;
}
+static void drm_dp_i2c_msg_write_status_update(struct drm_dp_aux_msg *msg)
+{
+ /*
+ * In case of i2c defer or short i2c ack reply to a write,
+ * we need to switch to WRITE_STATUS_UPDATE to drain the
+ * rest of the message
+ */
+ if ((msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_I2C_WRITE) {
+ msg->request &= DP_AUX_I2C_MOT;
+ msg->request |= DP_AUX_I2C_WRITE_STATUS_UPDATE;
+ }
+}
+
#define AUX_PRECHARGE_LEN 10 /* 10 to 16 */
#define AUX_SYNC_LEN (16 + 4) /* preamble + AUX_SYNC_END */
#define AUX_STOP_LEN 4
* Both native ACK and I2C ACK replies received. We
* can assume the transfer was successful.
*/
+ if (ret != msg->size)
+ drm_dp_i2c_msg_write_status_update(msg);
return ret;
case DP_AUX_I2C_REPLY_NACK:
if (defer_i2c < 7)
defer_i2c++;
usleep_range(AUX_RETRY_INTERVAL, AUX_RETRY_INTERVAL + 100);
+ drm_dp_i2c_msg_write_status_update(msg);
+
continue;
default:
return -EREMOTEIO;
}
+static void drm_dp_i2c_msg_set_request(struct drm_dp_aux_msg *msg,
+ const struct i2c_msg *i2c_msg)
+{
+ msg->request = (i2c_msg->flags & I2C_M_RD) ?
+ DP_AUX_I2C_READ : DP_AUX_I2C_WRITE;
+ msg->request |= DP_AUX_I2C_MOT;
+}
+
/*
* Keep retrying drm_dp_i2c_do_msg until all data has been transferred.
*
for (i = 0; i < num; i++) {
msg.address = msgs[i].addr;
- msg.request = (msgs[i].flags & I2C_M_RD) ?
- DP_AUX_I2C_READ :
- DP_AUX_I2C_WRITE;
- msg.request |= DP_AUX_I2C_MOT;
+ drm_dp_i2c_msg_set_request(&msg, &msgs[i]);
/* Send a bare address packet to start the transaction.
* Zero sized messages specify an address only (bare
* address) transaction.
msg.buffer = NULL;
msg.size = 0;
err = drm_dp_i2c_do_msg(aux, &msg);
+
+ /*
+ * Reset msg.request in case in case it got
+ * changed into a WRITE_STATUS_UPDATE.
+ */
+ drm_dp_i2c_msg_set_request(&msg, &msgs[i]);
+
if (err < 0)
break;
/* We want each transaction to be as large as possible, but
msg.size = min(transfer_size, msgs[i].len - j);
err = drm_dp_i2c_drain_msg(aux, &msg);
+
+ /*
+ * Reset msg.request in case in case it got
+ * changed into a WRITE_STATUS_UPDATE.
+ */
+ drm_dp_i2c_msg_set_request(&msg, &msgs[i]);
+
if (err < 0)
break;
transfer_size = err;
struct drm_dp_mst_port *port,
int offset, int size, u8 *bytes);
-static int drm_dp_send_link_address(struct drm_dp_mst_topology_mgr *mgr,
- struct drm_dp_mst_branch *mstb);
+static void drm_dp_send_link_address(struct drm_dp_mst_topology_mgr *mgr,
+ struct drm_dp_mst_branch *mstb);
static int drm_dp_send_enum_path_resources(struct drm_dp_mst_topology_mgr *mgr,
struct drm_dp_mst_branch *mstb,
struct drm_dp_mst_port *port);
struct drm_dp_mst_port *port, *tmp;
bool wake_tx = false;
- cancel_work_sync(&mstb->mgr->work);
-
/*
* destroy all ports - don't need lock
* as there are no more references to the mst branch
{
struct drm_dp_mst_port *port = container_of(kref, struct drm_dp_mst_port, kref);
struct drm_dp_mst_topology_mgr *mgr = port->mgr;
+
if (!port->input) {
port->vcpi.num_slots = 0;
kfree(port->cached_edid);
- /* we can't destroy the connector here, as
- we might be holding the mode_config.mutex
- from an EDID retrieval */
+ /*
+ * The only time we don't have a connector
+ * on an output port is if the connector init
+ * fails.
+ */
if (port->connector) {
+ /* we can't destroy the connector here, as
+ * we might be holding the mode_config.mutex
+ * from an EDID retrieval */
+
mutex_lock(&mgr->destroy_connector_lock);
list_add(&port->next, &mgr->destroy_connector_list);
mutex_unlock(&mgr->destroy_connector_lock);
schedule_work(&mgr->destroy_connector_work);
return;
}
+ /* no need to clean up vcpi
+ * as if we have no connector we never setup a vcpi */
drm_dp_port_teardown_pdt(port, port->pdt);
-
- if (!port->input && port->vcpi.vcpi > 0)
- drm_dp_mst_put_payload_id(mgr, port->vcpi.vcpi);
}
kfree(port);
-
- (*mgr->cbs->hotplug)(mgr);
}
static void drm_dp_put_port(struct drm_dp_mst_port *port)
}
}
-static void build_mst_prop_path(struct drm_dp_mst_port *port,
- struct drm_dp_mst_branch *mstb,
+static void build_mst_prop_path(const struct drm_dp_mst_branch *mstb,
+ int pnum,
char *proppath,
size_t proppath_size)
{
snprintf(temp, sizeof(temp), "-%d", port_num);
strlcat(proppath, temp, proppath_size);
}
- snprintf(temp, sizeof(temp), "-%d", port->port_num);
+ snprintf(temp, sizeof(temp), "-%d", pnum);
strlcat(proppath, temp, proppath_size);
}
drm_dp_port_teardown_pdt(port, old_pdt);
ret = drm_dp_port_setup_pdt(port);
- if (ret == true) {
+ if (ret == true)
drm_dp_send_link_address(mstb->mgr, port->mstb);
- port->mstb->link_address_sent = true;
- }
}
if (created && !port->input) {
char proppath[255];
- build_mst_prop_path(port, mstb, proppath, sizeof(proppath));
- port->connector = (*mstb->mgr->cbs->add_connector)(mstb->mgr, port, proppath);
- if (port->port_num >= 8) {
+ build_mst_prop_path(mstb, port->port_num, proppath, sizeof(proppath));
+ port->connector = (*mstb->mgr->cbs->add_connector)(mstb->mgr, port, proppath);
+ if (!port->connector) {
+ /* remove it from the port list */
+ mutex_lock(&mstb->mgr->lock);
+ list_del(&port->next);
+ mutex_unlock(&mstb->mgr->lock);
+ /* drop port list reference */
+ drm_dp_put_port(port);
+ goto out;
+ }
+ if (port->port_num >= DP_MST_LOGICAL_PORT_0) {
port->cached_edid = drm_get_edid(port->connector, &port->aux.ddc);
+ drm_mode_connector_set_tile_property(port->connector);
}
+ (*mstb->mgr->cbs->register_connector)(port->connector);
}
+out:
/* put reference to this port */
drm_dp_put_port(port);
}
list_for_each_entry(port, &mstb->ports, next) {
if (port->port_num == port_num) {
- if (!port->mstb) {
+ mstb = port->mstb;
+ if (!mstb) {
DRM_ERROR("failed to lookup MSTB with lct %d, rad %02x\n", lct, rad[0]);
- return NULL;
+ goto out;
}
- mstb = port->mstb;
break;
}
}
}
kref_get(&mstb->kref);
+out:
mutex_unlock(&mgr->lock);
return mstb;
}
{
struct drm_dp_mst_port *port;
struct drm_dp_mst_branch *mstb_child;
- if (!mstb->link_address_sent) {
+ if (!mstb->link_address_sent)
drm_dp_send_link_address(mgr, mstb);
- mstb->link_address_sent = true;
- }
+
list_for_each_entry(port, &mstb->ports, next) {
if (port->input)
continue;
mutex_unlock(&mgr->qlock);
}
-static int drm_dp_send_link_address(struct drm_dp_mst_topology_mgr *mgr,
- struct drm_dp_mst_branch *mstb)
+static void drm_dp_send_link_address(struct drm_dp_mst_topology_mgr *mgr,
+ struct drm_dp_mst_branch *mstb)
{
int len;
struct drm_dp_sideband_msg_tx *txmsg;
txmsg = kzalloc(sizeof(*txmsg), GFP_KERNEL);
if (!txmsg)
- return -ENOMEM;
+ return;
txmsg->dst = mstb;
len = build_link_address(txmsg);
+ mstb->link_address_sent = true;
drm_dp_queue_down_tx(mgr, txmsg);
ret = drm_dp_mst_wait_tx_reply(mstb, txmsg);
}
(*mgr->cbs->hotplug)(mgr);
}
- } else
+ } else {
+ mstb->link_address_sent = false;
DRM_DEBUG_KMS("link address failed %d\n", ret);
+ }
kfree(txmsg);
- return 0;
}
static int drm_dp_send_enum_path_resources(struct drm_dp_mst_topology_mgr *mgr,
drm_dp_dpcd_writeb(mgr->aux, DP_MSTM_CTRL,
DP_MST_EN | DP_UPSTREAM_IS_SRC);
mutex_unlock(&mgr->lock);
+ flush_work(&mgr->work);
+ flush_work(&mgr->destroy_connector_work);
}
EXPORT_SYMBOL(drm_dp_mst_topology_mgr_suspend);
if (port->cached_edid)
edid = drm_edid_duplicate(port->cached_edid);
- else
+ else {
edid = drm_get_edid(connector, &port->aux.ddc);
-
- drm_mode_connector_set_tile_property(connector);
+ drm_mode_connector_set_tile_property(connector);
+ }
drm_dp_put_port(port);
return edid;
}
{
struct drm_dp_mst_topology_mgr *mgr = container_of(work, struct drm_dp_mst_topology_mgr, destroy_connector_work);
struct drm_dp_mst_port *port;
-
+ bool send_hotplug = false;
/*
* Not a regular list traverse as we have to drop the destroy
* connector lock before destroying the connector, to avoid AB->BA
if (!port->input && port->vcpi.vcpi > 0)
drm_dp_mst_put_payload_id(mgr, port->vcpi.vcpi);
kfree(port);
+ send_hotplug = true;
}
+ if (send_hotplug)
+ (*mgr->cbs->hotplug)(mgr);
}
/**
*/
void drm_dp_mst_topology_mgr_destroy(struct drm_dp_mst_topology_mgr *mgr)
{
+ flush_work(&mgr->work);
flush_work(&mgr->destroy_connector_work);
mutex_lock(&mgr->payload_lock);
kfree(mgr->payloads);
#include "drm_legacy.h"
#include "drm_internal.h"
-unsigned int drm_debug = 0; /* 1 to enable debug output */
+unsigned int drm_debug = 0; /* bitmask of DRM_UT_x */
EXPORT_SYMBOL(drm_debug);
-bool drm_atomic = 0;
-
MODULE_AUTHOR(CORE_AUTHOR);
MODULE_DESCRIPTION(CORE_DESC);
MODULE_LICENSE("GPL and additional rights");
static DEFINE_SPINLOCK(drm_minor_lock);
static struct idr drm_minors_idr;
-struct class *drm_class;
static struct dentry *drm_debugfs_root;
void drm_err(const char *format, ...)
drm_dev_unref(minor->dev);
}
+/**
+ * DOC: driver instance overview
+ *
+ * A device instance for a drm driver is represented by struct &drm_device. This
+ * is allocated with drm_dev_alloc(), usually from bus-specific ->probe()
+ * callbacks implemented by the driver. The driver then needs to initialize all
+ * the various subsystems for the drm device like memory management, vblank
+ * handling, modesetting support and intial output configuration plus obviously
+ * initialize all the corresponding hardware bits. An important part of this is
+ * also calling drm_dev_set_unique() to set the userspace-visible unique name of
+ * this device instance. Finally when everything is up and running and ready for
+ * userspace the device instance can be published using drm_dev_register().
+ *
+ * There is also deprecated support for initalizing device instances using
+ * bus-specific helpers and the ->load() callback. But due to
+ * backwards-compatibility needs the device instance have to be published too
+ * early, which requires unpretty global locking to make safe and is therefore
+ * only support for existing drivers not yet converted to the new scheme.
+ *
+ * When cleaning up a device instance everything needs to be done in reverse:
+ * First unpublish the device instance with drm_dev_unregister(). Then clean up
+ * any other resources allocated at device initialization and drop the driver's
+ * reference to &drm_device using drm_dev_unref().
+ *
+ * Note that the lifetime rules for &drm_device instance has still a lot of
+ * historical baggage. Hence use the reference counting provided by
+ * drm_dev_ref() and drm_dev_unref() only carefully.
+ *
+ * Also note that embedding of &drm_device is currently not (yet) supported (but
+ * it would be easy to add). Drivers can store driver-private data in the
+ * dev_priv field of &drm_device.
+ */
+
/**
* drm_put_dev - Unregister and release a DRM device
* @dev: DRM device
*
* Called at module unload time or when a PCI device is unplugged.
*
- * Use of this function is discouraged. It will eventually go away completely.
- * Please use drm_dev_unregister() and drm_dev_unref() explicitly instead.
- *
* Cleans up all DRM device, calling drm_lastclose().
+ *
+ * Note: Use of this function is deprecated. It will eventually go away
+ * completely. Please use drm_dev_unregister() and drm_dev_unref() explicitly
+ * instead to make sure that the device isn't userspace accessible any more
+ * while teardown is in progress, ensuring that userspace can't access an
+ * inconsistent state.
*/
void drm_put_dev(struct drm_device *dev)
{
*
* Allocate and initialize a new DRM device. No device registration is done.
* Call drm_dev_register() to advertice the device to user space and register it
- * with other core subsystems.
+ * with other core subsystems. This should be done last in the device
+ * initialization sequence to make sure userspace can't access an inconsistent
+ * state.
*
* The initial ref-count of the object is 1. Use drm_dev_ref() and
* drm_dev_unref() to take and drop further ref-counts.
ret = drm_minor_alloc(dev, DRM_MINOR_CONTROL);
if (ret)
goto err_minors;
+
+ WARN_ON(driver->suspend || driver->resume);
}
if (drm_core_check_feature(dev, DRIVER_RENDER)) {
*
* Never call this twice on any device!
*
+ * NOTE: To ensure backward compatibility with existing drivers method this
+ * function calls the ->load() method after registering the device nodes,
+ * creating race conditions. Usage of the ->load() methods is therefore
+ * deprecated, drivers must perform all initialization before calling
+ * drm_dev_register().
+ *
* RETURNS:
* 0 on success, negative error code on failure.
*/
* Unregister the DRM device from the system. This does the reverse of
* drm_dev_register() but does not deallocate the device. The caller must call
* drm_dev_unref() to drop their final reference.
+ *
+ * This should be called first in the device teardown code to make sure
+ * userspace can't access the device instance any more.
*/
void drm_dev_unregister(struct drm_device *dev)
{
if (register_chrdev(DRM_MAJOR, "drm", &drm_stub_fops))
goto err_p1;
- drm_class = drm_sysfs_create(THIS_MODULE, "drm");
- if (IS_ERR(drm_class)) {
+ ret = drm_sysfs_init();
+ if (ret < 0) {
printk(KERN_ERR "DRM: Error creating drm class.\n");
- ret = PTR_ERR(drm_class);
goto err_p2;
}
static bool valid_inferred_mode(const struct drm_connector *connector,
const struct drm_display_mode *mode)
{
- struct drm_display_mode *m;
+ const struct drm_display_mode *m;
bool ok = false;
list_for_each_entry(m, &connector->probed_modes, head) {
return closure.modes;
}
+static void fixup_detailed_cea_mode_clock(struct drm_display_mode *mode);
+
static void
do_detailed_mode(struct detailed_timing *timing, void *c)
{
if (closure->preferred)
newmode->type |= DRM_MODE_TYPE_PREFERRED;
+ /*
+ * Detailed modes are limited to 10kHz pixel clock resolution,
+ * so fix up anything that looks like CEA/HDMI mode, but the clock
+ * is just slightly off.
+ */
+ fixup_detailed_cea_mode_clock(newmode);
+
drm_mode_probed_add(closure->connector, newmode);
closure->modes++;
closure->preferred = 0;
* and the 60Hz variant otherwise.
*/
if (cea_mode->vdisplay == 240 || cea_mode->vdisplay == 480)
- clock = clock * 1001 / 1000;
+ clock = DIV_ROUND_CLOSEST(clock * 1001, 1000);
else
- clock = DIV_ROUND_UP(clock * 1000, 1001);
+ clock = DIV_ROUND_CLOSEST(clock * 1000, 1001);
return clock;
}
return modes;
}
+static void fixup_detailed_cea_mode_clock(struct drm_display_mode *mode)
+{
+ const struct drm_display_mode *cea_mode;
+ int clock1, clock2, clock;
+ u8 mode_idx;
+ const char *type;
+
+ mode_idx = drm_match_cea_mode(mode) - 1;
+ if (mode_idx < ARRAY_SIZE(edid_cea_modes)) {
+ type = "CEA";
+ cea_mode = &edid_cea_modes[mode_idx];
+ clock1 = cea_mode->clock;
+ clock2 = cea_mode_alternate_clock(cea_mode);
+ } else {
+ mode_idx = drm_match_hdmi_mode(mode) - 1;
+ if (mode_idx < ARRAY_SIZE(edid_4k_modes)) {
+ type = "HDMI";
+ cea_mode = &edid_4k_modes[mode_idx];
+ clock1 = cea_mode->clock;
+ clock2 = hdmi_mode_alternate_clock(cea_mode);
+ } else {
+ return;
+ }
+ }
+
+ /* pick whichever is closest */
+ if (abs(mode->clock - clock1) < abs(mode->clock - clock2))
+ clock = clock1;
+ else
+ clock = clock2;
+
+ if (mode->clock == clock)
+ return;
+
+ DRM_DEBUG("detailed mode matches %s VIC %d, adjusting clock %d -> %d\n",
+ type, mode_idx + 1, mode->clock, clock);
+ mode->clock = clock;
+}
+
static void
parse_hdmi_vsdb(struct drm_connector *connector, const u8 *db)
{
* the sink doesn't support audio or video.
*/
int drm_av_sync_delay(struct drm_connector *connector,
- struct drm_display_mode *mode)
+ const struct drm_display_mode *mode)
{
int i = !!(mode->flags & DRM_MODE_FLAG_INTERLACE);
int a, v;
/**
* drm_select_eld - select one ELD from multiple HDMI/DP sinks
* @encoder: the encoder just changed display mode
- * @mode: the adjusted display mode
*
* It's possible for one encoder to be associated with multiple HDMI/DP sinks.
* The policy is now hard coded to simply use the first HDMI/DP sink's ELD.
* Return: The connector associated with the first HDMI/DP sink that has ELD
* attached to it.
*/
-struct drm_connector *drm_select_eld(struct drm_encoder *encoder,
- struct drm_display_mode *mode)
+struct drm_connector *drm_select_eld(struct drm_encoder *encoder)
{
struct drm_connector *connector;
struct drm_device *dev = encoder->dev;
"from built-in data or /lib/firmware instead. ");
#define GENERIC_EDIDS 6
-static const char *generic_edid_name[GENERIC_EDIDS] = {
+static const char * const generic_edid_name[GENERIC_EDIDS] = {
"edid/800x600.bin",
"edid/1024x768.bin",
"edid/1280x1024.bin",
int drm_load_edid_firmware(struct drm_connector *connector)
{
const char *connector_name = connector->name;
- char *edidname = edid_firmware, *last, *colon;
+ char *edidname, *last, *colon, *fwstr, *edidstr, *fallback = NULL;
int ret;
struct edid *edid;
- if (*edidname == '\0')
+ if (edid_firmware[0] == '\0')
return 0;
- colon = strchr(edidname, ':');
- if (colon != NULL) {
- if (strncmp(connector_name, edidname, colon - edidname))
- return 0;
- edidname = colon + 1;
- if (*edidname == '\0')
+ /*
+ * If there are multiple edid files specified and separated
+ * by commas, search through the list looking for one that
+ * matches the connector.
+ *
+ * If there's one or more that don't't specify a connector, keep
+ * the last one found one as a fallback.
+ */
+ fwstr = kstrdup(edid_firmware, GFP_KERNEL);
+ edidstr = fwstr;
+
+ while ((edidname = strsep(&edidstr, ","))) {
+ colon = strchr(edidname, ':');
+ if (colon != NULL) {
+ if (strncmp(connector_name, edidname, colon - edidname))
+ continue;
+ edidname = colon + 1;
+ break;
+ }
+
+ if (*edidname != '\0') /* corner case: multiple ',' */
+ fallback = edidname;
+ }
+
+ if (!edidname) {
+ if (!fallback) {
+ kfree(fwstr);
return 0;
+ }
+ edidname = fallback;
}
last = edidname + strlen(edidname) - 1;
*last = '\0';
edid = edid_load(connector, edidname, connector_name);
+ kfree(fwstr);
+
if (IS_ERR_OR_NULL(edid))
return 0;
#include <drm/drm_crtc.h>
#include <drm/drm_fb_helper.h>
#include <drm/drm_crtc_helper.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+
+static bool drm_fbdev_emulation = true;
+module_param_named(fbdev_emulation, drm_fbdev_emulation, bool, 0600);
+MODULE_PARM_DESC(fbdev_emulation,
+ "Enable legacy fbdev emulation [default=true]");
static LIST_HEAD(kernel_fb_helper_list);
struct drm_connector *connector;
int i;
+ if (!drm_fbdev_emulation)
+ return 0;
+
mutex_lock(&dev->mode_config.mutex);
drm_for_each_connector(connector, dev) {
struct drm_fb_helper_connector *fb_helper_connector;
struct drm_fb_helper_connector **temp;
struct drm_fb_helper_connector *fb_helper_connector;
+ if (!drm_fbdev_emulation)
+ return 0;
+
WARN_ON(!mutex_is_locked(&fb_helper->dev->mode_config.mutex));
if (fb_helper->connector_count + 1 > fb_helper->connector_info_alloc_count) {
temp = krealloc(fb_helper->connector_info, sizeof(struct drm_fb_helper_connector *) * (fb_helper->connector_count + 1), GFP_KERNEL);
struct drm_fb_helper_connector *fb_helper_connector;
int i, j;
+ if (!drm_fbdev_emulation)
+ return 0;
+
WARN_ON(!mutex_is_locked(&fb_helper->dev->mode_config.mutex));
for (i = 0; i < fb_helper->connector_count; i++) {
}
EXPORT_SYMBOL(drm_fb_helper_debug_leave);
-static bool restore_fbdev_mode(struct drm_fb_helper *fb_helper)
+static int restore_fbdev_mode_atomic(struct drm_fb_helper *fb_helper)
+{
+ struct drm_device *dev = fb_helper->dev;
+ struct drm_plane *plane;
+ struct drm_atomic_state *state;
+ int i, ret;
+
+ state = drm_atomic_state_alloc(dev);
+ if (!state)
+ return -ENOMEM;
+
+ state->acquire_ctx = dev->mode_config.acquire_ctx;
+retry:
+ drm_for_each_plane(plane, dev) {
+ struct drm_plane_state *plane_state;
+
+ plane->old_fb = plane->fb;
+
+ plane_state = drm_atomic_get_plane_state(state, plane);
+ if (IS_ERR(plane_state)) {
+ ret = PTR_ERR(plane_state);
+ goto fail;
+ }
+
+ plane_state->rotation = BIT(DRM_ROTATE_0);
+
+ /* disable non-primary: */
+ if (plane->type == DRM_PLANE_TYPE_PRIMARY)
+ continue;
+
+ ret = __drm_atomic_helper_disable_plane(plane, plane_state);
+ if (ret != 0)
+ goto fail;
+ }
+
+ for(i = 0; i < fb_helper->crtc_count; i++) {
+ struct drm_mode_set *mode_set = &fb_helper->crtc_info[i].mode_set;
+
+ ret = __drm_atomic_helper_set_config(mode_set, state);
+ if (ret != 0)
+ goto fail;
+ }
+
+ ret = drm_atomic_commit(state);
+
+fail:
+ drm_for_each_plane(plane, dev) {
+ if (ret == 0) {
+ struct drm_framebuffer *new_fb = plane->state->fb;
+ if (new_fb)
+ drm_framebuffer_reference(new_fb);
+ plane->fb = new_fb;
+ plane->crtc = plane->state->crtc;
+
+ if (plane->old_fb)
+ drm_framebuffer_unreference(plane->old_fb);
+ }
+ plane->old_fb = NULL;
+ }
+
+ if (ret == -EDEADLK)
+ goto backoff;
+
+ if (ret != 0)
+ drm_atomic_state_free(state);
+
+ return ret;
+
+backoff:
+ drm_atomic_state_clear(state);
+ drm_atomic_legacy_backoff(state);
+
+ goto retry;
+}
+
+static int restore_fbdev_mode(struct drm_fb_helper *fb_helper)
{
struct drm_device *dev = fb_helper->dev;
struct drm_plane *plane;
- bool error = false;
int i;
drm_warn_on_modeset_not_all_locked(dev);
+ if (fb_helper->atomic)
+ return restore_fbdev_mode_atomic(fb_helper);
+
drm_for_each_plane(plane, dev) {
if (plane->type != DRM_PLANE_TYPE_PRIMARY)
drm_plane_force_disable(plane);
struct drm_crtc *crtc = mode_set->crtc;
int ret;
- if (crtc->funcs->cursor_set) {
+ if (crtc->funcs->cursor_set2) {
+ ret = crtc->funcs->cursor_set2(crtc, NULL, 0, 0, 0, 0, 0);
+ if (ret)
+ return ret;
+ } else if (crtc->funcs->cursor_set) {
ret = crtc->funcs->cursor_set(crtc, NULL, 0, 0, 0);
if (ret)
- error = true;
+ return ret;
}
ret = drm_mode_set_config_internal(mode_set);
if (ret)
- error = true;
+ return ret;
}
- return error;
+
+ return 0;
}
/**
* This should be called from driver's drm ->lastclose callback
* when implementing an fbcon on top of kms using this helper. This ensures that
* the user isn't greeted with a black screen when e.g. X dies.
+ *
+ * RETURNS:
+ * Zero if everything went ok, negative error code otherwise.
*/
-bool drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper)
+int drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper)
{
struct drm_device *dev = fb_helper->dev;
- bool ret;
- bool do_delayed = false;
+ bool do_delayed;
+ int ret;
+
+ if (!drm_fbdev_emulation)
+ return -ENODEV;
drm_modeset_lock_all(dev);
ret = restore_fbdev_mode(fb_helper);
struct drm_crtc *crtc;
int i;
+ if (!drm_fbdev_emulation)
+ return 0;
+
if (!max_conn_count)
return -EINVAL;
i++;
}
+ fb_helper->atomic = !!drm_core_check_feature(dev, DRIVER_ATOMIC);
+
return 0;
out_free:
drm_fb_helper_crtc_free(fb_helper);
void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
{
+ if (!drm_fbdev_emulation)
+ return;
+
if (!list_empty(&fb_helper->kernel_fb_list)) {
list_del(&fb_helper->kernel_fb_list);
if (list_empty(&kernel_fb_helper_list)) {
}
EXPORT_SYMBOL(drm_fb_helper_set_par);
+static int pan_display_atomic(struct fb_var_screeninfo *var,
+ struct fb_info *info)
+{
+ struct drm_fb_helper *fb_helper = info->par;
+ struct drm_device *dev = fb_helper->dev;
+ struct drm_atomic_state *state;
+ int i, ret;
+
+ state = drm_atomic_state_alloc(dev);
+ if (!state)
+ return -ENOMEM;
+
+ state->acquire_ctx = dev->mode_config.acquire_ctx;
+retry:
+ for(i = 0; i < fb_helper->crtc_count; i++) {
+ struct drm_mode_set *mode_set;
+
+ mode_set = &fb_helper->crtc_info[i].mode_set;
+
+ mode_set->crtc->primary->old_fb = mode_set->crtc->primary->fb;
+
+ mode_set->x = var->xoffset;
+ mode_set->y = var->yoffset;
+
+ ret = __drm_atomic_helper_set_config(mode_set, state);
+ if (ret != 0)
+ goto fail;
+ }
+
+ ret = drm_atomic_commit(state);
+ if (ret != 0)
+ goto fail;
+
+ info->var.xoffset = var->xoffset;
+ info->var.yoffset = var->yoffset;
+
+
+fail:
+ for(i = 0; i < fb_helper->crtc_count; i++) {
+ struct drm_mode_set *mode_set;
+ struct drm_plane *plane;
+
+ mode_set = &fb_helper->crtc_info[i].mode_set;
+ plane = mode_set->crtc->primary;
+
+ if (ret == 0) {
+ struct drm_framebuffer *new_fb = plane->state->fb;
+
+ if (new_fb)
+ drm_framebuffer_reference(new_fb);
+ plane->fb = new_fb;
+ plane->crtc = plane->state->crtc;
+
+ if (plane->old_fb)
+ drm_framebuffer_unreference(plane->old_fb);
+ }
+ plane->old_fb = NULL;
+ }
+
+ if (ret == -EDEADLK)
+ goto backoff;
+
+ if (ret != 0)
+ drm_atomic_state_free(state);
+
+ return ret;
+
+backoff:
+ drm_atomic_state_clear(state);
+ drm_atomic_legacy_backoff(state);
+
+ goto retry;
+}
+
/**
* drm_fb_helper_pan_display - implementation for ->fb_pan_display
* @var: updated screen information
return -EBUSY;
}
+ if (fb_helper->atomic) {
+ ret = pan_display_atomic(var, info);
+ goto unlock;
+ }
+
for (i = 0; i < fb_helper->crtc_count; i++) {
modeset = &fb_helper->crtc_info[i].mode_set;
}
}
}
+unlock:
drm_modeset_unlock_all(dev);
return ret;
}
struct drm_device *dev = fb_helper->dev;
int count = 0;
+ if (!drm_fbdev_emulation)
+ return 0;
+
mutex_lock(&dev->mode_config.mutex);
count = drm_fb_helper_probe_connector_modes(fb_helper,
dev->mode_config.max_width,
struct drm_device *dev = fb_helper->dev;
u32 max_width, max_height;
+ if (!drm_fbdev_emulation)
+ return 0;
+
mutex_lock(&fb_helper->dev->mode_config.mutex);
if (!fb_helper->fb || !drm_fb_helper_is_bound(fb_helper)) {
fb_helper->delayed_hotplug = true;
void
drm_gem_object_free(struct kref *kref)
{
- struct drm_gem_object *obj = (struct drm_gem_object *) kref;
+ struct drm_gem_object *obj =
+ container_of(kref, struct drm_gem_object, refcount);
struct drm_device *dev = obj->dev;
WARN_ON(!mutex_is_locked(&dev->struct_mutex));
* drm_gem_mmap() prevents unprivileged users from mapping random objects. So
* callers must verify access restrictions before calling this helper.
*
- * NOTE: This function has to be protected with dev->struct_mutex
- *
* Return 0 or success or -EINVAL if the object size is smaller than the VMA
* size, or if no gem_vm_ops are provided.
*/
{
struct drm_device *dev = obj->dev;
- lockdep_assert_held(&dev->struct_mutex);
-
/* Check for valid size. */
if (obj_size < vma->vm_end - vma->vm_start)
return -EINVAL;
{
struct drm_file *priv = filp->private_data;
struct drm_device *dev = priv->minor->dev;
- struct drm_gem_object *obj;
+ struct drm_gem_object *obj = NULL;
struct drm_vma_offset_node *node;
int ret;
if (drm_device_is_unplugged(dev))
return -ENODEV;
- mutex_lock(&dev->struct_mutex);
+ drm_vma_offset_lock_lookup(dev->vma_offset_manager);
+ node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
+ vma->vm_pgoff,
+ vma_pages(vma));
+ if (likely(node)) {
+ obj = container_of(node, struct drm_gem_object, vma_node);
+ /*
+ * When the object is being freed, after it hits 0-refcnt it
+ * proceeds to tear down the object. In the process it will
+ * attempt to remove the VMA offset and so acquire this
+ * mgr->vm_lock. Therefore if we find an object with a 0-refcnt
+ * that matches our range, we know it is in the process of being
+ * destroyed and will be freed as soon as we release the lock -
+ * so we have to check for the 0-refcnted object and treat it as
+ * invalid.
+ */
+ if (!kref_get_unless_zero(&obj->refcount))
+ obj = NULL;
+ }
+ drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
- node = drm_vma_offset_exact_lookup(dev->vma_offset_manager,
- vma->vm_pgoff,
- vma_pages(vma));
- if (!node) {
- mutex_unlock(&dev->struct_mutex);
+ if (!obj)
return -EINVAL;
- } else if (!drm_vma_node_is_allowed(node, filp)) {
- mutex_unlock(&dev->struct_mutex);
+
+ if (!drm_vma_node_is_allowed(node, filp)) {
+ drm_gem_object_unreference_unlocked(obj);
return -EACCES;
}
- obj = container_of(node, struct drm_gem_object, vma_node);
- ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT, vma);
+ ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
+ vma);
- mutex_unlock(&dev->struct_mutex);
+ drm_gem_object_unreference_unlocked(obj);
return ret;
}
struct drm_device *dev = obj->dev;
int ret;
- mutex_lock(&dev->struct_mutex);
ret = drm_gem_mmap_obj(obj, obj->size, vma);
- mutex_unlock(&dev->struct_mutex);
if (ret < 0)
return ret;
/* drm_sysfs.c */
extern struct class *drm_class;
-struct class *drm_sysfs_create(struct module *owner, char *name);
+int drm_sysfs_init(void);
void drm_sysfs_destroy(void);
struct device *drm_sysfs_minor_alloc(struct drm_minor *minor);
int drm_sysfs_connector_add(struct drm_connector *connector);
return 0;
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
typedef struct drm_agp_mode32 {
u32 mode; /**< AGP mode */
} drm_agp_mode32_t;
return drm_ioctl(file, DRM_IOCTL_AGP_UNBIND, (unsigned long)request);
}
-#endif /* __OS_HAS_AGP */
+#endif /* CONFIG_AGP */
typedef struct drm_scatter_gather32 {
u32 size; /**< In bytes -- will round to page boundary */
[DRM_IOCTL_NR(DRM_IOCTL_GET_SAREA_CTX32)] = compat_drm_getsareactx,
[DRM_IOCTL_NR(DRM_IOCTL_RES_CTX32)] = compat_drm_resctx,
[DRM_IOCTL_NR(DRM_IOCTL_DMA32)] = compat_drm_dma,
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
[DRM_IOCTL_NR(DRM_IOCTL_AGP_ENABLE32)] = compat_drm_agp_enable,
[DRM_IOCTL_NR(DRM_IOCTL_AGP_INFO32)] = compat_drm_agp_info,
[DRM_IOCTL_NR(DRM_IOCTL_AGP_ALLOC32)] = compat_drm_agp_alloc,
static int drm_version(struct drm_device *dev, void *data,
struct drm_file *file_priv);
-/**
+/*
* Get the bus id.
*
* \param inode device inode.
master->unique_len = 0;
}
-/**
+/*
* Set the bus id.
*
* \param inode device inode.
return 0;
}
-/**
+/*
* Get a mapping information.
*
* \param inode device inode.
return 0;
}
-/**
+/*
* Get client information.
*
* \param inode device inode.
}
}
-/**
+/*
* Get statistics information.
*
* \param inode device inode.
return 0;
}
-/**
+/*
* Get device/driver capabilities
*/
static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_priv)
return 0;
}
-/**
+/*
* Set device/driver capabilities
*/
static int
return 0;
}
-/**
+/*
* Setversion ioctl.
*
* \param inode device inode.
return retcode;
}
-/** No-op ioctl. */
+/**
+ * drm_noop - DRM no-op ioctl implemntation
+ * @dev: DRM device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: DRM file for the ioctl call
+ *
+ * This no-op implementation for drm ioctls is useful for deprecated
+ * functionality where we can't return a failure code because existing userspace
+ * checks the result of the ioctl, but doesn't care about the action.
+ *
+ * Always returns successfully with 0.
+ */
int drm_noop(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
EXPORT_SYMBOL(drm_noop);
/**
+ * drm_invalid_op - DRM invalid ioctl implemntation
+ * @dev: DRM device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: DRM file for the ioctl call
+ *
+ * This no-op implementation for drm ioctls is useful for deprecated
+ * functionality where we really don't want to allow userspace to call the ioctl
+ * any more. This is the case for old ums interfaces for drivers that
+ * transitioned to kms gradually and so kept the old legacy tables around. This
+ * only applies to radeon and i915 kms drivers, other drivers shouldn't need to
+ * use this function.
+ *
+ * Always fails with a return value of -EINVAL.
+ */
+int drm_invalid_op(struct drm_device *dev, void *data,
+ struct drm_file *file_priv)
+{
+ return -EINVAL;
+}
+EXPORT_SYMBOL(drm_invalid_op);
+
+/*
* Copy and IOCTL return string to user space
*/
static int drm_copy_field(char __user *buf, size_t *buf_len, const char *value)
return 0;
}
-/**
+/*
* Get version information
*
* \param inode device inode.
return err;
}
-/**
+/*
* drm_ioctl_permit - Check ioctl permissions against caller
*
* @flags: ioctl permission flags.
.name = #ioctl \
}
-/** Ioctl table */
+/* Ioctl table */
static const struct drm_ioctl_desc drm_ioctls[] = {
- DRM_IOCTL_DEF(DRM_IOCTL_VERSION, drm_version, DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF(DRM_IOCTL_VERSION, drm_version,
+ DRM_UNLOCKED|DRM_RENDER_ALLOW|DRM_CONTROL_ALLOW),
DRM_IOCTL_DEF(DRM_IOCTL_GET_UNIQUE, drm_getunique, 0),
DRM_IOCTL_DEF(DRM_IOCTL_GET_MAGIC, drm_getmagic, 0),
DRM_IOCTL_DEF(DRM_IOCTL_IRQ_BUSID, drm_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY),
DRM_IOCTL_DEF(DRM_IOCTL_CONTROL, drm_control, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
DRM_IOCTL_DEF(DRM_IOCTL_AGP_ACQUIRE, drm_agp_acquire_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
DRM_IOCTL_DEF(DRM_IOCTL_AGP_RELEASE, drm_agp_release_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
DRM_IOCTL_DEF(DRM_IOCTL_AGP_ENABLE, drm_agp_enable_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
#define DRM_CORE_IOCTL_COUNT ARRAY_SIZE( drm_ioctls )
/**
- * Called whenever a process performs an ioctl on /dev/drm.
- *
- * \param inode device inode.
- * \param file_priv DRM file private.
- * \param cmd command.
- * \param arg user argument.
- * \return zero on success or negative number on failure.
+ * drm_ioctl - ioctl callback implementation for DRM drivers
+ * @filp: file this ioctl is called on
+ * @cmd: ioctl cmd number
+ * @arg: user argument
*
* Looks up the ioctl function in the ::ioctls table, checking for root
* previleges if so required, and dispatches to the respective function.
+ *
+ * Returns:
+ * Zero on success, negative error code on failure.
*/
long drm_ioctl(struct file *filp,
unsigned int cmd, unsigned long arg)
char stack_kdata[128];
char *kdata = NULL;
unsigned int usize, asize, drv_size;
+ bool is_driver_ioctl;
dev = file_priv->minor->dev;
if (drm_device_is_unplugged(dev))
return -ENODEV;
- if (nr >= DRM_COMMAND_BASE && nr < DRM_COMMAND_END) {
+ is_driver_ioctl = nr >= DRM_COMMAND_BASE && nr < DRM_COMMAND_END;
+
+ if (is_driver_ioctl) {
/* driver ioctl */
if (nr - DRM_COMMAND_BASE >= dev->driver->num_ioctls)
goto err_i1;
memset(kdata, 0, usize);
}
- if (ioctl->flags & DRM_UNLOCKED)
+ /* Enforce sane locking for kms driver ioctls. Core ioctls are
+ * too messy still. */
+ if ((drm_core_check_feature(dev, DRIVER_MODESET) && is_driver_ioctl) ||
+ (ioctl->flags & DRM_UNLOCKED))
retcode = func(dev, kdata, file_priv);
else {
mutex_lock(&drm_global_mutex);
/**
* drm_ioctl_flags - Check for core ioctl and return ioctl permission flags
+ * @nr: ioctl number
+ * @flags: where to return the ioctl permission flags
+ *
+ * This ioctl is only used by the vmwgfx driver to augment the access checks
+ * done by the drm core and insofar a pretty decent layering violation. This
+ * shouldn't be used by any drivers.
*
- * @nr: Ioctl number.
- * @flags: Where to return the ioctl permission flags
+ * Returns:
+ * True if the @nr corresponds to a DRM core ioctl numer, false otherwise.
*/
bool drm_ioctl_flags(unsigned int nr, unsigned int *flags)
{
module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
module_param_named(timestamp_monotonic, drm_timestamp_monotonic, int, 0600);
-static void store_vblank(struct drm_device *dev, int crtc,
+static void store_vblank(struct drm_device *dev, unsigned int pipe,
u32 vblank_count_inc,
- struct timeval *t_vblank)
+ struct timeval *t_vblank, u32 last)
{
- struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+ struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
u32 tslot;
assert_spin_locked(&dev->vblank_time_lock);
- if (t_vblank) {
- /* All writers hold the spinlock, but readers are serialized by
- * the latching of vblank->count below.
- */
- tslot = vblank->count + vblank_count_inc;
- vblanktimestamp(dev, crtc, tslot) = *t_vblank;
- }
+ vblank->last = last;
+
+ /* All writers hold the spinlock, but readers are serialized by
+ * the latching of vblank->count below.
+ */
+ tslot = vblank->count + vblank_count_inc;
+ vblanktimestamp(dev, pipe, tslot) = *t_vblank;
/*
* vblank timestamp updates are protected on the write side with
smp_wmb();
}
+/**
+ * drm_reset_vblank_timestamp - reset the last timestamp to the last vblank
+ * @dev: DRM device
+ * @pipe: index of CRTC for which to reset the timestamp
+ *
+ * Reset the stored timestamp for the current vblank count to correspond
+ * to the last vblank occurred.
+ *
+ * Only to be called from drm_vblank_on().
+ *
+ * Note: caller must hold dev->vbl_lock since this reads & writes
+ * device vblank fields.
+ */
+static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
+{
+ u32 cur_vblank;
+ bool rc;
+ struct timeval t_vblank;
+ int count = DRM_TIMESTAMP_MAXRETRIES;
+
+ spin_lock(&dev->vblank_time_lock);
+
+ /*
+ * sample the current counter to avoid random jumps
+ * when drm_vblank_enable() applies the diff
+ */
+ do {
+ cur_vblank = dev->driver->get_vblank_counter(dev, pipe);
+ rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, 0);
+ } while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe) && --count > 0);
+
+ /*
+ * Only reinitialize corresponding vblank timestamp if high-precision query
+ * available and didn't fail. Otherwise reinitialize delayed at next vblank
+ * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
+ */
+ if (!rc)
+ t_vblank = (struct timeval) {0, 0};
+
+ /*
+ * +1 to make sure user will never see the same
+ * vblank counter value before and after a modeset
+ */
+ store_vblank(dev, pipe, 1, &t_vblank, cur_vblank);
+
+ spin_unlock(&dev->vblank_time_lock);
+}
+
/**
* drm_update_vblank_count - update the master vblank counter
* @dev: DRM device
* @pipe: counter to update
*
* Call back into the driver to update the appropriate vblank counter
- * (specified by @crtc). Deal with wraparound, if it occurred, and
+ * (specified by @pipe). Deal with wraparound, if it occurred, and
* update the last read value so we can deal with wraparound on the next
* call if necessary.
*
* Note: caller must hold dev->vbl_lock since this reads & writes
* device vblank fields.
*/
-static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe)
+static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
+ unsigned long flags)
{
struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
u32 cur_vblank, diff;
bool rc;
struct timeval t_vblank;
+ int count = DRM_TIMESTAMP_MAXRETRIES;
+ int framedur_ns = vblank->framedur_ns;
/*
* Interrupts were disabled prior to this call, so deal with counter
*/
do {
cur_vblank = dev->driver->get_vblank_counter(dev, pipe);
- rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, 0);
- } while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe));
+ rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, flags);
+ } while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe) && --count > 0);
+
+ if (dev->max_vblank_count != 0) {
+ /* trust the hw counter when it's around */
+ diff = (cur_vblank - vblank->last) & dev->max_vblank_count;
+ } else if (rc && framedur_ns) {
+ const struct timeval *t_old;
+ u64 diff_ns;
- /* Deal with counter wrap */
- diff = cur_vblank - vblank->last;
- if (cur_vblank < vblank->last) {
- diff += dev->max_vblank_count + 1;
+ t_old = &vblanktimestamp(dev, pipe, vblank->count);
+ diff_ns = timeval_to_ns(&t_vblank) - timeval_to_ns(t_old);
+
+ /*
+ * Figure out how many vblanks we've missed based
+ * on the difference in the timestamps and the
+ * frame/field duration.
+ */
+ diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
- DRM_DEBUG("last_vblank[%u]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
- pipe, vblank->last, cur_vblank, diff);
+ if (diff == 0 && flags & DRM_CALLED_FROM_VBLIRQ)
+ DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored."
+ " diff_ns = %lld, framedur_ns = %d)\n",
+ pipe, (long long) diff_ns, framedur_ns);
+ } else {
+ /* some kind of default for drivers w/o accurate vbl timestamping */
+ diff = (flags & DRM_CALLED_FROM_VBLIRQ) != 0;
}
- DRM_DEBUG("updating vblank count on crtc %u, missed %d\n",
- pipe, diff);
+ DRM_DEBUG_VBL("updating vblank count on crtc %u:"
+ " current=%u, diff=%u, hw=%u hw_last=%u\n",
+ pipe, vblank->count, diff, cur_vblank, vblank->last);
- if (diff == 0)
+ if (diff == 0) {
+ WARN_ON_ONCE(cur_vblank != vblank->last);
return;
+ }
/*
* Only reinitialize corresponding vblank timestamp if high-precision query
- * available and didn't fail. Otherwise reinitialize delayed at next vblank
- * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
+ * available and didn't fail, or we were called from the vblank interrupt.
+ * Otherwise reinitialize delayed at next vblank interrupt and assign 0
+ * for now, to mark the vblanktimestamp as invalid.
*/
- if (!rc)
+ if (!rc && (flags & DRM_CALLED_FROM_VBLIRQ) == 0)
t_vblank = (struct timeval) {0, 0};
- store_vblank(dev, pipe, diff, &t_vblank);
+ store_vblank(dev, pipe, diff, &t_vblank, cur_vblank);
}
/*
{
struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
unsigned long irqflags;
- u32 vblcount;
- s64 diff_ns;
- bool vblrc;
- struct timeval tvblank;
- int count = DRM_TIMESTAMP_MAXRETRIES;
/* Prevent vblank irq processing while disabling vblank irqs,
* so no updates of timestamps or count can happen after we've
*/
spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
- /*
- * If the vblank interrupt was already disabled update the count
- * and timestamp to maintain the appearance that the counter
- * has been ticking all along until this time. This makes the
- * count account for the entire time between drm_vblank_on() and
- * drm_vblank_off().
- *
- * But only do this if precise vblank timestamps are available.
- * Otherwise we might read a totally bogus timestamp since drivers
- * lacking precise timestamp support rely upon sampling the system clock
- * at vblank interrupt time. Which obviously won't work out well if the
- * vblank interrupt is disabled.
- */
- if (!vblank->enabled &&
- drm_get_last_vbltimestamp(dev, pipe, &tvblank, 0)) {
- drm_update_vblank_count(dev, pipe);
- spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
- return;
- }
-
/*
* Only disable vblank interrupts if they're enabled. This avoids
* calling the ->disable_vblank() operation in atomic context with the
vblank->enabled = false;
}
- /* No further vblank irq's will be processed after
- * this point. Get current hardware vblank count and
- * vblank timestamp, repeat until they are consistent.
- *
- * FIXME: There is still a race condition here and in
- * drm_update_vblank_count() which can cause off-by-one
- * reinitialization of software vblank counter. If gpu
- * vblank counter doesn't increment exactly at the leading
- * edge of a vblank interval, then we can lose 1 count if
- * we happen to execute between start of vblank and the
- * delayed gpu counter increment.
- */
- do {
- vblank->last = dev->driver->get_vblank_counter(dev, pipe);
- vblrc = drm_get_last_vbltimestamp(dev, pipe, &tvblank, 0);
- } while (vblank->last != dev->driver->get_vblank_counter(dev, pipe) && (--count) && vblrc);
-
- if (!count)
- vblrc = 0;
-
- /* Compute time difference to stored timestamp of last vblank
- * as updated by last invocation of drm_handle_vblank() in vblank irq.
- */
- vblcount = vblank->count;
- diff_ns = timeval_to_ns(&tvblank) -
- timeval_to_ns(&vblanktimestamp(dev, pipe, vblcount));
-
- /* If there is at least 1 msec difference between the last stored
- * timestamp and tvblank, then we are currently executing our
- * disable inside a new vblank interval, the tvblank timestamp
- * corresponds to this new vblank interval and the irq handler
- * for this vblank didn't run yet and won't run due to our disable.
- * Therefore we need to do the job of drm_handle_vblank() and
- * increment the vblank counter by one to account for this vblank.
- *
- * Skip this step if there isn't any high precision timestamp
- * available. In that case we can't account for this and just
- * hope for the best.
+ /*
+ * Always update the count and timestamp to maintain the
+ * appearance that the counter has been ticking all along until
+ * this time. This makes the count account for the entire time
+ * between drm_vblank_on() and drm_vblank_off().
*/
- if (vblrc && (abs64(diff_ns) > 1000000))
- store_vblank(dev, pipe, 1, &tvblank);
+ drm_update_vblank_count(dev, pipe, 0);
spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
}
void drm_calc_timestamping_constants(struct drm_crtc *crtc,
const struct drm_display_mode *mode)
{
- int linedur_ns = 0, pixeldur_ns = 0, framedur_ns = 0;
+ struct drm_vblank_crtc *vblank = &crtc->dev->vblank[drm_crtc_index(crtc)];
+ int linedur_ns = 0, framedur_ns = 0;
int dotclock = mode->crtc_clock;
/* Valid dotclock? */
/*
* Convert scanline length in pixels and video
- * dot clock to line duration, frame duration
- * and pixel duration in nanoseconds:
+ * dot clock to line duration and frame duration
+ * in nanoseconds:
*/
- pixeldur_ns = 1000000 / dotclock;
linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
crtc->base.id);
- crtc->pixeldur_ns = pixeldur_ns;
- crtc->linedur_ns = linedur_ns;
- crtc->framedur_ns = framedur_ns;
+ vblank->linedur_ns = linedur_ns;
+ vblank->framedur_ns = framedur_ns;
DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
crtc->base.id, mode->crtc_htotal,
mode->crtc_vtotal, mode->crtc_vdisplay);
- DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d, pixeldur %d\n",
- crtc->base.id, dotclock, framedur_ns,
- linedur_ns, pixeldur_ns);
+ DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
+ crtc->base.id, dotclock, framedur_ns, linedur_ns);
}
EXPORT_SYMBOL(drm_calc_timestamping_constants);
* @flags: Flags to pass to driver:
* 0 = Default,
* DRM_CALLED_FROM_VBLIRQ = If function is called from vbl IRQ handler
- * @refcrtc: CRTC which defines scanout timing
* @mode: mode which defines the scanout timings
*
* Implements calculation of exact vblank timestamps from given drm_display_mode
int *max_error,
struct timeval *vblank_time,
unsigned flags,
- const struct drm_crtc *refcrtc,
const struct drm_display_mode *mode)
{
struct timeval tv_etime;
ktime_t stime, etime;
- int vbl_status;
+ unsigned int vbl_status;
+ int ret = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
int vpos, hpos, i;
- int framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns;
- bool invbl;
+ int delta_ns, duration_ns;
if (pipe >= dev->num_crtcs) {
DRM_ERROR("Invalid crtc %u\n", pipe);
return -EIO;
}
- /* Durations of frames, lines, pixels in nanoseconds. */
- framedur_ns = refcrtc->framedur_ns;
- linedur_ns = refcrtc->linedur_ns;
- pixeldur_ns = refcrtc->pixeldur_ns;
-
/* If mode timing undefined, just return as no-op:
* Happens during initial modesetting of a crtc.
*/
- if (framedur_ns == 0) {
+ if (mode->crtc_clock == 0) {
DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
return -EAGAIN;
}
* Get vertical and horizontal scanout position vpos, hpos,
* and bounding timestamps stime, etime, pre/post query.
*/
- vbl_status = dev->driver->get_scanout_position(dev, pipe, flags, &vpos,
- &hpos, &stime, &etime);
+ vbl_status = dev->driver->get_scanout_position(dev, pipe, flags,
+ &vpos, &hpos,
+ &stime, &etime,
+ mode);
/* Return as no-op if scanout query unsupported or failed. */
if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
- DRM_DEBUG("crtc %u : scanoutpos query failed [%d].\n",
+ DRM_DEBUG("crtc %u : scanoutpos query failed [0x%x].\n",
pipe, vbl_status);
return -EIO;
}
* within vblank area, counting down the number of lines until
* start of scanout.
*/
- invbl = vbl_status & DRM_SCANOUTPOS_IN_VBLANK;
+ if (vbl_status & DRM_SCANOUTPOS_IN_VBLANK)
+ ret |= DRM_VBLANKTIME_IN_VBLANK;
/* Convert scanout position into elapsed time at raw_time query
* since start of scanout at first display scanline. delta_ns
* can be negative if start of scanout hasn't happened yet.
*/
- delta_ns = vpos * linedur_ns + hpos * pixeldur_ns;
+ delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
+ mode->crtc_clock);
if (!drm_timestamp_monotonic)
etime = ktime_mono_to_real(etime);
etime = ktime_sub_ns(etime, delta_ns);
*vblank_time = ktime_to_timeval(etime);
- DRM_DEBUG("crtc %u : v %d p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n",
- pipe, (int)vbl_status, hpos, vpos,
- (long)tv_etime.tv_sec, (long)tv_etime.tv_usec,
- (long)vblank_time->tv_sec, (long)vblank_time->tv_usec,
- duration_ns/1000, i);
+ DRM_DEBUG_VBL("crtc %u : v 0x%x p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n",
+ pipe, vbl_status, hpos, vpos,
+ (long)tv_etime.tv_sec, (long)tv_etime.tv_usec,
+ (long)vblank_time->tv_sec, (long)vblank_time->tv_usec,
+ duration_ns/1000, i);
- vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
- if (invbl)
- vbl_status |= DRM_VBLANKTIME_IN_VBLANK;
-
- return vbl_status;
+ return ret;
}
EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
* Returns:
* The software vblank counter.
*/
-u32 drm_vblank_count(struct drm_device *dev, int pipe)
+u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
{
struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
* vblank events since the system was booted, including lost events due to
* modesetting activity. Returns corresponding system timestamp of the time
* of the vblank interval that corresponds to the current vblank counter value.
+ *
+ * This is the legacy version of drm_crtc_vblank_count_and_time().
*/
u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
struct timeval *vblanktime)
{
struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+ int count = DRM_TIMESTAMP_MAXRETRIES;
u32 cur_vblank;
if (WARN_ON(pipe >= dev->num_crtcs))
smp_rmb();
*vblanktime = vblanktimestamp(dev, pipe, cur_vblank);
smp_rmb();
- } while (cur_vblank != vblank->count);
+ } while (cur_vblank != vblank->count && --count > 0);
return cur_vblank;
}
EXPORT_SYMBOL(drm_vblank_count_and_time);
+/**
+ * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
+ * and the system timestamp corresponding to that vblank counter value
+ * @crtc: which counter to retrieve
+ * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
+ *
+ * Fetches the "cooked" vblank count value that represents the number of
+ * vblank events since the system was booted, including lost events due to
+ * modesetting activity. Returns corresponding system timestamp of the time
+ * of the vblank interval that corresponds to the current vblank counter value.
+ *
+ * This is the native KMS version of drm_vblank_count_and_time().
+ */
+u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
+ struct timeval *vblanktime)
+{
+ return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
+ vblanktime);
+}
+EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
+
static void send_vblank_event(struct drm_device *dev,
struct drm_pending_vblank_event *e,
unsigned long seq, struct timeval *now)
atomic_dec(&vblank->refcount);
else {
vblank->enabled = true;
- drm_update_vblank_count(dev, pipe);
+ drm_update_vblank_count(dev, pipe, 0);
}
}
* @dev: DRM device
* @pipe: CRTC index
*
- * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
- * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
+ * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
+ * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
* due to lack of driver support or because the crtc is off.
*/
void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
if (e->pipe != pipe)
continue;
- DRM_DEBUG("Sending premature vblank event on disable: \
- wanted %d, current %d\n",
+ DRM_DEBUG("Sending premature vblank event on disable: "
+ "wanted %d, current %d\n",
e->event.sequence, seq);
list_del(&e->base.link);
drm_vblank_put(dev, pipe);
/**
* drm_crtc_vblank_reset - reset vblank state to off on a CRTC
- * @drm_crtc: CRTC in question
+ * @crtc: CRTC in question
*
* Drivers can use this function to reset the vblank state to off at load time.
* Drivers should use this together with the drm_crtc_vblank_off() and
* drm_crtc_vblank_off() is that this function doesn't save the vblank counter
* and hence doesn't need to call any driver hooks.
*/
-void drm_crtc_vblank_reset(struct drm_crtc *drm_crtc)
+void drm_crtc_vblank_reset(struct drm_crtc *crtc)
{
- struct drm_device *dev = drm_crtc->dev;
+ struct drm_device *dev = crtc->dev;
unsigned long irqflags;
- int crtc = drm_crtc_index(drm_crtc);
- struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+ unsigned int pipe = drm_crtc_index(crtc);
+ struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
spin_lock_irqsave(&dev->vbl_lock, irqflags);
/*
vblank->inmodeset = 0;
}
- /*
- * sample the current counter to avoid random jumps
- * when drm_vblank_enable() applies the diff
- *
- * -1 to make sure user will never see the same
- * vblank counter value before and after a modeset
- */
- vblank->last =
- (dev->driver->get_vblank_counter(dev, pipe) - 1) &
- dev->max_vblank_count;
+ drm_reset_vblank_timestamp(dev, pipe);
+
/*
* re-enable interrupts if there are users left, or the
* user wishes vblank interrupts to be enabled all the time.
bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
- u32 vblcount;
- s64 diff_ns;
- struct timeval tvblank;
unsigned long irqflags;
if (WARN_ON_ONCE(!dev->num_crtcs))
return false;
}
- /* Fetch corresponding timestamp for this vblank interval from
- * driver and store it in proper slot of timestamp ringbuffer.
- */
-
- /* Get current timestamp and count. */
- vblcount = vblank->count;
- drm_get_last_vbltimestamp(dev, pipe, &tvblank, DRM_CALLED_FROM_VBLIRQ);
-
- /* Compute time difference to timestamp of last vblank */
- diff_ns = timeval_to_ns(&tvblank) -
- timeval_to_ns(&vblanktimestamp(dev, pipe, vblcount));
-
- /* Update vblank timestamp and count if at least
- * DRM_REDUNDANT_VBLIRQ_THRESH_NS nanoseconds
- * difference between last stored timestamp and current
- * timestamp. A smaller difference means basically
- * identical timestamps. Happens if this vblank has
- * been already processed and this is a redundant call,
- * e.g., due to spurious vblank interrupts. We need to
- * ignore those for accounting.
- */
- if (abs64(diff_ns) > DRM_REDUNDANT_VBLIRQ_THRESH_NS)
- store_vblank(dev, pipe, 1, &tvblank);
- else
- DRM_DEBUG("crtc %u: Redundant vblirq ignored. diff_ns = %d\n",
- pipe, (int) diff_ns);
+ drm_update_vblank_count(dev, pipe, DRM_CALLED_FROM_VBLIRQ);
spin_unlock(&dev->vblank_time_lock);
return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
}
EXPORT_SYMBOL(drm_crtc_handle_vblank);
+
+/**
+ * drm_vblank_no_hw_counter - "No hw counter" implementation of .get_vblank_counter()
+ * @dev: DRM device
+ * @pipe: CRTC for which to read the counter
+ *
+ * Drivers can plug this into the .get_vblank_counter() function if
+ * there is no useable hardware frame counter available.
+ *
+ * Returns:
+ * 0
+ */
+u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
+{
+ return 0;
+}
+EXPORT_SYMBOL(drm_vblank_no_hw_counter);
#include <drm/drmP.h>
#include "drm_legacy.h"
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
#ifdef HAVE_PAGE_AGP
# include <asm/agp.h>
return agp_unbind_memory(handle);
}
-#else /* __OS_HAS_AGP */
+#else /* CONFIG_AGP */
static inline void *agp_remap(unsigned long offset, unsigned long size,
struct drm_device * dev)
{
return NULL;
}
-#endif /* agp */
+#endif /* CONFIG_AGP */
void drm_legacy_ioremap(struct drm_local_map *map, struct drm_device *dev)
{
if (adj_end > end)
adj_end = end;
- if (flags & DRM_MM_CREATE_TOP)
- adj_start = adj_end - size;
-
if (mm->color_adjust)
mm->color_adjust(hole_node, color, &adj_start, &adj_end);
+ if (flags & DRM_MM_CREATE_TOP)
+ adj_start = adj_end - size;
+
if (alignment) {
u64 tmp = adj_start;
unsigned rem;
WARN_ON(ctx->contended);
if (ctx->trylock_only) {
+ lockdep_assert_held(&ctx->ww_ctx);
+
if (!ww_mutex_trylock(&lock->mutex))
return -EBUSY;
else
+#include <linux/component.h>
#include <linux/export.h>
#include <linux/list.h>
#include <linux/of_graph.h>
return possible_crtcs;
}
EXPORT_SYMBOL(drm_of_find_possible_crtcs);
+
+/**
+ * drm_of_component_probe - Generic probe function for a component based master
+ * @dev: master device containing the OF node
+ * @compare_of: compare function used for matching components
+ * @master_ops: component master ops to be used
+ *
+ * Parse the platform device OF node and bind all the components associated
+ * with the master. Interface ports are added before the encoders in order to
+ * satisfy their .bind requirements
+ * See Documentation/devicetree/bindings/graph.txt for the bindings.
+ *
+ * Returns zero if successful, or one of the standard error codes if it fails.
+ */
+int drm_of_component_probe(struct device *dev,
+ int (*compare_of)(struct device *, void *),
+ const struct component_master_ops *m_ops)
+{
+ struct device_node *ep, *port, *remote;
+ struct component_match *match = NULL;
+ int i;
+
+ if (!dev->of_node)
+ return -EINVAL;
+
+ /*
+ * Bind the crtc's ports first, so that drm_of_find_possible_crtcs()
+ * called from encoder's .bind callbacks works as expected
+ */
+ for (i = 0; ; i++) {
+ port = of_parse_phandle(dev->of_node, "ports", i);
+ if (!port)
+ break;
+
+ if (!of_device_is_available(port->parent)) {
+ of_node_put(port);
+ continue;
+ }
+
+ component_match_add(dev, &match, compare_of, port);
+ of_node_put(port);
+ }
+
+ if (i == 0) {
+ dev_err(dev, "missing 'ports' property\n");
+ return -ENODEV;
+ }
+
+ if (!match) {
+ dev_err(dev, "no available port\n");
+ return -ENODEV;
+ }
+
+ /*
+ * For bound crtcs, bind the encoders attached to their remote endpoint
+ */
+ for (i = 0; ; i++) {
+ port = of_parse_phandle(dev->of_node, "ports", i);
+ if (!port)
+ break;
+
+ if (!of_device_is_available(port->parent)) {
+ of_node_put(port);
+ continue;
+ }
+
+ for_each_child_of_node(port, ep) {
+ remote = of_graph_get_remote_port_parent(ep);
+ if (!remote || !of_device_is_available(remote)) {
+ of_node_put(remote);
+ continue;
+ } else if (!of_device_is_available(remote->parent)) {
+ dev_warn(dev, "parent device of %s is not available\n",
+ remote->full_name);
+ of_node_put(remote);
+ continue;
+ }
+
+ component_match_add(dev, &match, compare_of, remote);
+ of_node_put(remote);
+ }
+ of_node_put(port);
+ }
+
+ return component_master_add_with_match(dev, m_ops, match);
+}
+EXPORT_SYMBOL(drm_of_component_probe);
* then register the character device and inter module information.
* Try and register, if we fail to register, backout previous work.
*
+ * NOTE: This function is deprecated, please use drm_dev_alloc() and
+ * drm_dev_register() instead and remove your ->load() callback.
+ *
* Return: 0 on success or a negative error code on failure.
*/
int drm_get_pci_dev(struct pci_dev *pdev, const struct pci_device_id *ent,
* Initializes a drm_device structures, registering the stubs and initializing
* the AGP device.
*
+ * NOTE: This function is deprecated. Modern modesetting drm drivers should use
+ * pci_register_driver() directly, this function only provides shadow-binding
+ * support for old legacy drivers on top of that core pci function.
+ *
* Return: 0 on success or a negative error code on failure.
*/
int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
*
* Unregisters one or more devices matched by a PCI driver from the DRM
* subsystem.
+ *
+ * NOTE: This function is deprecated. Modern modesetting drm drivers should use
+ * pci_unregister_driver() directly, this function only provides shadow-binding
+ * support for old legacy drivers on top of that core pci function.
*/
void drm_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver)
{
if (plane_funcs->prepare_fb && plane_state->fb &&
plane_state->fb != old_fb) {
- ret = plane_funcs->prepare_fb(plane, plane_state->fb,
+ ret = plane_funcs->prepare_fb(plane,
plane_state);
if (ret)
goto out;
ret = 0;
}
- if (plane_funcs->cleanup_fb && old_fb)
- plane_funcs->cleanup_fb(plane, old_fb, plane_state);
+ if (plane_funcs->cleanup_fb)
+ plane_funcs->cleanup_fb(plane, plane_state);
out:
if (plane_state) {
if (plane->funcs->atomic_destroy_state)
* subsystem, initializing a drm_device structure and calling the driver's
* .load() function.
*
+ * NOTE: This function is deprecated, please use drm_dev_alloc() and
+ * drm_dev_register() instead and remove your ->load() callback.
+ *
* Return: 0 on success or a negative error code on failure.
*/
int drm_platform_init(struct drm_driver *driver, struct platform_device *platform_device)
}
#define DRM_OUTPUT_POLL_PERIOD (10*HZ)
-static void __drm_kms_helper_poll_enable(struct drm_device *dev)
+/**
+ * drm_kms_helper_poll_enable_locked - re-enable output polling.
+ * @dev: drm_device
+ *
+ * This function re-enables the output polling work without
+ * locking the mode_config mutex.
+ *
+ * This is like drm_kms_helper_poll_enable() however it is to be
+ * called from a context where the mode_config mutex is locked
+ * already.
+ */
+void drm_kms_helper_poll_enable_locked(struct drm_device *dev)
{
bool poll = false;
struct drm_connector *connector;
if (poll)
schedule_delayed_work(&dev->mode_config.output_poll_work, DRM_OUTPUT_POLL_PERIOD);
}
+EXPORT_SYMBOL(drm_kms_helper_poll_enable_locked);
+
static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connector *connector,
uint32_t maxX, uint32_t maxY, bool merge_type_bits)
/* Re-enable polling in case the global poll config changed. */
if (drm_kms_helper_poll != dev->mode_config.poll_running)
- __drm_kms_helper_poll_enable(dev);
+ drm_kms_helper_poll_enable_locked(dev);
dev->mode_config.poll_running = drm_kms_helper_poll;
void drm_kms_helper_poll_enable(struct drm_device *dev)
{
mutex_lock(&dev->mode_config.mutex);
- __drm_kms_helper_poll_enable(dev);
+ drm_kms_helper_poll_enable_locked(dev);
mutex_unlock(&dev->mode_config.mutex);
}
EXPORT_SYMBOL(drm_kms_helper_poll_enable);
}
}
- switch (rotation & 0xf) {
+ switch (rotation & DRM_ROTATE_MASK) {
case BIT(DRM_ROTATE_0):
break;
case BIT(DRM_ROTATE_90):
{
struct drm_rect tmp;
- switch (rotation & 0xf) {
+ switch (rotation & DRM_ROTATE_MASK) {
case BIT(DRM_ROTATE_0):
break;
case BIT(DRM_ROTATE_90):
.name = "drm_minor"
};
+struct class *drm_class;
+
/**
* __drm_class_suspend - internal DRM class suspend routine
* @dev: Linux device to suspend
CORE_DATE);
/**
- * drm_sysfs_create - create a struct drm_sysfs_class structure
- * @owner: pointer to the module that is to "own" this struct drm_sysfs_class
- * @name: pointer to a string for the name of this class.
+ * drm_sysfs_init - initialize sysfs helpers
+ *
+ * This is used to create the DRM class, which is the implicit parent of any
+ * other top-level DRM sysfs objects.
*
- * This is used to create DRM class pointer that can then be used
- * in calls to drm_sysfs_device_add().
+ * You must call drm_sysfs_destroy() to release the allocated resources.
*
- * Note, the pointer created here is to be destroyed when finished by making a
- * call to drm_sysfs_destroy().
+ * Return: 0 on success, negative error code on failure.
*/
-struct class *drm_sysfs_create(struct module *owner, char *name)
+int drm_sysfs_init(void)
{
- struct class *class;
int err;
- class = class_create(owner, name);
- if (IS_ERR(class)) {
- err = PTR_ERR(class);
- goto err_out;
- }
-
- class->pm = &drm_class_dev_pm_ops;
-
- err = class_create_file(class, &class_attr_version.attr);
- if (err)
- goto err_out_class;
+ drm_class = class_create(THIS_MODULE, "drm");
+ if (IS_ERR(drm_class))
+ return PTR_ERR(drm_class);
- class->devnode = drm_devnode;
+ drm_class->pm = &drm_class_dev_pm_ops;
- return class;
+ err = class_create_file(drm_class, &class_attr_version.attr);
+ if (err) {
+ class_destroy(drm_class);
+ drm_class = NULL;
+ return err;
+ }
-err_out_class:
- class_destroy(class);
-err_out:
- return ERR_PTR(err);
+ drm_class->devnode = drm_devnode;
+ return 0;
}
/**
*/
void drm_sysfs_destroy(void)
{
- if ((drm_class == NULL) || (IS_ERR(drm_class)))
+ if (IS_ERR_OR_NULL(drm_class))
return;
class_remove_file(drm_class, &class_attr_version.attr);
class_destroy(drm_class);
char *buf)
{
struct drm_connector *connector = to_drm_connector(device);
- struct drm_device *dev = connector->dev;
- uint64_t dpms_status;
- int ret;
+ int dpms;
- ret = drm_object_property_get_value(&connector->base,
- dev->mode_config.dpms_property,
- &dpms_status);
- if (ret)
- return 0;
+ dpms = READ_ONCE(connector->dpms);
return snprintf(buf, PAGE_SIZE, "%s\n",
- drm_get_dpms_name((int)dpms_status));
+ drm_get_dpms_name(dpms));
}
static ssize_t enabled_show(struct device *device,
* Find the right map and if it's AGP memory find the real physical page to
* map, get the page, increment the use count and return it.
*/
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
static int drm_do_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct drm_file *priv = vma->vm_file->private_data;
vm_fault_error:
return VM_FAULT_SIGBUS; /* Disallow mremap */
}
-#else /* __OS_HAS_AGP */
+#else
static int drm_do_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
return VM_FAULT_SIGBUS;
}
-#endif /* __OS_HAS_AGP */
+#endif
/**
* \c nopage method for shared virtual memory.
* --BenH.
*/
if (!vma->vm_pgoff
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
&& (!dev->agp
|| dev->agp->agp_info.device->vendor != PCI_VENDOR_ID_APPLE)
#endif
EXPORT_SYMBOL(drm_vma_offset_manager_destroy);
/**
- * drm_vma_offset_lookup() - Find node in offset space
+ * drm_vma_offset_lookup_locked() - Find node in offset space
* @mgr: Manager object
* @start: Start address for object (page-based)
* @pages: Size of object (page-based)
* region and the given node will be returned, as long as the node spans the
* whole requested area (given the size in number of pages as @pages).
*
- * RETURNS:
- * Returns NULL if no suitable node can be found. Otherwise, the best match
- * is returned. It's the caller's responsibility to make sure the node doesn't
- * get destroyed before the caller can access it.
- */
-struct drm_vma_offset_node *drm_vma_offset_lookup(struct drm_vma_offset_manager *mgr,
- unsigned long start,
- unsigned long pages)
-{
- struct drm_vma_offset_node *node;
-
- read_lock(&mgr->vm_lock);
- node = drm_vma_offset_lookup_locked(mgr, start, pages);
- read_unlock(&mgr->vm_lock);
-
- return node;
-}
-EXPORT_SYMBOL(drm_vma_offset_lookup);
-
-/**
- * drm_vma_offset_lookup_locked() - Find node in offset space
- * @mgr: Manager object
- * @start: Start address for object (page-based)
- * @pages: Size of object (page-based)
+ * Note that before lookup the vma offset manager lookup lock must be acquired
+ * with drm_vma_offset_lock_lookup(). See there for an example. This can then be
+ * used to implement weakly referenced lookups using kref_get_unless_zero().
*
- * Same as drm_vma_offset_lookup() but requires the caller to lock offset lookup
- * manually. See drm_vma_offset_lock_lookup() for an example.
+ * Example:
+ * drm_vma_offset_lock_lookup(mgr);
+ * node = drm_vma_offset_lookup_locked(mgr);
+ * if (node)
+ * kref_get_unless_zero(container_of(node, sth, entr));
+ * drm_vma_offset_unlock_lookup(mgr);
*
* RETURNS:
* Returns NULL if no suitable node can be found. Otherwise, the best match
- * is returned.
+ * is returned. It's the caller's responsibility to make sure the node doesn't
+ * get destroyed before the caller can access it.
*/
struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_manager *mgr,
unsigned long start,
Choose this option if you have a Samsung SoC EXYNOS chipset.
If M is selected the module will be called exynosdrm.
+if DRM_EXYNOS
+
config DRM_EXYNOS_IOMMU
bool
- depends on DRM_EXYNOS && EXYNOS_IOMMU && ARM_DMA_USE_IOMMU
+ depends on EXYNOS_IOMMU && ARM_DMA_USE_IOMMU
default y
+comment "CRTCs"
+
config DRM_EXYNOS_FIMD
- bool "Exynos DRM FIMD"
- depends on DRM_EXYNOS && !FB_S3C
+ bool "FIMD"
+ depends on !FB_S3C
select FB_MODE_HELPERS
select MFD_SYSCON
help
Choose this option if you want to use Exynos FIMD for DRM.
config DRM_EXYNOS5433_DECON
- bool "Exynos5433 DRM DECON"
- depends on DRM_EXYNOS
+ bool "DECON on Exynos5433"
help
Choose this option if you want to use Exynos5433 DECON for DRM.
config DRM_EXYNOS7_DECON
- bool "Exynos7 DRM DECON"
- depends on DRM_EXYNOS && !FB_S3C
+ bool "DECON on Exynos7"
+ depends on !FB_S3C
select FB_MODE_HELPERS
help
Choose this option if you want to use Exynos DECON for DRM.
+config DRM_EXYNOS_MIXER
+ bool "Mixer"
+ depends on !VIDEO_SAMSUNG_S5P_TV
+ help
+ Choose this option if you want to use Exynos Mixer for DRM.
+
+config DRM_EXYNOS_VIDI
+ bool "Virtual Display"
+ help
+ Choose this option if you want to use Exynos VIDI for DRM.
+
+comment "Encoders and Bridges"
+
config DRM_EXYNOS_DPI
- bool "EXYNOS DRM parallel output support"
- depends on DRM_EXYNOS && (DRM_EXYNOS_FIMD || DRM_EXYNOS7_DECON)
+ bool "Parallel output"
+ depends on DRM_EXYNOS_FIMD
select DRM_PANEL
default n
help
This enables support for Exynos parallel output.
config DRM_EXYNOS_DSI
- bool "EXYNOS DRM MIPI-DSI driver support"
- depends on DRM_EXYNOS && (DRM_EXYNOS_FIMD || DRM_EXYNOS5433_DECON || DRM_EXYNOS7_DECON)
+ bool "MIPI-DSI host"
+ depends on DRM_EXYNOS_FIMD || DRM_EXYNOS5433_DECON || DRM_EXYNOS7_DECON
select DRM_MIPI_DSI
select DRM_PANEL
default n
This enables support for Exynos MIPI-DSI device.
config DRM_EXYNOS_DP
- bool "EXYNOS DRM DP driver support"
- depends on DRM_EXYNOS && (DRM_EXYNOS_FIMD || DRM_EXYNOS7_DECON)
+ bool "Display Port"
+ depends on DRM_EXYNOS_FIMD || DRM_EXYNOS7_DECON
default DRM_EXYNOS
select DRM_PANEL
help
This enables support for DP device.
config DRM_EXYNOS_HDMI
- bool "Exynos DRM HDMI"
- depends on DRM_EXYNOS && !VIDEO_SAMSUNG_S5P_TV
+ bool "HDMI"
+ depends on !VIDEO_SAMSUNG_S5P_TV && (DRM_EXYNOS_MIXER || DRM_EXYNOS5433_DECON)
help
Choose this option if you want to use Exynos HDMI for DRM.
-config DRM_EXYNOS_VIDI
- bool "Exynos DRM Virtual Display"
- depends on DRM_EXYNOS
+config DRM_EXYNOS_MIC
+ bool "Mobile Image Compressor"
+ depends on DRM_EXYNOS5433_DECON
help
- Choose this option if you want to use Exynos VIDI for DRM.
+ Choose this option if you want to use Exynos MIC for DRM.
+
+comment "Sub-drivers"
config DRM_EXYNOS_G2D
- bool "Exynos DRM G2D"
- depends on DRM_EXYNOS && !VIDEO_SAMSUNG_S5P_G2D
+ bool "G2D"
+ depends on !VIDEO_SAMSUNG_S5P_G2D
select FRAME_VECTOR
help
Choose this option if you want to use Exynos G2D for DRM.
config DRM_EXYNOS_IPP
- bool "Exynos DRM IPP"
- depends on DRM_EXYNOS
+ bool "Image Post Processor"
help
Choose this option if you want to use IPP feature for DRM.
config DRM_EXYNOS_FIMC
- bool "Exynos DRM FIMC"
+ bool "FIMC"
depends on DRM_EXYNOS_IPP && MFD_SYSCON
help
Choose this option if you want to use Exynos FIMC for DRM.
config DRM_EXYNOS_ROTATOR
- bool "Exynos DRM Rotator"
+ bool "Rotator"
depends on DRM_EXYNOS_IPP
help
Choose this option if you want to use Exynos Rotator for DRM.
config DRM_EXYNOS_GSC
- bool "Exynos DRM GSC"
+ bool "GScaler"
depends on DRM_EXYNOS_IPP && ARCH_EXYNOS5 && !ARCH_MULTIPLATFORM
help
Choose this option if you want to use Exynos GSC for DRM.
-config DRM_EXYNOS_MIC
- bool "Exynos DRM MIC"
- depends on (DRM_EXYNOS && DRM_EXYNOS5433_DECON)
- help
- Choose this option if you want to use Exynos MIC for DRM.
+endif
exynosdrm-$(CONFIG_DRM_EXYNOS_DPI) += exynos_drm_dpi.o
exynosdrm-$(CONFIG_DRM_EXYNOS_DSI) += exynos_drm_dsi.o
exynosdrm-$(CONFIG_DRM_EXYNOS_DP) += exynos_dp_core.o exynos_dp_reg.o
-exynosdrm-$(CONFIG_DRM_EXYNOS_HDMI) += exynos_hdmi.o exynos_mixer.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_MIXER) += exynos_mixer.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_HDMI) += exynos_hdmi.o
exynosdrm-$(CONFIG_DRM_EXYNOS_VIDI) += exynos_drm_vidi.o
exynosdrm-$(CONFIG_DRM_EXYNOS_G2D) += exynos_drm_g2d.o
exynosdrm-$(CONFIG_DRM_EXYNOS_IPP) += exynos_drm_ipp.o
#include <linux/platform_device.h>
#include <linux/clk.h>
#include <linux/component.h>
+#include <linux/of_device.h>
#include <linux/of_gpio.h>
#include <linux/pm_runtime.h>
#include "exynos_drm_iommu.h"
#define WINDOWS_NR 3
+#define CURSOR_WIN 2
#define MIN_FB_WIDTH_FOR_16WORD_BURST 128
-struct decon_context {
- struct device *dev;
- struct drm_device *drm_dev;
- struct exynos_drm_crtc *crtc;
- struct exynos_drm_plane planes[WINDOWS_NR];
- void __iomem *addr;
- struct clk *clks[6];
- unsigned int default_win;
- unsigned long irq_flags;
- int pipe;
- bool suspended;
-
-#define BIT_CLKS_ENABLED 0
-#define BIT_IRQS_ENABLED 1
- unsigned long enabled;
- bool i80_if;
- atomic_t win_updated;
-};
-
static const char * const decon_clks_name[] = {
+ "pclk",
"aclk_decon",
"aclk_smmu_decon0x",
"aclk_xiu_decon0x",
"sclk_decon_eclk",
};
+enum decon_iftype {
+ IFTYPE_RGB,
+ IFTYPE_I80,
+ IFTYPE_HDMI
+};
+
+enum decon_flag_bits {
+ BIT_CLKS_ENABLED,
+ BIT_IRQS_ENABLED,
+ BIT_WIN_UPDATED,
+ BIT_SUSPENDED
+};
+
+struct decon_context {
+ struct device *dev;
+ struct drm_device *drm_dev;
+ struct exynos_drm_crtc *crtc;
+ struct exynos_drm_plane planes[WINDOWS_NR];
+ void __iomem *addr;
+ struct clk *clks[ARRAY_SIZE(decon_clks_name)];
+ int pipe;
+ unsigned long flags;
+ enum decon_iftype out_type;
+ int first_win;
+};
+
static const uint32_t decon_formats[] = {
DRM_FORMAT_XRGB1555,
DRM_FORMAT_RGB565,
DRM_FORMAT_ARGB8888,
};
+static inline void decon_set_bits(struct decon_context *ctx, u32 reg, u32 mask,
+ u32 val)
+{
+ val = (val & mask) | (readl(ctx->addr + reg) & ~mask);
+ writel(val, ctx->addr + reg);
+}
+
static int decon_enable_vblank(struct exynos_drm_crtc *crtc)
{
struct decon_context *ctx = crtc->ctx;
u32 val;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return -EPERM;
- if (test_and_set_bit(0, &ctx->irq_flags)) {
+ if (test_and_set_bit(BIT_IRQS_ENABLED, &ctx->flags)) {
val = VIDINTCON0_INTEN;
- if (ctx->i80_if)
+ if (ctx->out_type == IFTYPE_I80)
val |= VIDINTCON0_FRAMEDONE;
else
val |= VIDINTCON0_INTFRMEN;
{
struct decon_context *ctx = crtc->ctx;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return;
- if (test_and_clear_bit(0, &ctx->irq_flags))
+ if (test_and_clear_bit(BIT_IRQS_ENABLED, &ctx->flags))
writel(0, ctx->addr + DECON_VIDINTCON0);
}
static void decon_setup_trigger(struct decon_context *ctx)
{
- u32 val = TRIGCON_TRIGEN_PER_F | TRIGCON_TRIGEN_F |
- TRIGCON_TE_AUTO_MASK | TRIGCON_SWTRIGEN;
+ u32 val = (ctx->out_type != IFTYPE_HDMI)
+ ? TRIGCON_TRIGEN_PER_F | TRIGCON_TRIGEN_F |
+ TRIGCON_TE_AUTO_MASK | TRIGCON_SWTRIGEN
+ : TRIGCON_TRIGEN_PER_F | TRIGCON_TRIGEN_F |
+ TRIGCON_HWTRIGMASK_I80_RGB | TRIGCON_HWTRIGEN_I80_RGB;
writel(val, ctx->addr + DECON_TRIGCON);
}
static void decon_commit(struct exynos_drm_crtc *crtc)
{
struct decon_context *ctx = crtc->ctx;
- struct drm_display_mode *mode = &crtc->base.mode;
+ struct drm_display_mode *m = &crtc->base.mode;
u32 val;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return;
+ if (ctx->out_type == IFTYPE_HDMI) {
+ m->crtc_hsync_start = m->crtc_hdisplay + 10;
+ m->crtc_hsync_end = m->crtc_htotal - 92;
+ m->crtc_vsync_start = m->crtc_vdisplay + 1;
+ m->crtc_vsync_end = m->crtc_vsync_start + 1;
+ }
+
+ decon_set_bits(ctx, DECON_VIDCON0, VIDCON0_ENVID, 0);
+
/* enable clock gate */
val = CMU_CLKGAGE_MODE_SFR_F | CMU_CLKGAGE_MODE_MEM_F;
writel(val, ctx->addr + DECON_CMU);
/* lcd on and use command if */
val = VIDOUT_LCD_ON;
- if (ctx->i80_if)
+ if (ctx->out_type == IFTYPE_I80)
val |= VIDOUT_COMMAND_IF;
else
val |= VIDOUT_RGB_IF;
writel(val, ctx->addr + DECON_VIDOUTCON0);
- val = VIDTCON2_LINEVAL(mode->vdisplay - 1) |
- VIDTCON2_HOZVAL(mode->hdisplay - 1);
+ val = VIDTCON2_LINEVAL(m->vdisplay - 1) |
+ VIDTCON2_HOZVAL(m->hdisplay - 1);
writel(val, ctx->addr + DECON_VIDTCON2);
- if (!ctx->i80_if) {
+ if (ctx->out_type != IFTYPE_I80) {
val = VIDTCON00_VBPD_F(
- mode->crtc_vtotal - mode->crtc_vsync_end) |
+ m->crtc_vtotal - m->crtc_vsync_end - 1) |
VIDTCON00_VFPD_F(
- mode->crtc_vsync_start - mode->crtc_vdisplay);
+ m->crtc_vsync_start - m->crtc_vdisplay - 1);
writel(val, ctx->addr + DECON_VIDTCON00);
val = VIDTCON01_VSPW_F(
- mode->crtc_vsync_end - mode->crtc_vsync_start);
+ m->crtc_vsync_end - m->crtc_vsync_start - 1);
writel(val, ctx->addr + DECON_VIDTCON01);
val = VIDTCON10_HBPD_F(
- mode->crtc_htotal - mode->crtc_hsync_end) |
+ m->crtc_htotal - m->crtc_hsync_end - 1) |
VIDTCON10_HFPD_F(
- mode->crtc_hsync_start - mode->crtc_hdisplay);
+ m->crtc_hsync_start - m->crtc_hdisplay - 1);
writel(val, ctx->addr + DECON_VIDTCON10);
val = VIDTCON11_HSPW_F(
- mode->crtc_hsync_end - mode->crtc_hsync_start);
+ m->crtc_hsync_end - m->crtc_hsync_start - 1);
writel(val, ctx->addr + DECON_VIDTCON11);
}
decon_setup_trigger(ctx);
/* enable output and display signal */
- val = VIDCON0_ENVID | VIDCON0_ENVID_F;
- writel(val, ctx->addr + DECON_VIDCON0);
+ decon_set_bits(ctx, DECON_VIDCON0, VIDCON0_ENVID | VIDCON0_ENVID_F, ~0);
}
-#define COORDINATE_X(x) (((x) & 0xfff) << 12)
-#define COORDINATE_Y(x) ((x) & 0xfff)
-#define OFFSIZE(x) (((x) & 0x3fff) << 14)
-#define PAGEWIDTH(x) ((x) & 0x3fff)
-
static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win,
struct drm_framebuffer *fb)
{
static void decon_shadow_protect_win(struct decon_context *ctx, int win,
bool protect)
{
- u32 val;
-
- val = readl(ctx->addr + DECON_SHADOWCON);
-
- if (protect)
- val |= SHADOWCON_Wx_PROTECT(win);
- else
- val &= ~SHADOWCON_Wx_PROTECT(win);
-
- writel(val, ctx->addr + DECON_SHADOWCON);
+ decon_set_bits(ctx, DECON_SHADOWCON, SHADOWCON_Wx_PROTECT(win),
+ protect ? ~0 : 0);
}
static void decon_atomic_begin(struct exynos_drm_crtc *crtc,
{
struct decon_context *ctx = crtc->ctx;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return;
decon_shadow_protect_win(ctx, plane->zpos, true);
}
+#define BIT_VAL(x, e, s) (((x) & ((1 << ((e) - (s) + 1)) - 1)) << (s))
+#define COORDINATE_X(x) BIT_VAL((x), 23, 12)
+#define COORDINATE_Y(x) BIT_VAL((x), 11, 0)
+
static void decon_update_plane(struct exynos_drm_crtc *crtc,
struct exynos_drm_plane *plane)
{
unsigned int pitch = state->fb->pitches[0];
u32 val;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return;
val = COORDINATE_X(plane->crtc_x) | COORDINATE_Y(plane->crtc_y);
val = plane->dma_addr[0] + pitch * plane->crtc_h;
writel(val, ctx->addr + DECON_VIDW0xADD1B0(win));
- val = OFFSIZE(pitch - plane->crtc_w * bpp)
- | PAGEWIDTH(plane->crtc_w * bpp);
+ if (ctx->out_type != IFTYPE_HDMI)
+ val = BIT_VAL(pitch - plane->crtc_w * bpp, 27, 14)
+ | BIT_VAL(plane->crtc_w * bpp, 13, 0);
+ else
+ val = BIT_VAL(pitch - plane->crtc_w * bpp, 29, 15)
+ | BIT_VAL(plane->crtc_w * bpp, 14, 0);
writel(val, ctx->addr + DECON_VIDW0xADD2(win));
decon_win_set_pixfmt(ctx, win, state->fb);
/* window enable */
- val = readl(ctx->addr + DECON_WINCONx(win));
- val |= WINCONx_ENWIN_F;
- writel(val, ctx->addr + DECON_WINCONx(win));
+ decon_set_bits(ctx, DECON_WINCONx(win), WINCONx_ENWIN_F, ~0);
/* standalone update */
- val = readl(ctx->addr + DECON_UPDATE);
- val |= STANDALONE_UPDATE_F;
- writel(val, ctx->addr + DECON_UPDATE);
+ decon_set_bits(ctx, DECON_UPDATE, STANDALONE_UPDATE_F, ~0);
}
static void decon_disable_plane(struct exynos_drm_crtc *crtc,
{
struct decon_context *ctx = crtc->ctx;
unsigned int win = plane->zpos;
- u32 val;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return;
decon_shadow_protect_win(ctx, win, true);
/* window disable */
- val = readl(ctx->addr + DECON_WINCONx(win));
- val &= ~WINCONx_ENWIN_F;
- writel(val, ctx->addr + DECON_WINCONx(win));
+ decon_set_bits(ctx, DECON_WINCONx(win), WINCONx_ENWIN_F, 0);
decon_shadow_protect_win(ctx, win, false);
/* standalone update */
- val = readl(ctx->addr + DECON_UPDATE);
- val |= STANDALONE_UPDATE_F;
- writel(val, ctx->addr + DECON_UPDATE);
+ decon_set_bits(ctx, DECON_UPDATE, STANDALONE_UPDATE_F, ~0);
}
static void decon_atomic_flush(struct exynos_drm_crtc *crtc,
{
struct decon_context *ctx = crtc->ctx;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return;
decon_shadow_protect_win(ctx, plane->zpos, false);
- if (ctx->i80_if)
- atomic_set(&ctx->win_updated, 1);
+ if (ctx->out_type == IFTYPE_I80)
+ set_bit(BIT_WIN_UPDATED, &ctx->flags);
}
static void decon_swreset(struct decon_context *ctx)
}
WARN(tries == 0, "failed to software reset DECON\n");
+
+ if (ctx->out_type != IFTYPE_HDMI)
+ return;
+
+ writel(VIDCON0_CLKVALUP | VIDCON0_VLCKFREE, ctx->addr + DECON_VIDCON0);
+ decon_set_bits(ctx, DECON_CMU,
+ CMU_CLKGAGE_MODE_SFR_F | CMU_CLKGAGE_MODE_MEM_F, ~0);
+ writel(VIDCON1_VCLK_RUN_VDEN_DISABLE, ctx->addr + DECON_VIDCON1);
+ writel(CRCCTRL_CRCEN | CRCCTRL_CRCSTART_F | CRCCTRL_CRCCLKEN,
+ ctx->addr + DECON_CRCCTRL);
+ decon_setup_trigger(ctx);
}
static void decon_enable(struct exynos_drm_crtc *crtc)
int ret;
int i;
- if (!ctx->suspended)
+ if (!test_and_clear_bit(BIT_SUSPENDED, &ctx->flags))
return;
- ctx->suspended = false;
-
pm_runtime_get_sync(ctx->dev);
for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++) {
goto err;
}
- set_bit(BIT_CLKS_ENABLED, &ctx->enabled);
+ set_bit(BIT_CLKS_ENABLED, &ctx->flags);
/* if vblank was enabled status, enable it again. */
- if (test_and_clear_bit(0, &ctx->irq_flags))
+ if (test_and_clear_bit(BIT_IRQS_ENABLED, &ctx->flags))
decon_enable_vblank(ctx->crtc);
decon_commit(ctx->crtc);
while (--i >= 0)
clk_disable_unprepare(ctx->clks[i]);
- ctx->suspended = true;
+ set_bit(BIT_SUSPENDED, &ctx->flags);
}
static void decon_disable(struct exynos_drm_crtc *crtc)
struct decon_context *ctx = crtc->ctx;
int i;
- if (ctx->suspended)
+ if (test_bit(BIT_SUSPENDED, &ctx->flags))
return;
/*
* suspend that connector. Otherwise we might try to scan from
* a destroyed buffer later.
*/
- for (i = 0; i < WINDOWS_NR; i++)
+ for (i = ctx->first_win; i < WINDOWS_NR; i++)
decon_disable_plane(crtc, &ctx->planes[i]);
decon_swreset(ctx);
for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++)
clk_disable_unprepare(ctx->clks[i]);
- clear_bit(BIT_CLKS_ENABLED, &ctx->enabled);
+ clear_bit(BIT_CLKS_ENABLED, &ctx->flags);
pm_runtime_put_sync(ctx->dev);
- ctx->suspended = true;
+ set_bit(BIT_SUSPENDED, &ctx->flags);
}
void decon_te_irq_handler(struct exynos_drm_crtc *crtc)
{
struct decon_context *ctx = crtc->ctx;
- u32 val;
- if (!test_bit(BIT_CLKS_ENABLED, &ctx->enabled))
+ if (!test_bit(BIT_CLKS_ENABLED, &ctx->flags))
return;
- if (atomic_add_unless(&ctx->win_updated, -1, 0)) {
- /* trigger */
- val = readl(ctx->addr + DECON_TRIGCON);
- val |= TRIGCON_SWTRIGCMD;
- writel(val, ctx->addr + DECON_TRIGCON);
- }
+ if (test_and_clear_bit(BIT_WIN_UPDATED, &ctx->flags))
+ decon_set_bits(ctx, DECON_TRIGCON, TRIGCON_SWTRIGCMD, ~0);
drm_crtc_handle_vblank(&ctx->crtc->base);
}
{
struct decon_context *ctx = crtc->ctx;
int win, i, ret;
- u32 val;
DRM_DEBUG_KMS("%s\n", __FILE__);
}
for (win = 0; win < WINDOWS_NR; win++) {
- /* shadow update disable */
- val = readl(ctx->addr + DECON_SHADOWCON);
- val |= SHADOWCON_Wx_PROTECT(win);
- writel(val, ctx->addr + DECON_SHADOWCON);
-
- /* window disable */
- val = readl(ctx->addr + DECON_WINCONx(win));
- val &= ~WINCONx_ENWIN_F;
- writel(val, ctx->addr + DECON_WINCONx(win));
-
- /* shadow update enable */
- val = readl(ctx->addr + DECON_SHADOWCON);
- val &= ~SHADOWCON_Wx_PROTECT(win);
- writel(val, ctx->addr + DECON_SHADOWCON);
-
- /* standalone update */
- val = readl(ctx->addr + DECON_UPDATE);
- val |= STANDALONE_UPDATE_F;
- writel(val, ctx->addr + DECON_UPDATE);
+ decon_shadow_protect_win(ctx, win, true);
+ decon_set_bits(ctx, DECON_WINCONx(win), WINCONx_ENWIN_F, 0);
+ decon_shadow_protect_win(ctx, win, false);
+ decon_set_bits(ctx, DECON_UPDATE, STANDALONE_UPDATE_F, ~0);
}
/* TODO: wait for possible vsync */
msleep(50);
.commit = decon_commit,
.enable_vblank = decon_enable_vblank,
.disable_vblank = decon_disable_vblank,
- .commit = decon_commit,
.atomic_begin = decon_atomic_begin,
.update_plane = decon_update_plane,
.disable_plane = decon_disable_plane,
struct drm_device *drm_dev = data;
struct exynos_drm_private *priv = drm_dev->dev_private;
struct exynos_drm_plane *exynos_plane;
+ enum exynos_drm_output_type out_type;
enum drm_plane_type type;
- unsigned int zpos;
+ unsigned int win;
int ret;
ctx->drm_dev = drm_dev;
ctx->pipe = priv->pipe++;
- for (zpos = 0; zpos < WINDOWS_NR; zpos++) {
- type = (zpos == ctx->default_win) ? DRM_PLANE_TYPE_PRIMARY :
- DRM_PLANE_TYPE_OVERLAY;
- ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
+ for (win = ctx->first_win; win < WINDOWS_NR; win++) {
+ int tmp = (win == ctx->first_win) ? 0 : win;
+
+ type = exynos_plane_get_type(tmp, CURSOR_WIN);
+ ret = exynos_plane_init(drm_dev, &ctx->planes[win],
1 << ctx->pipe, type, decon_formats,
- ARRAY_SIZE(decon_formats), zpos);
+ ARRAY_SIZE(decon_formats), win);
if (ret)
return ret;
}
- exynos_plane = &ctx->planes[ctx->default_win];
+ exynos_plane = &ctx->planes[ctx->first_win];
+ out_type = (ctx->out_type == IFTYPE_HDMI) ? EXYNOS_DISPLAY_TYPE_HDMI
+ : EXYNOS_DISPLAY_TYPE_LCD;
ctx->crtc = exynos_drm_crtc_create(drm_dev, &exynos_plane->base,
- ctx->pipe, EXYNOS_DISPLAY_TYPE_LCD,
+ ctx->pipe, out_type,
&decon_crtc_ops, ctx);
if (IS_ERR(ctx->crtc)) {
ret = PTR_ERR(ctx->crtc);
.unbind = decon_unbind,
};
-static irqreturn_t decon_vsync_irq_handler(int irq, void *dev_id)
-{
- struct decon_context *ctx = dev_id;
- u32 val;
-
- if (!test_bit(BIT_CLKS_ENABLED, &ctx->enabled))
- goto out;
-
- val = readl(ctx->addr + DECON_VIDINTCON1);
- if (val & VIDINTCON1_INTFRMPEND) {
- drm_crtc_handle_vblank(&ctx->crtc->base);
-
- /* clear */
- writel(VIDINTCON1_INTFRMPEND, ctx->addr + DECON_VIDINTCON1);
- }
-
-out:
- return IRQ_HANDLED;
-}
-
-static irqreturn_t decon_lcd_sys_irq_handler(int irq, void *dev_id)
+static irqreturn_t decon_irq_handler(int irq, void *dev_id)
{
struct decon_context *ctx = dev_id;
u32 val;
int win;
- if (!test_bit(BIT_CLKS_ENABLED, &ctx->enabled))
+ if (!test_bit(BIT_CLKS_ENABLED, &ctx->flags))
goto out;
val = readl(ctx->addr + DECON_VIDINTCON1);
- if (val & VIDINTCON1_INTFRMDONEPEND) {
- for (win = 0 ; win < WINDOWS_NR ; win++) {
+ val &= VIDINTCON1_INTFRMDONEPEND | VIDINTCON1_INTFRMPEND;
+
+ if (val) {
+ for (win = ctx->first_win; win < WINDOWS_NR ; win++) {
struct exynos_drm_plane *plane = &ctx->planes[win];
if (!plane->pending_fb)
}
/* clear */
- writel(VIDINTCON1_INTFRMDONEPEND,
- ctx->addr + DECON_VIDINTCON1);
+ writel(val, ctx->addr + DECON_VIDINTCON1);
}
out:
return IRQ_HANDLED;
}
+static const struct of_device_id exynos5433_decon_driver_dt_match[] = {
+ {
+ .compatible = "samsung,exynos5433-decon",
+ .data = (void *)IFTYPE_RGB
+ },
+ {
+ .compatible = "samsung,exynos5433-decon-tv",
+ .data = (void *)IFTYPE_HDMI
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, exynos5433_decon_driver_dt_match);
+
static int exynos5433_decon_probe(struct platform_device *pdev)
{
+ const struct of_device_id *of_id;
struct device *dev = &pdev->dev;
struct decon_context *ctx;
struct resource *res;
if (!ctx)
return -ENOMEM;
- ctx->default_win = 0;
- ctx->suspended = true;
+ __set_bit(BIT_SUSPENDED, &ctx->flags);
ctx->dev = dev;
- if (of_get_child_by_name(dev->of_node, "i80-if-timings"))
- ctx->i80_if = true;
+
+ of_id = of_match_device(exynos5433_decon_driver_dt_match, &pdev->dev);
+ ctx->out_type = (enum decon_iftype)of_id->data;
+
+ if (ctx->out_type == IFTYPE_HDMI)
+ ctx->first_win = 1;
+ else if (of_get_child_by_name(dev->of_node, "i80-if-timings"))
+ ctx->out_type = IFTYPE_I80;
for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++) {
struct clk *clk;
}
res = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
- ctx->i80_if ? "lcd_sys" : "vsync");
+ (ctx->out_type == IFTYPE_I80) ? "lcd_sys" : "vsync");
if (!res) {
dev_err(dev, "cannot find IRQ resource\n");
return -ENXIO;
}
- ret = devm_request_irq(dev, res->start, ctx->i80_if ?
- decon_lcd_sys_irq_handler : decon_vsync_irq_handler, 0,
- "drm_decon", ctx);
+ ret = devm_request_irq(dev, res->start, decon_irq_handler, 0,
+ "drm_decon", ctx);
if (ret < 0) {
dev_err(dev, "lcd_sys irq request failed\n");
return ret;
return 0;
}
-static const struct of_device_id exynos5433_decon_driver_dt_match[] = {
- { .compatible = "samsung,exynos5433-decon" },
- {},
-};
-MODULE_DEVICE_TABLE(of, exynos5433_decon_driver_dt_match);
-
struct platform_driver exynos5433_decon_driver = {
.probe = exynos5433_decon_probe,
.remove = exynos5433_decon_remove,
* DECON stands for Display and Enhancement controller.
*/
-#define DECON_DEFAULT_FRAMERATE 60
#define MIN_FB_WIDTH_FOR_16WORD_BURST 128
#define WINDOWS_NR 2
+#define CURSOR_WIN 1
struct decon_context {
struct device *dev;
struct clk *eclk;
struct clk *vclk;
void __iomem *regs;
- unsigned int default_win;
unsigned long irq_flags;
bool i80_if;
bool suspended;
return (clkdiv < 0x100) ? clkdiv : 0xff;
}
-static bool decon_mode_fixup(struct exynos_drm_crtc *crtc,
- const struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
-{
- if (adjusted_mode->vrefresh == 0)
- adjusted_mode->vrefresh = DECON_DEFAULT_FRAMERATE;
-
- return true;
-}
-
static void decon_commit(struct exynos_drm_crtc *crtc)
{
struct decon_context *ctx = crtc->ctx;
static const struct exynos_drm_crtc_ops decon_crtc_ops = {
.enable = decon_enable,
.disable = decon_disable,
- .mode_fixup = decon_mode_fixup,
.commit = decon_commit,
.enable_vblank = decon_enable_vblank,
.disable_vblank = decon_disable_vblank,
}
for (zpos = 0; zpos < WINDOWS_NR; zpos++) {
- type = (zpos == ctx->default_win) ? DRM_PLANE_TYPE_PRIMARY :
- DRM_PLANE_TYPE_OVERLAY;
+ type = exynos_plane_get_type(zpos, CURSOR_WIN);
ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
1 << ctx->pipe, type, decon_formats,
ARRAY_SIZE(decon_formats), zpos);
return ret;
}
- exynos_plane = &ctx->planes[ctx->default_win];
+ exynos_plane = &ctx->planes[DEFAULT_WIN];
ctx->crtc = exynos_drm_crtc_create(drm_dev, &exynos_plane->base,
ctx->pipe, EXYNOS_DISPLAY_TYPE_LCD,
&decon_crtc_ops, ctx);
return 0;
}
-#ifdef CONFIG_PM_SLEEP
-static int exynos_dp_suspend(struct device *dev)
-{
- struct exynos_dp_device *dp = dev_get_drvdata(dev);
-
- exynos_dp_disable(&dp->encoder);
- return 0;
-}
-
-static int exynos_dp_resume(struct device *dev)
-{
- struct exynos_dp_device *dp = dev_get_drvdata(dev);
-
- exynos_dp_enable(&dp->encoder);
- return 0;
-}
-#endif
-
-static const struct dev_pm_ops exynos_dp_pm_ops = {
- SET_SYSTEM_SLEEP_PM_OPS(exynos_dp_suspend, exynos_dp_resume)
-};
-
static const struct of_device_id exynos_dp_match[] = {
{ .compatible = "samsung,exynos5-dp" },
{},
.driver = {
.name = "exynos-dp",
.owner = THIS_MODULE,
- .pm = &exynos_dp_pm_ops,
.of_match_table = exynos_dp_match,
},
};
return 0;
}
-EXPORT_SYMBOL_GPL(exynos_drm_subdrv_register);
int exynos_drm_subdrv_unregister(struct exynos_drm_subdrv *subdrv)
{
return 0;
}
-EXPORT_SYMBOL_GPL(exynos_drm_subdrv_unregister);
int exynos_drm_device_subdrv_probe(struct drm_device *dev)
{
return 0;
}
-EXPORT_SYMBOL_GPL(exynos_drm_device_subdrv_probe);
int exynos_drm_device_subdrv_remove(struct drm_device *dev)
{
return 0;
}
-EXPORT_SYMBOL_GPL(exynos_drm_device_subdrv_remove);
int exynos_drm_subdrv_open(struct drm_device *dev, struct drm_file *file)
{
}
return ret;
}
-EXPORT_SYMBOL_GPL(exynos_drm_subdrv_open);
void exynos_drm_subdrv_close(struct drm_device *dev, struct drm_file *file)
{
subdrv->close(dev, subdrv->dev, file);
}
}
-EXPORT_SYMBOL_GPL(exynos_drm_subdrv_close);
exynos_crtc->ops->disable(exynos_crtc);
}
-static bool
-exynos_drm_crtc_mode_fixup(struct drm_crtc *crtc,
- const struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
+static void
+exynos_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
{
struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
- if (exynos_crtc->ops->mode_fixup)
- return exynos_crtc->ops->mode_fixup(exynos_crtc, mode,
- adjusted_mode);
-
- return true;
+ if (exynos_crtc->ops->commit)
+ exynos_crtc->ops->commit(exynos_crtc);
}
-static void
-exynos_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
+static int exynos_crtc_atomic_check(struct drm_crtc *crtc,
+ struct drm_crtc_state *state)
{
struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
- if (exynos_crtc->ops->commit)
- exynos_crtc->ops->commit(exynos_crtc);
+ if (exynos_crtc->ops->atomic_check)
+ return exynos_crtc->ops->atomic_check(exynos_crtc, state);
+
+ return 0;
}
static void exynos_crtc_atomic_begin(struct drm_crtc *crtc,
static struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = {
.enable = exynos_drm_crtc_enable,
.disable = exynos_drm_crtc_disable,
- .mode_fixup = exynos_drm_crtc_mode_fixup,
.mode_set_nofb = exynos_drm_crtc_mode_set_nofb,
+ .atomic_check = exynos_crtc_atomic_check,
.atomic_begin = exynos_crtc_atomic_begin,
.atomic_flush = exynos_crtc_atomic_flush,
};
return ERR_PTR(ret);
}
-int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int pipe)
+int exynos_drm_crtc_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct exynos_drm_private *private = dev->dev_private;
struct exynos_drm_crtc *exynos_crtc =
return 0;
}
-void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int pipe)
+void exynos_drm_crtc_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct exynos_drm_private *private = dev->dev_private;
struct exynos_drm_crtc *exynos_crtc =
enum exynos_drm_output_type type,
const struct exynos_drm_crtc_ops *ops,
void *context);
-int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int pipe);
-void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int pipe);
+int exynos_drm_crtc_enable_vblank(struct drm_device *dev, unsigned int pipe);
+void exynos_drm_crtc_disable_vblank(struct drm_device *dev, unsigned int pipe);
void exynos_drm_crtc_wait_pending_update(struct exynos_drm_crtc *exynos_crtc);
void exynos_drm_crtc_finish_update(struct exynos_drm_crtc *exynos_crtc,
struct exynos_drm_plane *exynos_plane);
atomic_inc(&exynos_crtc->pending_update);
}
- drm_atomic_helper_commit_planes(dev, state);
+ drm_atomic_helper_commit_planes(dev, state, false);
exynos_atomic_wait_for_commit(state);
return 0;
}
+#ifdef CONFIG_PM_SLEEP
static int exynos_drm_suspend(struct drm_device *dev, pm_message_t state)
{
struct drm_connector *connector;
return 0;
}
+#endif
static int exynos_drm_open(struct drm_device *dev, struct drm_file *file)
{
static const struct drm_ioctl_desc exynos_ioctls[] = {
DRM_IOCTL_DEF_DRV(EXYNOS_GEM_CREATE, exynos_drm_gem_create_ioctl,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_GEM_GET, exynos_drm_gem_get_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_VIDI_CONNECTION, vidi_connection_ioctl,
- DRM_UNLOCKED | DRM_AUTH),
+ DRM_AUTH),
DRM_IOCTL_DEF_DRV(EXYNOS_G2D_GET_VER, exynos_g2d_get_ver_ioctl,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_G2D_SET_CMDLIST, exynos_g2d_set_cmdlist_ioctl,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_G2D_EXEC, exynos_g2d_exec_ioctl,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_IPP_GET_PROPERTY, exynos_drm_ipp_get_property,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_IPP_SET_PROPERTY, exynos_drm_ipp_set_property,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_IPP_QUEUE_BUF, exynos_drm_ipp_queue_buf,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(EXYNOS_IPP_CMD_CTRL, exynos_drm_ipp_cmd_ctrl,
- DRM_UNLOCKED | DRM_AUTH | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
};
static const struct file_operations exynos_drm_driver_fops = {
.lastclose = exynos_drm_lastclose,
.postclose = exynos_drm_postclose,
.set_busid = drm_platform_set_busid,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = exynos_drm_crtc_enable_vblank,
.disable_vblank = exynos_drm_crtc_disable_vblank,
.gem_free_object = exynos_drm_gem_free_object,
#ifdef CONFIG_DRM_EXYNOS_DSI
&dsi_driver,
#endif
-#ifdef CONFIG_DRM_EXYNOS_HDMI
+#ifdef CONFIG_DRM_EXYNOS_MIXER
&mixer_driver,
+#endif
+#ifdef CONFIG_DRM_EXYNOS_HDMI
&hdmi_driver,
#endif
#ifdef CONFIG_DRM_EXYNOS_VIDI
#define MAX_PLANE 5
#define MAX_FB_BUFFER 4
+#define DEFAULT_WIN 0
+
#define to_exynos_crtc(x) container_of(x, struct exynos_drm_crtc, base)
#define to_exynos_plane(x) container_of(x, struct exynos_drm_plane, base)
*
* @enable: enable the device
* @disable: disable the device
- * @mode_fixup: fix mode data before applying it
* @commit: set current hw specific display mode to hw.
* @enable_vblank: specific driver callback for enabling vblank interrupt.
* @disable_vblank: specific driver callback for disabling vblank interrupt.
* @wait_for_vblank: wait for vblank interrupt to make sure that
* hardware overlay is updated.
+ * @atomic_check: validate state
* @atomic_begin: prepare a window to receive a update
* @atomic_flush: mark the end of a window update
* @update_plane: apply hardware specific overlay data to registers.
struct exynos_drm_crtc_ops {
void (*enable)(struct exynos_drm_crtc *crtc);
void (*disable)(struct exynos_drm_crtc *crtc);
- bool (*mode_fixup)(struct exynos_drm_crtc *crtc,
- const struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode);
void (*commit)(struct exynos_drm_crtc *crtc);
int (*enable_vblank)(struct exynos_drm_crtc *crtc);
void (*disable_vblank)(struct exynos_drm_crtc *crtc);
void (*wait_for_vblank)(struct exynos_drm_crtc *crtc);
+ int (*atomic_check)(struct exynos_drm_crtc *crtc,
+ struct drm_crtc_state *state);
void (*atomic_begin)(struct exynos_drm_crtc *crtc,
struct exynos_drm_plane *plane);
void (*update_plane)(struct exynos_drm_crtc *crtc,
* exynos specific framebuffer structure.
*
* @fb: drm framebuffer obejct.
- * @exynos_gem_obj: array of exynos specific gem object containing a gem object.
+ * @exynos_gem: array of exynos specific gem object containing a gem object.
*/
struct exynos_drm_fb {
- struct drm_framebuffer fb;
- struct exynos_drm_gem_obj *exynos_gem_obj[MAX_FB_BUFFER];
+ struct drm_framebuffer fb;
+ struct exynos_drm_gem *exynos_gem[MAX_FB_BUFFER];
};
static int check_fb_gem_memory_type(struct drm_device *drm_dev,
- struct exynos_drm_gem_obj *exynos_gem_obj)
+ struct exynos_drm_gem *exynos_gem)
{
unsigned int flags;
if (is_drm_iommu_supported(drm_dev))
return 0;
- flags = exynos_gem_obj->flags;
+ flags = exynos_gem->flags;
/*
* without iommu support, not support physically non-continuous memory
drm_framebuffer_cleanup(fb);
- for (i = 0; i < ARRAY_SIZE(exynos_fb->exynos_gem_obj); i++) {
+ for (i = 0; i < ARRAY_SIZE(exynos_fb->exynos_gem); i++) {
struct drm_gem_object *obj;
- if (exynos_fb->exynos_gem_obj[i] == NULL)
+ if (exynos_fb->exynos_gem[i] == NULL)
continue;
- obj = &exynos_fb->exynos_gem_obj[i]->base;
+ obj = &exynos_fb->exynos_gem[i]->base;
drm_gem_object_unreference_unlocked(obj);
}
struct exynos_drm_fb *exynos_fb = to_exynos_fb(fb);
return drm_gem_handle_create(file_priv,
- &exynos_fb->exynos_gem_obj[0]->base, handle);
+ &exynos_fb->exynos_gem[0]->base, handle);
}
static int exynos_drm_fb_dirty(struct drm_framebuffer *fb,
struct drm_framebuffer *
exynos_drm_framebuffer_init(struct drm_device *dev,
struct drm_mode_fb_cmd2 *mode_cmd,
- struct exynos_drm_gem_obj **gem_obj,
+ struct exynos_drm_gem **exynos_gem,
int count)
{
struct exynos_drm_fb *exynos_fb;
return ERR_PTR(-ENOMEM);
for (i = 0; i < count; i++) {
- ret = check_fb_gem_memory_type(dev, gem_obj[i]);
+ ret = check_fb_gem_memory_type(dev, exynos_gem[i]);
if (ret < 0)
goto err;
- exynos_fb->exynos_gem_obj[i] = gem_obj[i];
+ exynos_fb->exynos_gem[i] = exynos_gem[i];
}
drm_helper_mode_fill_fb_struct(&exynos_fb->fb, mode_cmd);
exynos_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
struct drm_mode_fb_cmd2 *mode_cmd)
{
- struct exynos_drm_gem_obj *gem_objs[MAX_FB_BUFFER];
+ struct exynos_drm_gem *exynos_gem[MAX_FB_BUFFER];
struct drm_gem_object *obj;
struct drm_framebuffer *fb;
int i;
goto err;
}
- gem_objs[i] = to_exynos_gem_obj(obj);
+ exynos_gem[i] = to_exynos_gem(obj);
}
- fb = exynos_drm_framebuffer_init(dev, mode_cmd, gem_objs, i);
+ fb = exynos_drm_framebuffer_init(dev, mode_cmd, exynos_gem, i);
if (IS_ERR(fb)) {
ret = PTR_ERR(fb);
goto err;
err:
while (i--)
- drm_gem_object_unreference_unlocked(&gem_objs[i]->base);
+ drm_gem_object_unreference_unlocked(&exynos_gem[i]->base);
return ERR_PTR(ret);
}
-struct exynos_drm_gem_obj *exynos_drm_fb_gem_obj(struct drm_framebuffer *fb,
- int index)
+struct exynos_drm_gem *exynos_drm_fb_gem(struct drm_framebuffer *fb, int index)
{
struct exynos_drm_fb *exynos_fb = to_exynos_fb(fb);
- struct exynos_drm_gem_obj *obj;
+ struct exynos_drm_gem *exynos_gem;
if (index >= MAX_FB_BUFFER)
return NULL;
- obj = exynos_fb->exynos_gem_obj[index];
- if (!obj)
+ exynos_gem = exynos_fb->exynos_gem[index];
+ if (!exynos_gem)
return NULL;
- DRM_DEBUG_KMS("dma_addr = 0x%lx\n", (unsigned long)obj->dma_addr);
+ DRM_DEBUG_KMS("dma_addr: 0x%lx\n", (unsigned long)exynos_gem->dma_addr);
- return obj;
+ return exynos_gem;
}
static void exynos_drm_output_poll_changed(struct drm_device *dev)
struct drm_framebuffer *
exynos_drm_framebuffer_init(struct drm_device *dev,
struct drm_mode_fb_cmd2 *mode_cmd,
- struct exynos_drm_gem_obj **gem_obj,
+ struct exynos_drm_gem **exynos_gem,
int count);
/* get gem object of a drm framebuffer */
-struct exynos_drm_gem_obj *exynos_drm_fb_gem_obj(struct drm_framebuffer *fb,
- int index);
+struct exynos_drm_gem *exynos_drm_fb_gem(struct drm_framebuffer *fb, int index);
void exynos_drm_mode_config_init(struct drm_device *dev);
drm_fb_helper)
struct exynos_drm_fbdev {
- struct drm_fb_helper drm_fb_helper;
- struct exynos_drm_gem_obj *obj;
+ struct drm_fb_helper drm_fb_helper;
+ struct exynos_drm_gem *exynos_gem;
};
static int exynos_drm_fb_mmap(struct fb_info *info,
{
struct drm_fb_helper *helper = info->par;
struct exynos_drm_fbdev *exynos_fbd = to_exynos_fbdev(helper);
- struct exynos_drm_gem_obj *obj = exynos_fbd->obj;
+ struct exynos_drm_gem *exynos_gem = exynos_fbd->exynos_gem;
unsigned long vm_size;
int ret;
vm_size = vma->vm_end - vma->vm_start;
- if (vm_size > obj->size)
+ if (vm_size > exynos_gem->size)
return -EINVAL;
- ret = dma_mmap_attrs(helper->dev->dev, vma, obj->pages, obj->dma_addr,
- obj->size, &obj->dma_attrs);
+ ret = dma_mmap_attrs(helper->dev->dev, vma, exynos_gem->pages,
+ exynos_gem->dma_addr, exynos_gem->size,
+ &exynos_gem->dma_attrs);
if (ret < 0) {
DRM_ERROR("failed to mmap.\n");
return ret;
static int exynos_drm_fbdev_update(struct drm_fb_helper *helper,
struct drm_fb_helper_surface_size *sizes,
- struct exynos_drm_gem_obj *obj)
+ struct exynos_drm_gem *exynos_gem)
{
struct fb_info *fbi;
struct drm_framebuffer *fb = helper->fb;
drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->depth);
drm_fb_helper_fill_var(fbi, helper, sizes->fb_width, sizes->fb_height);
- nr_pages = obj->size >> PAGE_SHIFT;
+ nr_pages = exynos_gem->size >> PAGE_SHIFT;
- obj->kvaddr = (void __iomem *) vmap(obj->pages, nr_pages, VM_MAP,
- pgprot_writecombine(PAGE_KERNEL));
- if (!obj->kvaddr) {
+ exynos_gem->kvaddr = (void __iomem *) vmap(exynos_gem->pages, nr_pages,
+ VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+ if (!exynos_gem->kvaddr) {
DRM_ERROR("failed to map pages to kernel space.\n");
drm_fb_helper_release_fbi(helper);
return -EIO;
offset = fbi->var.xoffset * (fb->bits_per_pixel >> 3);
offset += fbi->var.yoffset * fb->pitches[0];
- fbi->screen_base = obj->kvaddr + offset;
+ fbi->screen_base = exynos_gem->kvaddr + offset;
fbi->screen_size = size;
fbi->fix.smem_len = size;
struct drm_fb_helper_surface_size *sizes)
{
struct exynos_drm_fbdev *exynos_fbdev = to_exynos_fbdev(helper);
- struct exynos_drm_gem_obj *obj;
+ struct exynos_drm_gem *exynos_gem;
struct drm_device *dev = helper->dev;
struct drm_mode_fb_cmd2 mode_cmd = { 0 };
struct platform_device *pdev = dev->platformdev;
size = mode_cmd.pitches[0] * mode_cmd.height;
- obj = exynos_drm_gem_create(dev, EXYNOS_BO_CONTIG, size);
+ exynos_gem = exynos_drm_gem_create(dev, EXYNOS_BO_CONTIG, size);
/*
* If physically contiguous memory allocation fails and if IOMMU is
* supported then try to get buffer from non physically contiguous
* memory area.
*/
- if (IS_ERR(obj) && is_drm_iommu_supported(dev)) {
+ if (IS_ERR(exynos_gem) && is_drm_iommu_supported(dev)) {
dev_warn(&pdev->dev, "contiguous FB allocation failed, falling back to non-contiguous\n");
- obj = exynos_drm_gem_create(dev, EXYNOS_BO_NONCONTIG, size);
+ exynos_gem = exynos_drm_gem_create(dev, EXYNOS_BO_NONCONTIG,
+ size);
}
- if (IS_ERR(obj)) {
- ret = PTR_ERR(obj);
+ if (IS_ERR(exynos_gem)) {
+ ret = PTR_ERR(exynos_gem);
goto out;
}
- exynos_fbdev->obj = obj;
+ exynos_fbdev->exynos_gem = exynos_gem;
- helper->fb = exynos_drm_framebuffer_init(dev, &mode_cmd, &obj, 1);
+ helper->fb =
+ exynos_drm_framebuffer_init(dev, &mode_cmd, &exynos_gem, 1);
if (IS_ERR(helper->fb)) {
DRM_ERROR("failed to create drm framebuffer.\n");
ret = PTR_ERR(helper->fb);
goto err_destroy_gem;
}
- ret = exynos_drm_fbdev_update(helper, sizes, obj);
+ ret = exynos_drm_fbdev_update(helper, sizes, exynos_gem);
if (ret < 0)
goto err_destroy_framebuffer;
err_destroy_framebuffer:
drm_framebuffer_cleanup(helper->fb);
err_destroy_gem:
- exynos_drm_gem_destroy(obj);
+ exynos_drm_gem_destroy(exynos_gem);
/*
* if failed, all resources allocated above would be released by
struct drm_fb_helper *fb_helper)
{
struct exynos_drm_fbdev *exynos_fbd = to_exynos_fbdev(fb_helper);
- struct exynos_drm_gem_obj *obj = exynos_fbd->obj;
+ struct exynos_drm_gem *exynos_gem = exynos_fbd->exynos_gem;
struct drm_framebuffer *fb;
- if (obj->kvaddr)
- vunmap(obj->kvaddr);
+ if (exynos_gem->kvaddr)
+ vunmap(exynos_gem->kvaddr);
/* release drm framebuffer and real buffer */
if (fb_helper->fb && fb_helper->fb->funcs) {
EXYNOS_MSCTRL_C_INT_IN_2PLANE);
break;
default:
- dev_err(ippdrv->dev, "inavlid source yuv order 0x%x.\n", fmt);
+ dev_err(ippdrv->dev, "invalid source yuv order 0x%x.\n", fmt);
return -EINVAL;
}
cfg |= EXYNOS_MSCTRL_INFORMAT_YCBCR420;
break;
default:
- dev_err(ippdrv->dev, "inavlid source format 0x%x.\n", fmt);
+ dev_err(ippdrv->dev, "invalid source format 0x%x.\n", fmt);
return -EINVAL;
}
cfg1 &= ~EXYNOS_MSCTRL_FLIP_Y_MIRROR;
break;
default:
- dev_err(ippdrv->dev, "inavlid degree value %d.\n", degree);
+ dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
return -EINVAL;
}
property->prop_id, buf_id, buf_type);
if (buf_id > FIMC_MAX_SRC) {
- dev_info(ippdrv->dev, "inavlid buf_id %d.\n", buf_id);
+ dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
return -ENOMEM;
}
cfg |= EXYNOS_CIOCTRL_YCBCR_2PLANE;
break;
default:
- dev_err(ippdrv->dev, "inavlid target yuv order 0x%x.\n", fmt);
+ dev_err(ippdrv->dev, "invalid target yuv order 0x%x.\n", fmt);
return -EINVAL;
}
cfg |= EXYNOS_CITRGFMT_OUTFORMAT_YCBCR420;
break;
default:
- dev_err(ippdrv->dev, "inavlid target format 0x%x.\n",
+ dev_err(ippdrv->dev, "invalid target format 0x%x.\n",
fmt);
return -EINVAL;
}
cfg &= ~EXYNOS_CITRGFMT_FLIP_Y_MIRROR;
break;
default:
- dev_err(ippdrv->dev, "inavlid degree value %d.\n", degree);
+ dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
return -EINVAL;
}
property->prop_id, buf_id, buf_type);
if (buf_id > FIMC_MAX_DST) {
- dev_info(ippdrv->dev, "inavlid buf_id %d.\n", buf_id);
+ dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
return -ENOMEM;
}
.set_addr = fimc_dst_set_addr,
};
-static int fimc_clk_ctrl(struct fimc_context *ctx, bool enable)
-{
- DRM_DEBUG_KMS("enable[%d]\n", enable);
-
- if (enable) {
- clk_prepare_enable(ctx->clocks[FIMC_CLK_GATE]);
- clk_prepare_enable(ctx->clocks[FIMC_CLK_WB_A]);
- ctx->suspended = false;
- } else {
- clk_disable_unprepare(ctx->clocks[FIMC_CLK_GATE]);
- clk_disable_unprepare(ctx->clocks[FIMC_CLK_WB_A]);
- ctx->suspended = true;
- }
-
- return 0;
-}
-
static irqreturn_t fimc_irq_handler(int irq, void *dev_id)
{
struct fimc_context *ctx = dev_id;
return 0;
}
+#ifdef CONFIG_PM
+static int fimc_clk_ctrl(struct fimc_context *ctx, bool enable)
+{
+ DRM_DEBUG_KMS("enable[%d]\n", enable);
+
+ if (enable) {
+ clk_prepare_enable(ctx->clocks[FIMC_CLK_GATE]);
+ clk_prepare_enable(ctx->clocks[FIMC_CLK_WB_A]);
+ ctx->suspended = false;
+ } else {
+ clk_disable_unprepare(ctx->clocks[FIMC_CLK_GATE]);
+ clk_disable_unprepare(ctx->clocks[FIMC_CLK_WB_A]);
+ ctx->suspended = true;
+ }
+
+ return 0;
+}
+
#ifdef CONFIG_PM_SLEEP
static int fimc_suspend(struct device *dev)
{
}
#endif
-#ifdef CONFIG_PM
static int fimc_runtime_suspend(struct device *dev)
{
struct fimc_context *ctx = get_fimc_context(dev);
* CPU Interface.
*/
-#define FIMD_DEFAULT_FRAMERATE 60
#define MIN_FB_WIDTH_FOR_16WORD_BURST 128
/* position control register for hardware window 0, 2 ~ 4.*/
/* FIMD has totally five hardware windows. */
#define WINDOWS_NR 5
+#define CURSOR_WIN 4
struct fimd_driver_data {
unsigned int timing_base;
struct clk *lcd_clk;
void __iomem *regs;
struct regmap *sysreg;
- unsigned int default_win;
unsigned long irq_flags;
u32 vidcon0;
u32 vidcon1;
return (clkdiv < 0x100) ? clkdiv : 0xff;
}
-static bool fimd_mode_fixup(struct exynos_drm_crtc *crtc,
- const struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
-{
- if (adjusted_mode->vrefresh == 0)
- adjusted_mode->vrefresh = FIMD_DEFAULT_FRAMERATE;
-
- return true;
-}
-
static void fimd_commit(struct exynos_drm_crtc *crtc)
{
struct fimd_context *ctx = crtc->ctx;
return;
val = enable ? DP_MIE_CLK_DP_ENABLE : DP_MIE_CLK_DISABLE;
- writel(DP_MIE_CLK_DP_ENABLE, ctx->regs + DP_MIE_CLKCON);
+ writel(val, ctx->regs + DP_MIE_CLKCON);
}
static const struct exynos_drm_crtc_ops fimd_crtc_ops = {
.enable = fimd_enable,
.disable = fimd_disable,
- .mode_fixup = fimd_mode_fixup,
.commit = fimd_commit,
.enable_vblank = fimd_enable_vblank,
.disable_vblank = fimd_disable_vblank,
ctx->pipe = priv->pipe++;
for (zpos = 0; zpos < WINDOWS_NR; zpos++) {
- type = (zpos == ctx->default_win) ? DRM_PLANE_TYPE_PRIMARY :
- DRM_PLANE_TYPE_OVERLAY;
+ type = exynos_plane_get_type(zpos, CURSOR_WIN);
ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
1 << ctx->pipe, type, fimd_formats,
ARRAY_SIZE(fimd_formats), zpos);
return ret;
}
- exynos_plane = &ctx->planes[ctx->default_win];
+ exynos_plane = &ctx->planes[DEFAULT_WIN];
ctx->crtc = exynos_drm_crtc_create(drm_dev, &exynos_plane->base,
ctx->pipe, EXYNOS_DISPLAY_TYPE_LCD,
&fimd_crtc_ops, ctx);
return 0;
}
-EXPORT_SYMBOL_GPL(exynos_g2d_get_ver_ioctl);
int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
struct drm_file *file)
g2d_put_cmdlist(g2d, node);
return ret;
}
-EXPORT_SYMBOL_GPL(exynos_g2d_set_cmdlist_ioctl);
int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
struct drm_file *file)
out:
return 0;
}
-EXPORT_SYMBOL_GPL(exynos_g2d_exec_ioctl);
static int g2d_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
{
#include "exynos_drm_gem.h"
#include "exynos_drm_iommu.h"
-static int exynos_drm_alloc_buf(struct exynos_drm_gem_obj *obj)
+static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem)
{
- struct drm_device *dev = obj->base.dev;
+ struct drm_device *dev = exynos_gem->base.dev;
enum dma_attr attr;
unsigned int nr_pages;
+ struct sg_table sgt;
+ int ret = -ENOMEM;
- if (obj->dma_addr) {
+ if (exynos_gem->dma_addr) {
DRM_DEBUG_KMS("already allocated.\n");
return 0;
}
- init_dma_attrs(&obj->dma_attrs);
+ init_dma_attrs(&exynos_gem->dma_attrs);
/*
* if EXYNOS_BO_CONTIG, fully physically contiguous memory
* region will be allocated else physically contiguous
* as possible.
*/
- if (!(obj->flags & EXYNOS_BO_NONCONTIG))
- dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, &obj->dma_attrs);
+ if (!(exynos_gem->flags & EXYNOS_BO_NONCONTIG))
+ dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, &exynos_gem->dma_attrs);
/*
* if EXYNOS_BO_WC or EXYNOS_BO_NONCACHABLE, writecombine mapping
* else cachable mapping.
*/
- if (obj->flags & EXYNOS_BO_WC || !(obj->flags & EXYNOS_BO_CACHABLE))
+ if (exynos_gem->flags & EXYNOS_BO_WC ||
+ !(exynos_gem->flags & EXYNOS_BO_CACHABLE))
attr = DMA_ATTR_WRITE_COMBINE;
else
attr = DMA_ATTR_NON_CONSISTENT;
- dma_set_attr(attr, &obj->dma_attrs);
- dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &obj->dma_attrs);
-
- nr_pages = obj->size >> PAGE_SHIFT;
-
- if (!is_drm_iommu_supported(dev)) {
- dma_addr_t start_addr;
- unsigned int i = 0;
-
- obj->pages = drm_calloc_large(nr_pages, sizeof(struct page *));
- if (!obj->pages) {
- DRM_ERROR("failed to allocate pages.\n");
- return -ENOMEM;
- }
-
- obj->cookie = dma_alloc_attrs(dev->dev,
- obj->size,
- &obj->dma_addr, GFP_KERNEL,
- &obj->dma_attrs);
- if (!obj->cookie) {
- DRM_ERROR("failed to allocate buffer.\n");
- drm_free_large(obj->pages);
- return -ENOMEM;
- }
-
- start_addr = obj->dma_addr;
- while (i < nr_pages) {
- obj->pages[i] = phys_to_page(start_addr);
- start_addr += PAGE_SIZE;
- i++;
- }
- } else {
- obj->pages = dma_alloc_attrs(dev->dev, obj->size,
- &obj->dma_addr, GFP_KERNEL,
- &obj->dma_attrs);
- if (!obj->pages) {
- DRM_ERROR("failed to allocate buffer.\n");
- return -ENOMEM;
- }
+ dma_set_attr(attr, &exynos_gem->dma_attrs);
+ dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &exynos_gem->dma_attrs);
+
+ nr_pages = exynos_gem->size >> PAGE_SHIFT;
+
+ exynos_gem->pages = drm_calloc_large(nr_pages, sizeof(struct page *));
+ if (!exynos_gem->pages) {
+ DRM_ERROR("failed to allocate pages.\n");
+ return -ENOMEM;
+ }
+
+ exynos_gem->cookie = dma_alloc_attrs(dev->dev, exynos_gem->size,
+ &exynos_gem->dma_addr, GFP_KERNEL,
+ &exynos_gem->dma_attrs);
+ if (!exynos_gem->cookie) {
+ DRM_ERROR("failed to allocate buffer.\n");
+ goto err_free;
+ }
+
+ ret = dma_get_sgtable_attrs(dev->dev, &sgt, exynos_gem->cookie,
+ exynos_gem->dma_addr, exynos_gem->size,
+ &exynos_gem->dma_attrs);
+ if (ret < 0) {
+ DRM_ERROR("failed to get sgtable.\n");
+ goto err_dma_free;
+ }
+
+ if (drm_prime_sg_to_page_addr_arrays(&sgt, exynos_gem->pages, NULL,
+ nr_pages)) {
+ DRM_ERROR("invalid sgtable.\n");
+ ret = -EINVAL;
+ goto err_sgt_free;
}
+ sg_free_table(&sgt);
+
DRM_DEBUG_KMS("dma_addr(0x%lx), size(0x%lx)\n",
- (unsigned long)obj->dma_addr,
- obj->size);
+ (unsigned long)exynos_gem->dma_addr, exynos_gem->size);
return 0;
+
+err_sgt_free:
+ sg_free_table(&sgt);
+err_dma_free:
+ dma_free_attrs(dev->dev, exynos_gem->size, exynos_gem->cookie,
+ exynos_gem->dma_addr, &exynos_gem->dma_attrs);
+err_free:
+ drm_free_large(exynos_gem->pages);
+
+ return ret;
}
-static void exynos_drm_free_buf(struct exynos_drm_gem_obj *obj)
+static void exynos_drm_free_buf(struct exynos_drm_gem *exynos_gem)
{
- struct drm_device *dev = obj->base.dev;
+ struct drm_device *dev = exynos_gem->base.dev;
- if (!obj->dma_addr) {
+ if (!exynos_gem->dma_addr) {
DRM_DEBUG_KMS("dma_addr is invalid.\n");
return;
}
DRM_DEBUG_KMS("dma_addr(0x%lx), size(0x%lx)\n",
- (unsigned long)obj->dma_addr, obj->size);
+ (unsigned long)exynos_gem->dma_addr, exynos_gem->size);
- if (!is_drm_iommu_supported(dev)) {
- dma_free_attrs(dev->dev, obj->size, obj->cookie,
- (dma_addr_t)obj->dma_addr, &obj->dma_attrs);
- drm_free_large(obj->pages);
- } else
- dma_free_attrs(dev->dev, obj->size, obj->pages,
- (dma_addr_t)obj->dma_addr, &obj->dma_attrs);
+ dma_free_attrs(dev->dev, exynos_gem->size, exynos_gem->cookie,
+ (dma_addr_t)exynos_gem->dma_addr,
+ &exynos_gem->dma_attrs);
- obj->dma_addr = (dma_addr_t)NULL;
+ drm_free_large(exynos_gem->pages);
}
static int exynos_drm_gem_handle_create(struct drm_gem_object *obj,
return 0;
}
-void exynos_drm_gem_destroy(struct exynos_drm_gem_obj *exynos_gem_obj)
+void exynos_drm_gem_destroy(struct exynos_drm_gem *exynos_gem)
{
- struct drm_gem_object *obj = &exynos_gem_obj->base;
+ struct drm_gem_object *obj = &exynos_gem->base;
DRM_DEBUG_KMS("handle count = %d\n", obj->handle_count);
* once dmabuf's refcount becomes 0.
*/
if (obj->import_attach)
- goto out;
-
- exynos_drm_free_buf(exynos_gem_obj);
-
-out:
- drm_gem_free_mmap_offset(obj);
+ drm_prime_gem_destroy(obj, exynos_gem->sgt);
+ else
+ exynos_drm_free_buf(exynos_gem);
/* release file pointer to gem object. */
drm_gem_object_release(obj);
- kfree(exynos_gem_obj);
- exynos_gem_obj = NULL;
+ kfree(exynos_gem);
}
unsigned long exynos_drm_gem_get_size(struct drm_device *dev,
unsigned int gem_handle,
struct drm_file *file_priv)
{
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
struct drm_gem_object *obj;
obj = drm_gem_object_lookup(dev, file_priv, gem_handle);
return 0;
}
- exynos_gem_obj = to_exynos_gem_obj(obj);
+ exynos_gem = to_exynos_gem(obj);
drm_gem_object_unreference_unlocked(obj);
- return exynos_gem_obj->size;
+ return exynos_gem->size;
}
-
-struct exynos_drm_gem_obj *exynos_drm_gem_init(struct drm_device *dev,
- unsigned long size)
+static struct exynos_drm_gem *exynos_drm_gem_init(struct drm_device *dev,
+ unsigned long size)
{
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
struct drm_gem_object *obj;
int ret;
- exynos_gem_obj = kzalloc(sizeof(*exynos_gem_obj), GFP_KERNEL);
- if (!exynos_gem_obj)
+ exynos_gem = kzalloc(sizeof(*exynos_gem), GFP_KERNEL);
+ if (!exynos_gem)
return ERR_PTR(-ENOMEM);
- exynos_gem_obj->size = size;
- obj = &exynos_gem_obj->base;
+ exynos_gem->size = size;
+ obj = &exynos_gem->base;
ret = drm_gem_object_init(dev, obj, size);
if (ret < 0) {
DRM_ERROR("failed to initialize gem object\n");
- kfree(exynos_gem_obj);
+ kfree(exynos_gem);
+ return ERR_PTR(ret);
+ }
+
+ ret = drm_gem_create_mmap_offset(obj);
+ if (ret < 0) {
+ drm_gem_object_release(obj);
+ kfree(exynos_gem);
return ERR_PTR(ret);
}
DRM_DEBUG_KMS("created file object = 0x%x\n", (unsigned int)obj->filp);
- return exynos_gem_obj;
+ return exynos_gem;
}
-struct exynos_drm_gem_obj *exynos_drm_gem_create(struct drm_device *dev,
- unsigned int flags,
- unsigned long size)
+struct exynos_drm_gem *exynos_drm_gem_create(struct drm_device *dev,
+ unsigned int flags,
+ unsigned long size)
{
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
int ret;
if (flags & ~(EXYNOS_BO_MASK)) {
size = roundup(size, PAGE_SIZE);
- exynos_gem_obj = exynos_drm_gem_init(dev, size);
- if (IS_ERR(exynos_gem_obj))
- return exynos_gem_obj;
+ exynos_gem = exynos_drm_gem_init(dev, size);
+ if (IS_ERR(exynos_gem))
+ return exynos_gem;
/* set memory type and cache attribute from user side. */
- exynos_gem_obj->flags = flags;
+ exynos_gem->flags = flags;
- ret = exynos_drm_alloc_buf(exynos_gem_obj);
+ ret = exynos_drm_alloc_buf(exynos_gem);
if (ret < 0) {
- drm_gem_object_release(&exynos_gem_obj->base);
- kfree(exynos_gem_obj);
+ drm_gem_object_release(&exynos_gem->base);
+ kfree(exynos_gem);
return ERR_PTR(ret);
}
- return exynos_gem_obj;
+ return exynos_gem;
}
int exynos_drm_gem_create_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_exynos_gem_create *args = data;
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
int ret;
- exynos_gem_obj = exynos_drm_gem_create(dev, args->flags, args->size);
- if (IS_ERR(exynos_gem_obj))
- return PTR_ERR(exynos_gem_obj);
+ exynos_gem = exynos_drm_gem_create(dev, args->flags, args->size);
+ if (IS_ERR(exynos_gem))
+ return PTR_ERR(exynos_gem);
- ret = exynos_drm_gem_handle_create(&exynos_gem_obj->base, file_priv,
- &args->handle);
+ ret = exynos_drm_gem_handle_create(&exynos_gem->base, file_priv,
+ &args->handle);
if (ret) {
- exynos_drm_gem_destroy(exynos_gem_obj);
+ exynos_drm_gem_destroy(exynos_gem);
return ret;
}
unsigned int gem_handle,
struct drm_file *filp)
{
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
struct drm_gem_object *obj;
obj = drm_gem_object_lookup(dev, filp, gem_handle);
return ERR_PTR(-EINVAL);
}
- exynos_gem_obj = to_exynos_gem_obj(obj);
+ exynos_gem = to_exynos_gem(obj);
- return &exynos_gem_obj->dma_addr;
+ return &exynos_gem->dma_addr;
}
void exynos_drm_gem_put_dma_addr(struct drm_device *dev,
drm_gem_object_unreference_unlocked(obj);
}
-int exynos_drm_gem_mmap_buffer(struct exynos_drm_gem_obj *exynos_gem_obj,
+static int exynos_drm_gem_mmap_buffer(struct exynos_drm_gem *exynos_gem,
struct vm_area_struct *vma)
{
- struct drm_device *drm_dev = exynos_gem_obj->base.dev;
+ struct drm_device *drm_dev = exynos_gem->base.dev;
unsigned long vm_size;
int ret;
vm_size = vma->vm_end - vma->vm_start;
/* check if user-requested size is valid. */
- if (vm_size > exynos_gem_obj->size)
+ if (vm_size > exynos_gem->size)
return -EINVAL;
- ret = dma_mmap_attrs(drm_dev->dev, vma, exynos_gem_obj->pages,
- exynos_gem_obj->dma_addr, exynos_gem_obj->size,
- &exynos_gem_obj->dma_attrs);
+ ret = dma_mmap_attrs(drm_dev->dev, vma, exynos_gem->pages,
+ exynos_gem->dma_addr, exynos_gem->size,
+ &exynos_gem->dma_attrs);
if (ret < 0) {
DRM_ERROR("failed to mmap.\n");
return ret;
int exynos_drm_gem_get_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
-{ struct exynos_drm_gem_obj *exynos_gem_obj;
+{
+ struct exynos_drm_gem *exynos_gem;
struct drm_exynos_gem_info *args = data;
struct drm_gem_object *obj;
return -EINVAL;
}
- exynos_gem_obj = to_exynos_gem_obj(obj);
+ exynos_gem = to_exynos_gem(obj);
- args->flags = exynos_gem_obj->flags;
- args->size = exynos_gem_obj->size;
+ args->flags = exynos_gem->flags;
+ args->size = exynos_gem->size;
drm_gem_object_unreference(obj);
mutex_unlock(&dev->struct_mutex);
void exynos_drm_gem_free_object(struct drm_gem_object *obj)
{
- exynos_drm_gem_destroy(to_exynos_gem_obj(obj));
+ exynos_drm_gem_destroy(to_exynos_gem(obj));
}
int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
struct drm_device *dev,
struct drm_mode_create_dumb *args)
{
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
+ unsigned int flags;
int ret;
/*
args->pitch = args->width * ((args->bpp + 7) / 8);
args->size = args->pitch * args->height;
- if (is_drm_iommu_supported(dev)) {
- exynos_gem_obj = exynos_drm_gem_create(dev,
- EXYNOS_BO_NONCONTIG | EXYNOS_BO_WC,
- args->size);
- } else {
- exynos_gem_obj = exynos_drm_gem_create(dev,
- EXYNOS_BO_CONTIG | EXYNOS_BO_WC,
- args->size);
- }
+ if (is_drm_iommu_supported(dev))
+ flags = EXYNOS_BO_NONCONTIG | EXYNOS_BO_WC;
+ else
+ flags = EXYNOS_BO_CONTIG | EXYNOS_BO_WC;
- if (IS_ERR(exynos_gem_obj)) {
+ exynos_gem = exynos_drm_gem_create(dev, flags, args->size);
+ if (IS_ERR(exynos_gem)) {
dev_warn(dev->dev, "FB allocation failed.\n");
- return PTR_ERR(exynos_gem_obj);
+ return PTR_ERR(exynos_gem);
}
- ret = exynos_drm_gem_handle_create(&exynos_gem_obj->base, file_priv,
- &args->handle);
+ ret = exynos_drm_gem_handle_create(&exynos_gem->base, file_priv,
+ &args->handle);
if (ret) {
- exynos_drm_gem_destroy(exynos_gem_obj);
+ exynos_drm_gem_destroy(exynos_gem);
return ret;
}
goto unlock;
}
- ret = drm_gem_create_mmap_offset(obj);
- if (ret)
- goto out;
-
*offset = drm_vma_node_offset_addr(&obj->vma_node);
DRM_DEBUG_KMS("offset = 0x%lx\n", (unsigned long)*offset);
-out:
drm_gem_object_unreference(obj);
unlock:
mutex_unlock(&dev->struct_mutex);
int exynos_drm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct drm_gem_object *obj = vma->vm_private_data;
- struct exynos_drm_gem_obj *exynos_gem_obj = to_exynos_gem_obj(obj);
+ struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
unsigned long pfn;
pgoff_t page_offset;
int ret;
page_offset = ((unsigned long)vmf->virtual_address -
vma->vm_start) >> PAGE_SHIFT;
- if (page_offset >= (exynos_gem_obj->size >> PAGE_SHIFT)) {
+ if (page_offset >= (exynos_gem->size >> PAGE_SHIFT)) {
DRM_ERROR("invalid page offset\n");
ret = -EINVAL;
goto out;
}
- pfn = page_to_pfn(exynos_gem_obj->pages[page_offset]);
+ pfn = page_to_pfn(exynos_gem->pages[page_offset]);
ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);
out:
int exynos_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
{
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
struct drm_gem_object *obj;
int ret;
}
obj = vma->vm_private_data;
- exynos_gem_obj = to_exynos_gem_obj(obj);
+ exynos_gem = to_exynos_gem(obj);
- DRM_DEBUG_KMS("flags = 0x%x\n", exynos_gem_obj->flags);
+ DRM_DEBUG_KMS("flags = 0x%x\n", exynos_gem->flags);
/* non-cachable as default. */
- if (exynos_gem_obj->flags & EXYNOS_BO_CACHABLE)
+ if (exynos_gem->flags & EXYNOS_BO_CACHABLE)
vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
- else if (exynos_gem_obj->flags & EXYNOS_BO_WC)
+ else if (exynos_gem->flags & EXYNOS_BO_WC)
vma->vm_page_prot =
pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
else
vma->vm_page_prot =
pgprot_noncached(vm_get_page_prot(vma->vm_flags));
- ret = exynos_drm_gem_mmap_buffer(exynos_gem_obj, vma);
+ ret = exynos_drm_gem_mmap_buffer(exynos_gem, vma);
if (ret)
goto err_close_vm;
err_close_vm:
drm_gem_vm_close(vma);
- drm_gem_free_mmap_offset(obj);
return ret;
}
/* low-level interface prime helpers */
struct sg_table *exynos_drm_gem_prime_get_sg_table(struct drm_gem_object *obj)
{
- struct exynos_drm_gem_obj *exynos_gem_obj = to_exynos_gem_obj(obj);
+ struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
int npages;
- npages = exynos_gem_obj->size >> PAGE_SHIFT;
+ npages = exynos_gem->size >> PAGE_SHIFT;
- return drm_prime_pages_to_sg(exynos_gem_obj->pages, npages);
+ return drm_prime_pages_to_sg(exynos_gem->pages, npages);
}
struct drm_gem_object *
struct dma_buf_attachment *attach,
struct sg_table *sgt)
{
- struct exynos_drm_gem_obj *exynos_gem_obj;
+ struct exynos_drm_gem *exynos_gem;
int npages;
int ret;
- exynos_gem_obj = exynos_drm_gem_init(dev, attach->dmabuf->size);
- if (IS_ERR(exynos_gem_obj)) {
- ret = PTR_ERR(exynos_gem_obj);
+ exynos_gem = exynos_drm_gem_init(dev, attach->dmabuf->size);
+ if (IS_ERR(exynos_gem)) {
+ ret = PTR_ERR(exynos_gem);
return ERR_PTR(ret);
}
- exynos_gem_obj->dma_addr = sg_dma_address(sgt->sgl);
+ exynos_gem->dma_addr = sg_dma_address(sgt->sgl);
- npages = exynos_gem_obj->size >> PAGE_SHIFT;
- exynos_gem_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
- if (!exynos_gem_obj->pages) {
+ npages = exynos_gem->size >> PAGE_SHIFT;
+ exynos_gem->pages = drm_malloc_ab(npages, sizeof(struct page *));
+ if (!exynos_gem->pages) {
ret = -ENOMEM;
goto err;
}
- ret = drm_prime_sg_to_page_addr_arrays(sgt, exynos_gem_obj->pages, NULL,
- npages);
+ ret = drm_prime_sg_to_page_addr_arrays(sgt, exynos_gem->pages, NULL,
+ npages);
if (ret < 0)
goto err_free_large;
+ exynos_gem->sgt = sgt;
+
if (sgt->nents == 1) {
/* always physically continuous memory if sgt->nents is 1. */
- exynos_gem_obj->flags |= EXYNOS_BO_CONTIG;
+ exynos_gem->flags |= EXYNOS_BO_CONTIG;
} else {
/*
* this case could be CONTIG or NONCONTIG type but for now
* TODO. we have to find a way that exporter can notify
* the type of its own buffer to importer.
*/
- exynos_gem_obj->flags |= EXYNOS_BO_NONCONTIG;
+ exynos_gem->flags |= EXYNOS_BO_NONCONTIG;
}
- return &exynos_gem_obj->base;
+ return &exynos_gem->base;
err_free_large:
- drm_free_large(exynos_gem_obj->pages);
+ drm_free_large(exynos_gem->pages);
err:
- drm_gem_object_release(&exynos_gem_obj->base);
- kfree(exynos_gem_obj);
+ drm_gem_object_release(&exynos_gem->base);
+ kfree(exynos_gem);
return ERR_PTR(ret);
}
#include <drm/drm_gem.h>
-#define to_exynos_gem_obj(x) container_of(x,\
- struct exynos_drm_gem_obj, base)
+#define to_exynos_gem(x) container_of(x, struct exynos_drm_gem, base)
#define IS_NONCONTIG_BUFFER(f) (f & EXYNOS_BO_NONCONTIG)
* - this address could be physical address without IOMMU and
* device address with IOMMU.
* @pages: Array of backing pages.
+ * @sgt: Imported sg_table.
*
* P.S. this object would be transferred to user as kms_bo.handle so
* user can access the buffer through kms_bo.handle.
*/
-struct exynos_drm_gem_obj {
+struct exynos_drm_gem {
struct drm_gem_object base;
unsigned int flags;
unsigned long size;
dma_addr_t dma_addr;
struct dma_attrs dma_attrs;
struct page **pages;
+ struct sg_table *sgt;
};
struct page **exynos_gem_get_pages(struct drm_gem_object *obj, gfp_t gfpmask);
/* destroy a buffer with gem object */
-void exynos_drm_gem_destroy(struct exynos_drm_gem_obj *exynos_gem_obj);
-
-/* create a private gem object and initialize it. */
-struct exynos_drm_gem_obj *exynos_drm_gem_init(struct drm_device *dev,
- unsigned long size);
+void exynos_drm_gem_destroy(struct exynos_drm_gem *exynos_gem);
/* create a new buffer with gem object */
-struct exynos_drm_gem_obj *exynos_drm_gem_create(struct drm_device *dev,
- unsigned int flags,
- unsigned long size);
+struct exynos_drm_gem *exynos_drm_gem_create(struct drm_device *dev,
+ unsigned int flags,
+ unsigned long size);
/*
* request gem object creation and buffer allocation as the size
struct drm_file *file_priv);
/* free gem object. */
-void exynos_drm_gem_free_object(struct drm_gem_object *gem_obj);
+void exynos_drm_gem_free_object(struct drm_gem_object *obj);
/* create memory region for drm framebuffer. */
int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
GSC_IN_YUV420_2P);
break;
default:
- dev_err(ippdrv->dev, "inavlid target yuv order 0x%x.\n", fmt);
+ dev_err(ippdrv->dev, "invalid target yuv order 0x%x.\n", fmt);
return -EINVAL;
}
cfg &= ~GSC_IN_ROT_YFLIP;
break;
default:
- dev_err(ippdrv->dev, "inavlid degree value %d.\n", degree);
+ dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
return -EINVAL;
}
property->prop_id, buf_id, buf_type);
if (buf_id > GSC_MAX_SRC) {
- dev_info(ippdrv->dev, "inavlid buf_id %d.\n", buf_id);
+ dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
return -EINVAL;
}
GSC_OUT_YUV420_2P);
break;
default:
- dev_err(ippdrv->dev, "inavlid target yuv order 0x%x.\n", fmt);
+ dev_err(ippdrv->dev, "invalid target yuv order 0x%x.\n", fmt);
return -EINVAL;
}
cfg &= ~GSC_IN_ROT_YFLIP;
break;
default:
- dev_err(ippdrv->dev, "inavlid degree value %d.\n", degree);
+ dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
return -EINVAL;
}
property->prop_id, buf_id, buf_type);
if (buf_id > GSC_MAX_DST) {
- dev_info(ippdrv->dev, "inavlid buf_id %d.\n", buf_id);
+ dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
return -EINVAL;
}
if (!mapping || !mapping->domain)
return;
- iommu_detach_device(mapping->domain, subdrv_dev);
- drm_release_iommu_mapping(drm_dev);
+ arm_iommu_detach_device(subdrv_dev);
}
nr = drm_format_num_planes(state->fb->pixel_format);
for (i = 0; i < nr; i++) {
- struct exynos_drm_gem_obj *obj =
- exynos_drm_fb_gem_obj(state->fb, i);
-
- if (!obj) {
+ struct exynos_drm_gem *exynos_gem =
+ exynos_drm_fb_gem(state->fb, i);
+ if (!exynos_gem) {
DRM_DEBUG_KMS("gem object is null\n");
return -EFAULT;
}
- exynos_plane->dma_addr[i] = obj->dma_addr +
+ exynos_plane->dma_addr[i] = exynos_gem->dma_addr +
state->fb->offsets[i];
DRM_DEBUG_KMS("buffer: %d, dma_addr = 0x%lx\n",
drm_object_attach_property(&plane->base, prop, zpos);
}
+enum drm_plane_type exynos_plane_get_type(unsigned int zpos,
+ unsigned int cursor_win)
+{
+ if (zpos == DEFAULT_WIN)
+ return DRM_PLANE_TYPE_PRIMARY;
+ else if (zpos == cursor_win)
+ return DRM_PLANE_TYPE_CURSOR;
+ else
+ return DRM_PLANE_TYPE_OVERLAY;
+}
+
int exynos_plane_init(struct drm_device *dev,
struct exynos_drm_plane *exynos_plane,
unsigned long possible_crtcs, enum drm_plane_type type,
*
*/
+enum drm_plane_type exynos_plane_get_type(unsigned int zpos,
+ unsigned int cursor_win);
int exynos_plane_init(struct drm_device *dev,
struct exynos_drm_plane *exynos_plane,
unsigned long possible_crtcs, enum drm_plane_type type,
return 0;
}
+#ifdef CONFIG_PM
static int rotator_clk_crtl(struct rot_context *rot, bool enable)
{
if (enable) {
}
#endif
-#ifdef CONFIG_PM
static int rotator_runtime_suspend(struct device *dev)
{
struct rot_context *rot = dev_get_drvdata(dev);
/* vidi has totally three virtual windows. */
#define WINDOWS_NR 3
+#define CURSOR_WIN 2
#define ctx_from_connector(c) container_of(c, struct vidi_context, \
connector)
struct exynos_drm_plane planes[WINDOWS_NR];
struct edid *raw_edid;
unsigned int clkdiv;
- unsigned int default_win;
unsigned long irq_flags;
unsigned int connected;
bool vblank_on;
vidi_ctx_initialize(ctx, drm_dev);
for (zpos = 0; zpos < WINDOWS_NR; zpos++) {
- type = (zpos == ctx->default_win) ? DRM_PLANE_TYPE_PRIMARY :
- DRM_PLANE_TYPE_OVERLAY;
+ type = exynos_plane_get_type(zpos, CURSOR_WIN);
ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
1 << ctx->pipe, type, formats,
ARRAY_SIZE(formats), zpos);
return ret;
}
- exynos_plane = &ctx->planes[ctx->default_win];
+ exynos_plane = &ctx->planes[DEFAULT_WIN];
ctx->crtc = exynos_drm_crtc_create(drm_dev, &exynos_plane->base,
ctx->pipe, EXYNOS_DISPLAY_TYPE_VIDI,
&vidi_crtc_ops, ctx);
if (!ctx)
return -ENOMEM;
- ctx->default_win = 0;
ctx->pdev = pdev;
INIT_WORK(&ctx->work, vidi_fake_vblank_handler);
#include <linux/delay.h>
#include <linux/pm_runtime.h>
#include <linux/clk.h>
+#include <linux/gpio/consumer.h>
#include <linux/regulator/consumer.h>
#include <linux/io.h>
#include <linux/of_address.h>
#include <linux/of_device.h>
-#include <linux/of_gpio.h>
#include <linux/hdmi.h>
#include <linux/component.h>
#include <linux/mfd/syscon.h>
#include "exynos_drm_drv.h"
#include "exynos_drm_crtc.h"
-#include "exynos_mixer.h"
-
-#include <linux/gpio.h>
-
-#define ctx_from_connector(c) container_of(c, struct hdmi_context, connector)
#define HOTPLUG_DEBOUNCE_MS 1100
enum hdmi_type {
HDMI_TYPE13,
HDMI_TYPE14,
+ HDMI_TYPE_COUNT
+};
+
+#define HDMI_MAPPED_BASE 0xffff0000
+
+enum hdmi_mapped_regs {
+ HDMI_PHY_STATUS = HDMI_MAPPED_BASE,
+ HDMI_PHY_RSTOUT,
+ HDMI_ACR_CON,
+ HDMI_ACR_MCTS0,
+ HDMI_ACR_CTS0,
+ HDMI_ACR_N0
+};
+
+static const u32 hdmi_reg_map[][HDMI_TYPE_COUNT] = {
+ { HDMI_V13_PHY_STATUS, HDMI_PHY_STATUS_0 },
+ { HDMI_V13_PHY_RSTOUT, HDMI_V14_PHY_RSTOUT },
+ { HDMI_V13_ACR_CON, HDMI_V14_ACR_CON },
+ { HDMI_V13_ACR_MCTS0, HDMI_V14_ACR_MCTS0 },
+ { HDMI_V13_ACR_CTS0, HDMI_V14_ACR_CTS0 },
+ { HDMI_V13_ACR_N0, HDMI_V14_ACR_N0 },
+};
+
+static const char * const supply[] = {
+ "vdd",
+ "vdd_osc",
+ "vdd_pll",
};
struct hdmi_driver_data {
unsigned int is_apb_phy:1;
};
-struct hdmi_resources {
- struct clk *hdmi;
- struct clk *sclk_hdmi;
- struct clk *sclk_pixel;
- struct clk *sclk_hdmiphy;
- struct clk *mout_hdmi;
- struct regulator_bulk_data *regul_bulk;
- struct regulator *reg_hdmi_en;
- int regul_count;
-};
-
struct hdmi_context {
struct drm_encoder encoder;
struct device *dev;
struct drm_device *drm_dev;
struct drm_connector connector;
- bool hpd;
bool powered;
bool dvi_mode;
-
- void __iomem *regs;
- int irq;
struct delayed_work hotplug_work;
-
- struct i2c_adapter *ddc_adpt;
- struct i2c_client *hdmiphy_port;
-
- /* current hdmiphy conf regs */
struct drm_display_mode current_mode;
u8 cea_video_id;
-
- struct hdmi_resources res;
const struct hdmi_driver_data *drv_data;
- int hpd_gpio;
+ void __iomem *regs;
void __iomem *regs_hdmiphy;
-
+ struct i2c_client *hdmiphy_port;
+ struct i2c_adapter *ddc_adpt;
+ struct gpio_desc *hpd_gpio;
+ int irq;
struct regmap *pmureg;
+ struct clk *hdmi;
+ struct clk *sclk_hdmi;
+ struct clk *sclk_pixel;
+ struct clk *sclk_hdmiphy;
+ struct clk *mout_hdmi;
+ struct regulator_bulk_data regul_bulk[ARRAY_SIZE(supply)];
+ struct regulator *reg_hdmi_en;
};
static inline struct hdmi_context *encoder_to_hdmi(struct drm_encoder *e)
return container_of(e, struct hdmi_context, encoder);
}
+static inline struct hdmi_context *connector_to_hdmi(struct drm_connector *c)
+{
+ return container_of(c, struct hdmi_context, connector);
+}
+
struct hdmiphy_config {
int pixel_clock;
u8 conf[32];
0x01, 0x05, 0x00, 0xD8, 0x10, 0x1C, 0x30, 0x40,
0x6B, 0x10, 0x02, 0x51, 0xDF, 0xF2, 0x54, 0x87,
0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
- 0x22, 0x40, 0xE3, 0x26, 0x00, 0x00, 0x00, 0x00,
+ 0x22, 0x40, 0xE3, 0x26, 0x00, 0x00, 0x00, 0x80,
},
},
{
0x01, 0x05, 0x00, 0xD4, 0x10, 0x9C, 0x09, 0x64,
0x6B, 0x10, 0x02, 0x51, 0xDF, 0xF2, 0x54, 0x87,
0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
- 0x22, 0x40, 0xE3, 0x26, 0x00, 0x00, 0x00, 0x00,
+ 0x22, 0x40, 0xE3, 0x26, 0x00, 0x00, 0x00, 0x80,
},
},
{
0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xef, 0x5B,
0x6D, 0x10, 0x01, 0x51, 0xef, 0xF3, 0x54, 0xb9,
0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
- 0x22, 0x40, 0xa5, 0x26, 0x01, 0x00, 0x00, 0x00,
+ 0x22, 0x40, 0xa5, 0x26, 0x01, 0x00, 0x00, 0x80,
},
},
{
0x01, 0x05, 0x00, 0xd8, 0x10, 0x9c, 0xf8, 0x40,
0x6a, 0x10, 0x01, 0x51, 0xff, 0xf1, 0x54, 0xba,
0x84, 0x00, 0x10, 0x38, 0x00, 0x08, 0x10, 0xe0,
- 0x22, 0x40, 0xa4, 0x26, 0x01, 0x00, 0x00, 0x00,
+ 0x22, 0x40, 0xa4, 0x26, 0x01, 0x00, 0x00, 0x80,
},
},
{
0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xf8, 0x40,
0x6A, 0x18, 0x00, 0x51, 0xff, 0xF1, 0x54, 0xba,
0x84, 0x00, 0x10, 0x38, 0x00, 0x08, 0x10, 0xE0,
- 0x22, 0x40, 0xa4, 0x26, 0x02, 0x00, 0x00, 0x00,
+ 0x22, 0x40, 0xa4, 0x26, 0x02, 0x00, 0x00, 0x80,
},
},
};
0x01, 0xd1, 0x2d, 0x72, 0x40, 0x64, 0x12, 0x08,
0x43, 0xa0, 0x0e, 0xd9, 0x45, 0xa0, 0xac, 0x80,
0x08, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
- 0x54, 0xe3, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00,
+ 0x54, 0xe3, 0x24, 0x00, 0x00, 0x00, 0x01, 0x80,
},
},
{
0x01, 0xd1, 0x1f, 0x10, 0x40, 0x40, 0xf8, 0x08,
0x81, 0xa0, 0xba, 0xd8, 0x45, 0xa0, 0xac, 0x80,
0x3c, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
- 0x54, 0xa5, 0x24, 0x01, 0x00, 0x00, 0x01, 0x00,
+ 0x54, 0xa5, 0x24, 0x01, 0x00, 0x00, 0x01, 0x80,
},
},
{
0x01, 0xd1, 0x1f, 0x00, 0x40, 0x40, 0xf8, 0x08,
0x81, 0xa0, 0xba, 0xd8, 0x45, 0xa0, 0xac, 0x80,
0x3c, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
- 0x54, 0x4b, 0x25, 0x03, 0x00, 0x00, 0x01, 0x00,
+ 0x54, 0x4b, 0x25, 0x03, 0x00, 0x00, 0x01, 0x80,
},
},
};
.is_apb_phy = 0,
};
-static struct hdmi_driver_data exynos5_hdmi_driver_data = {
- .type = HDMI_TYPE14,
- .phy_confs = hdmiphy_v13_configs,
- .phy_conf_count = ARRAY_SIZE(hdmiphy_v13_configs),
- .is_apb_phy = 0,
-};
+static inline u32 hdmi_map_reg(struct hdmi_context *hdata, u32 reg_id)
+{
+ if ((reg_id & 0xffff0000) == HDMI_MAPPED_BASE)
+ return hdmi_reg_map[reg_id & 0xffff][hdata->drv_data->type];
+ return reg_id;
+}
static inline u32 hdmi_reg_read(struct hdmi_context *hdata, u32 reg_id)
{
- return readl(hdata->regs + reg_id);
+ return readl(hdata->regs + hdmi_map_reg(hdata, reg_id));
}
static inline void hdmi_reg_writeb(struct hdmi_context *hdata,
u32 reg_id, u8 value)
{
- writeb(value, hdata->regs + reg_id);
+ writel(value, hdata->regs + hdmi_map_reg(hdata, reg_id));
}
static inline void hdmi_reg_writev(struct hdmi_context *hdata, u32 reg_id,
int bytes, u32 val)
{
+ reg_id = hdmi_map_reg(hdata, reg_id);
+
while (--bytes >= 0) {
- writeb(val & 0xff, hdata->regs + reg_id);
+ writel(val & 0xff, hdata->regs + reg_id);
val >>= 8;
reg_id += 4;
}
static inline void hdmi_reg_writemask(struct hdmi_context *hdata,
u32 reg_id, u32 value, u32 mask)
{
- u32 old = readl(hdata->regs + reg_id);
+ u32 old;
+
+ reg_id = hdmi_map_reg(hdata, reg_id);
+ old = readl(hdata->regs + reg_id);
value = (value & mask) | (old & ~mask);
writel(value, hdata->regs + reg_id);
}
-static int hdmiphy_reg_writeb(struct hdmi_context *hdata,
- u32 reg_offset, u8 value)
-{
- if (hdata->hdmiphy_port) {
- u8 buffer[2];
- int ret;
-
- buffer[0] = reg_offset;
- buffer[1] = value;
-
- ret = i2c_master_send(hdata->hdmiphy_port, buffer, 2);
- if (ret == 2)
- return 0;
- return ret;
- } else {
- writeb(value, hdata->regs_hdmiphy + (reg_offset<<2));
- return 0;
- }
-}
-
static int hdmiphy_reg_write_buf(struct hdmi_context *hdata,
u32 reg_offset, const u8 *buf, u32 len)
{
} else {
int i;
for (i = 0; i < len; i++)
- writeb(buf[i], hdata->regs_hdmiphy +
+ writel(buf[i], hdata->regs_hdmiphy +
((reg_offset + i)<<2));
return 0;
}
DUMPREG(HDMI_PHY_STATUS_0);
DUMPREG(HDMI_PHY_STATUS_PLL);
DUMPREG(HDMI_PHY_CON_0);
- DUMPREG(HDMI_PHY_RSTOUT);
+ DUMPREG(HDMI_V14_PHY_RSTOUT);
DUMPREG(HDMI_PHY_VPLL);
DUMPREG(HDMI_PHY_CMU);
DUMPREG(HDMI_CORE_RSTOUT);
static enum drm_connector_status hdmi_detect(struct drm_connector *connector,
bool force)
{
- struct hdmi_context *hdata = ctx_from_connector(connector);
+ struct hdmi_context *hdata = connector_to_hdmi(connector);
- if (gpio_get_value(hdata->hpd_gpio))
+ if (gpiod_get_value(hdata->hpd_gpio))
return connector_status_connected;
return connector_status_disconnected;
static int hdmi_get_modes(struct drm_connector *connector)
{
- struct hdmi_context *hdata = ctx_from_connector(connector);
+ struct hdmi_context *hdata = connector_to_hdmi(connector);
struct edid *edid;
int ret;
static int hdmi_mode_valid(struct drm_connector *connector,
struct drm_display_mode *mode)
{
- struct hdmi_context *hdata = ctx_from_connector(connector);
+ struct hdmi_context *hdata = connector_to_hdmi(connector);
int ret;
DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d clock=%d\n",
(mode->flags & DRM_MODE_FLAG_INTERLACE) ? true :
false, mode->clock * 1000);
- ret = mixer_check_mode(mode);
- if (ret)
- return MODE_BAD;
-
ret = hdmi_find_phy_conf(hdata, mode->clock * 1000);
if (ret < 0)
return MODE_BAD;
static struct drm_encoder *hdmi_best_encoder(struct drm_connector *connector)
{
- struct hdmi_context *hdata = ctx_from_connector(connector);
+ struct hdmi_context *hdata = connector_to_hdmi(connector);
return &hdata->encoder;
}
return true;
}
-static void hdmi_set_acr(u32 freq, u8 *acr)
+static void hdmi_reg_acr(struct hdmi_context *hdata, u32 freq)
{
u32 n, cts;
- switch (freq) {
- case 32000:
- n = 4096;
- cts = 27000;
- break;
- case 44100:
- n = 6272;
- cts = 30000;
- break;
- case 88200:
- n = 12544;
- cts = 30000;
- break;
- case 176400:
- n = 25088;
- cts = 30000;
- break;
- case 48000:
- n = 6144;
- cts = 27000;
- break;
- case 96000:
- n = 12288;
- cts = 27000;
- break;
- case 192000:
- n = 24576;
- cts = 27000;
- break;
- default:
- n = 0;
- cts = 0;
- break;
- }
-
- acr[1] = cts >> 16;
- acr[2] = cts >> 8 & 0xff;
- acr[3] = cts & 0xff;
+ cts = (freq % 9) ? 27000 : 30000;
+ n = 128 * freq / (27000000 / cts);
- acr[4] = n >> 16;
- acr[5] = n >> 8 & 0xff;
- acr[6] = n & 0xff;
-}
-
-static void hdmi_reg_acr(struct hdmi_context *hdata, u8 *acr)
-{
- hdmi_reg_writeb(hdata, HDMI_ACR_N0, acr[6]);
- hdmi_reg_writeb(hdata, HDMI_ACR_N1, acr[5]);
- hdmi_reg_writeb(hdata, HDMI_ACR_N2, acr[4]);
- hdmi_reg_writeb(hdata, HDMI_ACR_MCTS0, acr[3]);
- hdmi_reg_writeb(hdata, HDMI_ACR_MCTS1, acr[2]);
- hdmi_reg_writeb(hdata, HDMI_ACR_MCTS2, acr[1]);
- hdmi_reg_writeb(hdata, HDMI_ACR_CTS0, acr[3]);
- hdmi_reg_writeb(hdata, HDMI_ACR_CTS1, acr[2]);
- hdmi_reg_writeb(hdata, HDMI_ACR_CTS2, acr[1]);
-
- if (hdata->drv_data->type == HDMI_TYPE13)
- hdmi_reg_writeb(hdata, HDMI_V13_ACR_CON, 4);
- else
- hdmi_reg_writeb(hdata, HDMI_ACR_CON, 4);
+ hdmi_reg_writev(hdata, HDMI_ACR_N0, 3, n);
+ hdmi_reg_writev(hdata, HDMI_ACR_MCTS0, 3, cts);
+ hdmi_reg_writev(hdata, HDMI_ACR_CTS0, 3, cts);
+ hdmi_reg_writeb(hdata, HDMI_ACR_CON, 4);
}
static void hdmi_audio_init(struct hdmi_context *hdata)
u32 sample_rate, bits_per_sample;
u32 data_num, bit_ch, sample_frq;
u32 val;
- u8 acr[7];
sample_rate = 44100;
bits_per_sample = 16;
break;
}
- hdmi_set_acr(sample_rate, acr);
- hdmi_reg_acr(hdata, acr);
+ hdmi_reg_acr(hdata, sample_rate);
hdmi_reg_writeb(hdata, HDMI_I2S_MUX_CON, HDMI_I2S_IN_DISABLE
| HDMI_I2S_AUD_I2S | HDMI_I2S_CUV_I2S_ENABLE
}
}
+static void hdmiphy_wait_for_pll(struct hdmi_context *hdata)
+{
+ int tries;
+
+ for (tries = 0; tries < 10; ++tries) {
+ u32 val = hdmi_reg_read(hdata, HDMI_PHY_STATUS);
+
+ if (val & HDMI_PHY_STATUS_READY) {
+ DRM_DEBUG_KMS("PLL stabilized after %d tries\n", tries);
+ return;
+ }
+ usleep_range(10, 20);
+ }
+
+ DRM_ERROR("PLL could not reach steady state\n");
+}
+
static void hdmi_v13_mode_apply(struct hdmi_context *hdata)
{
struct drm_display_mode *m = &hdata->current_mode;
unsigned int val;
- int tries;
hdmi_reg_writev(hdata, HDMI_H_BLANK_0, 2, m->htotal - m->hdisplay);
hdmi_reg_writev(hdata, HDMI_V13_H_V_LINE_0, 3,
hdmi_reg_writev(hdata, HDMI_TG_VSYNC_BOT_HDMI_L, 2, 0x233);
hdmi_reg_writev(hdata, HDMI_TG_FIELD_TOP_HDMI_L, 2, 0x1);
hdmi_reg_writev(hdata, HDMI_TG_FIELD_BOT_HDMI_L, 2, 0x233);
-
- /* waiting for HDMIPHY's PLL to get to steady state */
- for (tries = 100; tries; --tries) {
- u32 val = hdmi_reg_read(hdata, HDMI_V13_PHY_STATUS);
- if (val & HDMI_PHY_STATUS_READY)
- break;
- usleep_range(1000, 2000);
- }
- /* steady state not achieved */
- if (tries == 0) {
- DRM_ERROR("hdmiphy's pll could not reach steady state.\n");
- hdmi_regs_dump(hdata, "timing apply");
- }
-
- clk_disable_unprepare(hdata->res.sclk_hdmi);
- clk_set_parent(hdata->res.mout_hdmi, hdata->res.sclk_hdmiphy);
- clk_prepare_enable(hdata->res.sclk_hdmi);
-
- /* enable HDMI and timing generator */
- hdmi_start(hdata, true);
}
static void hdmi_v14_mode_apply(struct hdmi_context *hdata)
{
struct drm_display_mode *m = &hdata->current_mode;
- int tries;
hdmi_reg_writev(hdata, HDMI_H_BLANK_0, 2, m->htotal - m->hdisplay);
hdmi_reg_writev(hdata, HDMI_V_LINE_0, 2, m->vtotal);
hdmi_reg_writev(hdata, HDMI_TG_VSYNC_TOP_HDMI_L, 2, 0x1);
hdmi_reg_writev(hdata, HDMI_TG_FIELD_TOP_HDMI_L, 2, 0x1);
hdmi_reg_writev(hdata, HDMI_TG_3D, 1, 0x0);
-
- /* waiting for HDMIPHY's PLL to get to steady state */
- for (tries = 100; tries; --tries) {
- u32 val = hdmi_reg_read(hdata, HDMI_PHY_STATUS_0);
- if (val & HDMI_PHY_STATUS_READY)
- break;
- usleep_range(1000, 2000);
- }
- /* steady state not achieved */
- if (tries == 0) {
- DRM_ERROR("hdmiphy's pll could not reach steady state.\n");
- hdmi_regs_dump(hdata, "timing apply");
- }
-
- clk_disable_unprepare(hdata->res.sclk_hdmi);
- clk_set_parent(hdata->res.mout_hdmi, hdata->res.sclk_hdmiphy);
- clk_prepare_enable(hdata->res.sclk_hdmi);
-
- /* enable HDMI and timing generator */
- hdmi_start(hdata, true);
}
static void hdmi_mode_apply(struct hdmi_context *hdata)
hdmi_v13_mode_apply(hdata);
else
hdmi_v14_mode_apply(hdata);
-}
-static void hdmiphy_conf_reset(struct hdmi_context *hdata)
-{
- u32 reg;
+ hdmiphy_wait_for_pll(hdata);
- clk_disable_unprepare(hdata->res.sclk_hdmi);
- clk_set_parent(hdata->res.mout_hdmi, hdata->res.sclk_pixel);
- clk_prepare_enable(hdata->res.sclk_hdmi);
+ clk_set_parent(hdata->mout_hdmi, hdata->sclk_hdmiphy);
- /* operation mode */
- hdmiphy_reg_writeb(hdata, HDMIPHY_MODE_SET_DONE,
- HDMI_PHY_ENABLE_MODE_SET);
+ /* enable HDMI and timing generator */
+ hdmi_start(hdata, true);
+}
- if (hdata->drv_data->type == HDMI_TYPE13)
- reg = HDMI_V13_PHY_RSTOUT;
- else
- reg = HDMI_PHY_RSTOUT;
+static void hdmiphy_conf_reset(struct hdmi_context *hdata)
+{
+ clk_set_parent(hdata->mout_hdmi, hdata->sclk_pixel);
/* reset hdmiphy */
- hdmi_reg_writemask(hdata, reg, ~0, HDMI_PHY_SW_RSTOUT);
+ hdmi_reg_writemask(hdata, HDMI_PHY_RSTOUT, ~0, HDMI_PHY_SW_RSTOUT);
usleep_range(10000, 12000);
- hdmi_reg_writemask(hdata, reg, 0, HDMI_PHY_SW_RSTOUT);
+ hdmi_reg_writemask(hdata, HDMI_PHY_RSTOUT, 0, HDMI_PHY_SW_RSTOUT);
usleep_range(10000, 12000);
}
-static void hdmiphy_poweron(struct hdmi_context *hdata)
-{
- if (hdata->drv_data->type != HDMI_TYPE14)
- return;
-
- DRM_DEBUG_KMS("\n");
-
- /* For PHY Mode Setting */
- hdmiphy_reg_writeb(hdata, HDMIPHY_MODE_SET_DONE,
- HDMI_PHY_ENABLE_MODE_SET);
- /* Phy Power On */
- hdmiphy_reg_writeb(hdata, HDMIPHY_POWER,
- HDMI_PHY_POWER_ON);
- /* For PHY Mode Setting */
- hdmiphy_reg_writeb(hdata, HDMIPHY_MODE_SET_DONE,
- HDMI_PHY_DISABLE_MODE_SET);
- /* PHY SW Reset */
- hdmiphy_conf_reset(hdata);
-}
-
-static void hdmiphy_poweroff(struct hdmi_context *hdata)
-{
- if (hdata->drv_data->type != HDMI_TYPE14)
- return;
-
- DRM_DEBUG_KMS("\n");
-
- /* PHY SW Reset */
- hdmiphy_conf_reset(hdata);
- /* For PHY Mode Setting */
- hdmiphy_reg_writeb(hdata, HDMIPHY_MODE_SET_DONE,
- HDMI_PHY_ENABLE_MODE_SET);
-
- /* PHY Power Off */
- hdmiphy_reg_writeb(hdata, HDMIPHY_POWER,
- HDMI_PHY_POWER_OFF);
-
- /* For PHY Mode Setting */
- hdmiphy_reg_writeb(hdata, HDMIPHY_MODE_SET_DONE,
- HDMI_PHY_DISABLE_MODE_SET);
-}
-
static void hdmiphy_conf_apply(struct hdmi_context *hdata)
{
int ret;
}
usleep_range(10000, 12000);
-
- ret = hdmiphy_reg_writeb(hdata, HDMIPHY_MODE_SET_DONE,
- HDMI_PHY_DISABLE_MODE_SET);
- if (ret) {
- DRM_ERROR("failed to enable hdmiphy\n");
- return;
- }
-
}
static void hdmi_conf_apply(struct hdmi_context *hdata)
static void hdmi_enable(struct drm_encoder *encoder)
{
struct hdmi_context *hdata = encoder_to_hdmi(encoder);
- struct hdmi_resources *res = &hdata->res;
if (hdata->powered)
return;
pm_runtime_get_sync(hdata->dev);
- if (regulator_bulk_enable(res->regul_count, res->regul_bulk))
+ if (regulator_bulk_enable(ARRAY_SIZE(supply), hdata->regul_bulk))
DRM_DEBUG_KMS("failed to enable regulator bulk\n");
/* set pmu hdmiphy control bit to enable hdmiphy */
regmap_update_bits(hdata->pmureg, PMU_HDMI_PHY_CONTROL,
PMU_HDMI_PHY_ENABLE_BIT, 1);
- clk_prepare_enable(res->hdmi);
- clk_prepare_enable(res->sclk_hdmi);
+ clk_prepare_enable(hdata->hdmi);
+ clk_prepare_enable(hdata->sclk_hdmi);
- hdmiphy_poweron(hdata);
hdmi_conf_apply(hdata);
}
static void hdmi_disable(struct drm_encoder *encoder)
{
struct hdmi_context *hdata = encoder_to_hdmi(encoder);
- struct hdmi_resources *res = &hdata->res;
struct drm_crtc *crtc = encoder->crtc;
const struct drm_crtc_helper_funcs *funcs = NULL;
/* HDMI System Disable */
hdmi_reg_writemask(hdata, HDMI_CON_0, 0, HDMI_EN);
- hdmiphy_poweroff(hdata);
-
cancel_delayed_work(&hdata->hotplug_work);
- clk_disable_unprepare(res->sclk_hdmi);
- clk_disable_unprepare(res->hdmi);
+ clk_disable_unprepare(hdata->sclk_hdmi);
+ clk_disable_unprepare(hdata->hdmi);
/* reset pmu hdmiphy control bit to disable hdmiphy */
regmap_update_bits(hdata->pmureg, PMU_HDMI_PHY_CONTROL,
PMU_HDMI_PHY_ENABLE_BIT, 0);
- regulator_bulk_disable(res->regul_count, res->regul_bulk);
+ regulator_bulk_disable(ARRAY_SIZE(supply), hdata->regul_bulk);
pm_runtime_put_sync(hdata->dev);
static int hdmi_resources_init(struct hdmi_context *hdata)
{
struct device *dev = hdata->dev;
- struct hdmi_resources *res = &hdata->res;
- static char *supply[] = {
- "vdd",
- "vdd_osc",
- "vdd_pll",
- };
int i, ret;
DRM_DEBUG_KMS("HDMI resource init\n");
+ hdata->hpd_gpio = devm_gpiod_get(dev, "hpd", GPIOD_IN);
+ if (IS_ERR(hdata->hpd_gpio)) {
+ DRM_ERROR("cannot get hpd gpio property\n");
+ return PTR_ERR(hdata->hpd_gpio);
+ }
+
+ hdata->irq = gpiod_to_irq(hdata->hpd_gpio);
+ if (hdata->irq < 0) {
+ DRM_ERROR("failed to get GPIO irq\n");
+ return hdata->irq;
+ }
/* get clocks, power */
- res->hdmi = devm_clk_get(dev, "hdmi");
- if (IS_ERR(res->hdmi)) {
+ hdata->hdmi = devm_clk_get(dev, "hdmi");
+ if (IS_ERR(hdata->hdmi)) {
DRM_ERROR("failed to get clock 'hdmi'\n");
- ret = PTR_ERR(res->hdmi);
+ ret = PTR_ERR(hdata->hdmi);
goto fail;
}
- res->sclk_hdmi = devm_clk_get(dev, "sclk_hdmi");
- if (IS_ERR(res->sclk_hdmi)) {
+ hdata->sclk_hdmi = devm_clk_get(dev, "sclk_hdmi");
+ if (IS_ERR(hdata->sclk_hdmi)) {
DRM_ERROR("failed to get clock 'sclk_hdmi'\n");
- ret = PTR_ERR(res->sclk_hdmi);
+ ret = PTR_ERR(hdata->sclk_hdmi);
goto fail;
}
- res->sclk_pixel = devm_clk_get(dev, "sclk_pixel");
- if (IS_ERR(res->sclk_pixel)) {
+ hdata->sclk_pixel = devm_clk_get(dev, "sclk_pixel");
+ if (IS_ERR(hdata->sclk_pixel)) {
DRM_ERROR("failed to get clock 'sclk_pixel'\n");
- ret = PTR_ERR(res->sclk_pixel);
+ ret = PTR_ERR(hdata->sclk_pixel);
goto fail;
}
- res->sclk_hdmiphy = devm_clk_get(dev, "sclk_hdmiphy");
- if (IS_ERR(res->sclk_hdmiphy)) {
+ hdata->sclk_hdmiphy = devm_clk_get(dev, "sclk_hdmiphy");
+ if (IS_ERR(hdata->sclk_hdmiphy)) {
DRM_ERROR("failed to get clock 'sclk_hdmiphy'\n");
- ret = PTR_ERR(res->sclk_hdmiphy);
+ ret = PTR_ERR(hdata->sclk_hdmiphy);
goto fail;
}
- res->mout_hdmi = devm_clk_get(dev, "mout_hdmi");
- if (IS_ERR(res->mout_hdmi)) {
+ hdata->mout_hdmi = devm_clk_get(dev, "mout_hdmi");
+ if (IS_ERR(hdata->mout_hdmi)) {
DRM_ERROR("failed to get clock 'mout_hdmi'\n");
- ret = PTR_ERR(res->mout_hdmi);
+ ret = PTR_ERR(hdata->mout_hdmi);
goto fail;
}
- clk_set_parent(res->mout_hdmi, res->sclk_pixel);
+ clk_set_parent(hdata->mout_hdmi, hdata->sclk_pixel);
- res->regul_bulk = devm_kzalloc(dev, ARRAY_SIZE(supply) *
- sizeof(res->regul_bulk[0]), GFP_KERNEL);
- if (!res->regul_bulk) {
- ret = -ENOMEM;
- goto fail;
- }
for (i = 0; i < ARRAY_SIZE(supply); ++i) {
- res->regul_bulk[i].supply = supply[i];
- res->regul_bulk[i].consumer = NULL;
+ hdata->regul_bulk[i].supply = supply[i];
+ hdata->regul_bulk[i].consumer = NULL;
}
- ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(supply), res->regul_bulk);
+ ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(supply), hdata->regul_bulk);
if (ret) {
DRM_ERROR("failed to get regulators\n");
return ret;
}
- res->regul_count = ARRAY_SIZE(supply);
- res->reg_hdmi_en = devm_regulator_get(dev, "hdmi-en");
- if (IS_ERR(res->reg_hdmi_en) && PTR_ERR(res->reg_hdmi_en) != -ENOENT) {
- DRM_ERROR("failed to get hdmi-en regulator\n");
- return PTR_ERR(res->reg_hdmi_en);
- }
- if (!IS_ERR(res->reg_hdmi_en)) {
- ret = regulator_enable(res->reg_hdmi_en);
- if (ret) {
- DRM_ERROR("failed to enable hdmi-en regulator\n");
- return ret;
- }
- } else
- res->reg_hdmi_en = NULL;
+ hdata->reg_hdmi_en = devm_regulator_get_optional(dev, "hdmi-en");
+
+ if (PTR_ERR(hdata->reg_hdmi_en) == -ENODEV)
+ return 0;
+
+ if (IS_ERR(hdata->reg_hdmi_en))
+ return PTR_ERR(hdata->reg_hdmi_en);
+
+ ret = regulator_enable(hdata->reg_hdmi_en);
+ if (ret)
+ DRM_ERROR("failed to enable hdmi-en regulator\n");
return ret;
fail:
static struct of_device_id hdmi_match_types[] = {
{
- .compatible = "samsung,exynos5-hdmi",
- .data = &exynos5_hdmi_driver_data,
- }, {
.compatible = "samsung,exynos4210-hdmi",
.data = &exynos4210_hdmi_driver_data,
}, {
platform_set_drvdata(pdev, hdata);
hdata->dev = dev;
- hdata->hpd_gpio = of_get_named_gpio(dev->of_node, "hpd-gpio", 0);
- if (hdata->hpd_gpio < 0) {
- DRM_ERROR("cannot get hpd gpio property\n");
- return hdata->hpd_gpio;
- }
ret = hdmi_resources_init(hdata);
if (ret) {
return ret;
}
- ret = devm_gpio_request(dev, hdata->hpd_gpio, "HPD");
- if (ret) {
- DRM_ERROR("failed to request HPD gpio\n");
- return ret;
- }
-
ddc_node = hdmi_legacy_ddc_dt_binding(dev);
if (ddc_node)
goto out_get_ddc_adpt;
}
}
- hdata->irq = gpio_to_irq(hdata->hpd_gpio);
- if (hdata->irq < 0) {
- DRM_ERROR("failed to get GPIO irq\n");
- ret = hdata->irq;
- goto err_hdmiphy;
- }
-
INIT_DELAYED_WORK(&hdata->hotplug_work, hdmi_hotplug_work_func);
ret = devm_request_threaded_irq(dev, hdata->irq, NULL,
cancel_delayed_work_sync(&hdata->hotplug_work);
- if (hdata->res.reg_hdmi_en)
- regulator_disable(hdata->res.reg_hdmi_en);
+ component_del(&pdev->dev, &hdmi_component_ops);
+
+ pm_runtime_disable(&pdev->dev);
+
+ if (!IS_ERR(hdata->reg_hdmi_en))
+ regulator_disable(hdata->reg_hdmi_en);
if (hdata->hdmiphy_port)
put_device(&hdata->hdmiphy_port->dev);
- put_device(&hdata->ddc_adpt->dev);
- pm_runtime_disable(&pdev->dev);
- component_del(&pdev->dev, &hdmi_component_ops);
+ put_device(&hdata->ddc_adpt->dev);
return 0;
}
#include "exynos_drm_crtc.h"
#include "exynos_drm_plane.h"
#include "exynos_drm_iommu.h"
-#include "exynos_mixer.h"
#define MIXER_WIN_NR 3
-#define MIXER_DEFAULT_WIN 0
#define VP_DEFAULT_WIN 2
+#define CURSOR_WIN 1
/* The pixelformats that are natively supported by the mixer. */
#define MXR_FORMAT_RGB565 4
/* setup display size */
if (ctx->mxr_ver == MXR_VER_128_0_0_184 &&
- win == MIXER_DEFAULT_WIN) {
+ win == DEFAULT_WIN) {
val = MXR_MXR_RES_HEIGHT(mode->vdisplay);
val |= MXR_MXR_RES_WIDTH(mode->hdisplay);
mixer_reg_write(res, MXR_RESOLUTION, val);
/* waiting until VP_SRESET_PROCESSING is 0 */
if (~vp_reg_read(res, VP_SRESET) & VP_SRESET_PROCESSING)
break;
- usleep_range(10000, 12000);
+ mdelay(10);
}
WARN(tries == 0, "failed to reset Video Processor\n");
}
}
/* Only valid for Mixer version 16.0.33.0 */
-int mixer_check_mode(struct drm_display_mode *mode)
+static int mixer_atomic_check(struct exynos_drm_crtc *crtc,
+ struct drm_crtc_state *state)
{
+ struct drm_display_mode *mode = &state->adjusted_mode;
u32 w, h;
w = mode->hdisplay;
.wait_for_vblank = mixer_wait_for_vblank,
.update_plane = mixer_update_plane,
.disable_plane = mixer_disable_plane,
+ .atomic_check = mixer_atomic_check,
};
static struct mixer_drv_data exynos5420_mxr_drv_data = {
const uint32_t *formats;
unsigned int fcount;
- type = (zpos == MIXER_DEFAULT_WIN) ? DRM_PLANE_TYPE_PRIMARY :
- DRM_PLANE_TYPE_OVERLAY;
if (zpos < VP_DEFAULT_WIN) {
formats = mixer_formats;
fcount = ARRAY_SIZE(mixer_formats);
fcount = ARRAY_SIZE(vp_formats);
}
+ type = exynos_plane_get_type(zpos, CURSOR_WIN);
ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
1 << ctx->pipe, type, formats, fcount,
zpos);
return ret;
}
- exynos_plane = &ctx->planes[MIXER_DEFAULT_WIN];
+ exynos_plane = &ctx->planes[DEFAULT_WIN];
ctx->crtc = exynos_drm_crtc_create(drm_dev, &exynos_plane->base,
ctx->pipe, EXYNOS_DISPLAY_TYPE_HDMI,
&mixer_crtc_ops, ctx);
+++ /dev/null
-/*
- * Copyright (C) 2013 Google, Inc.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
-
-#ifndef _EXYNOS_MIXER_H_
-#define _EXYNOS_MIXER_H_
-
-/* This function returns 0 if the given timing is valid for the mixer */
-int mixer_check_mode(struct drm_display_mode *mode);
-
-#endif
#define HDMI_V13_V_SYNC_GEN_3_0 HDMI_CORE_BASE(0x0150)
#define HDMI_V13_V_SYNC_GEN_3_1 HDMI_CORE_BASE(0x0154)
#define HDMI_V13_V_SYNC_GEN_3_2 HDMI_CORE_BASE(0x0158)
-#define HDMI_V13_ACR_CON HDMI_CORE_BASE(0x0180)
#define HDMI_V13_AVI_CON HDMI_CORE_BASE(0x0300)
#define HDMI_V13_AVI_BYTE(n) HDMI_CORE_BASE(0x0320 + 4 * (n))
#define HDMI_V13_DC_CONTROL HDMI_CORE_BASE(0x05C0)
#define HDMI_HPD_ST HDMI_CTRL_BASE(0x0044)
#define HDMI_HPD_TH_X HDMI_CTRL_BASE(0x0050)
#define HDMI_AUDIO_CLKSEL HDMI_CTRL_BASE(0x0070)
-#define HDMI_PHY_RSTOUT HDMI_CTRL_BASE(0x0074)
+#define HDMI_V14_PHY_RSTOUT HDMI_CTRL_BASE(0x0074)
#define HDMI_PHY_VPLL HDMI_CTRL_BASE(0x0078)
#define HDMI_PHY_CMU HDMI_CTRL_BASE(0x007C)
#define HDMI_CORE_RSTOUT HDMI_CTRL_BASE(0x0080)
#define HDMI_ASP_CHCFG2 HDMI_CORE_BASE(0x0318)
#define HDMI_ASP_CHCFG3 HDMI_CORE_BASE(0x031C)
-#define HDMI_ACR_CON HDMI_CORE_BASE(0x0400)
-#define HDMI_ACR_MCTS0 HDMI_CORE_BASE(0x0410)
-#define HDMI_ACR_MCTS1 HDMI_CORE_BASE(0x0414)
-#define HDMI_ACR_MCTS2 HDMI_CORE_BASE(0x0418)
-#define HDMI_ACR_CTS0 HDMI_CORE_BASE(0x0420)
-#define HDMI_ACR_CTS1 HDMI_CORE_BASE(0x0424)
-#define HDMI_ACR_CTS2 HDMI_CORE_BASE(0x0428)
-#define HDMI_ACR_N0 HDMI_CORE_BASE(0x0430)
-#define HDMI_ACR_N1 HDMI_CORE_BASE(0x0434)
-#define HDMI_ACR_N2 HDMI_CORE_BASE(0x0438)
+#define HDMI_V13_ACR_CON HDMI_CORE_BASE(0x0180)
+#define HDMI_V13_ACR_MCTS0 HDMI_CORE_BASE(0x0184)
+#define HDMI_V13_ACR_MCTS1 HDMI_CORE_BASE(0x0188)
+#define HDMI_V13_ACR_MCTS2 HDMI_CORE_BASE(0x018C)
+#define HDMI_V13_ACR_CTS0 HDMI_CORE_BASE(0x0190)
+#define HDMI_V13_ACR_CTS1 HDMI_CORE_BASE(0x0194)
+#define HDMI_V13_ACR_CTS2 HDMI_CORE_BASE(0x0198)
+#define HDMI_V13_ACR_N0 HDMI_CORE_BASE(0x01A0)
+#define HDMI_V13_ACR_N1 HDMI_CORE_BASE(0x01A4)
+#define HDMI_V13_ACR_N2 HDMI_CORE_BASE(0x01A8)
+#define HDMI_V14_ACR_CON HDMI_CORE_BASE(0x0400)
+#define HDMI_V14_ACR_MCTS0 HDMI_CORE_BASE(0x0410)
+#define HDMI_V14_ACR_MCTS1 HDMI_CORE_BASE(0x0414)
+#define HDMI_V14_ACR_MCTS2 HDMI_CORE_BASE(0x0418)
+#define HDMI_V14_ACR_CTS0 HDMI_CORE_BASE(0x0420)
+#define HDMI_V14_ACR_CTS1 HDMI_CORE_BASE(0x0424)
+#define HDMI_V14_ACR_CTS2 HDMI_CORE_BASE(0x0428)
+#define HDMI_V14_ACR_N0 HDMI_CORE_BASE(0x0430)
+#define HDMI_V14_ACR_N1 HDMI_CORE_BASE(0x0434)
+#define HDMI_V14_ACR_N2 HDMI_CORE_BASE(0x0438)
/* Packet related registers */
#define HDMI_ACP_CON HDMI_CORE_BASE(0x0500)
return IRQ_HANDLED;
}
-static int fsl_dcu_drm_enable_vblank(struct drm_device *dev, int crtc)
+static int fsl_dcu_drm_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
unsigned int value;
return 0;
}
-static void fsl_dcu_drm_disable_vblank(struct drm_device *dev, int crtc)
+static void fsl_dcu_drm_disable_vblank(struct drm_device *dev,
+ unsigned int pipe)
{
struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
unsigned int value;
.unload = fsl_dcu_unload,
.preclose = fsl_dcu_drm_preclose,
.irq_handler = fsl_dcu_drm_irq,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = fsl_dcu_drm_enable_vblank,
.disable_vblank = fsl_dcu_drm_disable_vblank,
.gem_free_object = drm_gem_cma_free_object,
struct drm_plane_state *old_state)
{
struct fsl_dcu_drm_device *fsl_dev = plane->dev->dev_private;
- unsigned int index, value, ret;
+ unsigned int value;
+ int index, ret;
index = fsl_dcu_drm_plane_index(plane);
if (index < 0)
static void
fsl_dcu_drm_plane_cleanup_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
}
static int
fsl_dcu_drm_plane_prepare_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
return 0;
#define wait_for(COND, MS) _wait_for(COND, MS, 1)
-#define DP_LINK_STATUS_SIZE 6
#define DP_LINK_CHECK_TIMEOUT (10 * 1000)
#define DP_LINK_CONFIGURATION_SIZE 9
extern void psb_irq_uninstall_islands(struct drm_device *dev, int hw_islands);
extern int psb_vblank_wait2(struct drm_device *dev, unsigned int *sequence);
extern int psb_vblank_wait(struct drm_device *dev, unsigned int *sequence);
-extern int psb_enable_vblank(struct drm_device *dev, int crtc);
-extern void psb_disable_vblank(struct drm_device *dev, int crtc);
+extern int psb_enable_vblank(struct drm_device *dev, unsigned int pipe);
+extern void psb_disable_vblank(struct drm_device *dev, unsigned int pipe);
void
psb_enable_pipestat(struct drm_psb_private *dev_priv, int pipe, u32 mask);
void
psb_disable_pipestat(struct drm_psb_private *dev_priv, int pipe, u32 mask);
-extern u32 psb_get_vblank_counter(struct drm_device *dev, int crtc);
+extern u32 psb_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
/* framebuffer.c */
extern int psbfb_probed(struct drm_device *dev);
/*
* It is used to enable VBLANK interrupt
*/
-int psb_enable_vblank(struct drm_device *dev, int pipe)
+int psb_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_psb_private *dev_priv = dev->dev_private;
unsigned long irqflags;
/*
* It is used to disable VBLANK interrupt
*/
-void psb_disable_vblank(struct drm_device *dev, int pipe)
+void psb_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_psb_private *dev_priv = dev->dev_private;
unsigned long irqflags;
/* Called from drm generic code, passed a 'crtc', which
* we use as a pipe index
*/
-u32 psb_get_vblank_counter(struct drm_device *dev, int pipe)
+u32 psb_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
uint32_t high_frame = PIPEAFRAMEHIGH;
uint32_t low_frame = PIPEAFRAMEPIXEL;
reg_val = REG_READ(pipeconf_reg);
if (!(reg_val & PIPEACONF_ENABLE)) {
- dev_err(dev->dev, "trying to get vblank count for disabled pipe %d\n",
+ dev_err(dev->dev, "trying to get vblank count for disabled pipe %u\n",
pipe);
goto psb_get_vblank_counter_exit;
}
int psb_irq_disable_dpst(struct drm_device *dev);
void psb_irq_turn_on_dpst(struct drm_device *dev);
void psb_irq_turn_off_dpst(struct drm_device *dev);
-int psb_enable_vblank(struct drm_device *dev, int pipe);
-void psb_disable_vblank(struct drm_device *dev, int pipe);
-u32 psb_get_vblank_counter(struct drm_device *dev, int pipe);
+int psb_enable_vblank(struct drm_device *dev, unsigned int pipe);
+void psb_disable_vblank(struct drm_device *dev, unsigned int pipe);
+u32 psb_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
int mdfld_enable_te(struct drm_device *dev, int pipe);
void mdfld_disable_te(struct drm_device *dev, int pipe);
struct ch7006_encoder_params *params = &priv->params;
struct ch7006_state *state = &priv->state;
uint8_t *regs = state->regs;
- struct ch7006_mode *mode = priv->mode;
- struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
+ const struct ch7006_mode *mode = priv->mode;
+ const struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
int start_active;
ch7006_dbg(client, "\n");
struct drm_connector *connector)
{
struct ch7006_priv *priv = to_ch7006_priv(encoder);
- struct ch7006_mode *mode;
+ const struct ch7006_mode *mode;
int n = 0;
for (mode = ch7006_modes; mode->mode.clock; mode++) {
#include "ch7006_priv.h"
-char *ch7006_tv_norm_names[] = {
+const char * const ch7006_tv_norm_names[] = {
[TV_NORM_PAL] = "PAL",
[TV_NORM_PAL_M] = "PAL-M",
[TV_NORM_PAL_N] = "PAL-N",
.vtotal = 625, \
.hvirtual = 810
-struct ch7006_tv_norm_info ch7006_tv_norms[] = {
+const struct ch7006_tv_norm_info ch7006_tv_norms[] = {
[TV_NORM_NTSC_M] = {
NTSC_LIKE_TIMINGS,
.black_level = 0.339 * fixed1,
#define PAL_LIKE (1 << TV_NORM_PAL | 1 << TV_NORM_PAL_N | 1 << TV_NORM_PAL_NC)
-struct ch7006_mode ch7006_modes[] = {
+const struct ch7006_mode ch7006_modes[] = {
MODE(21000, 512, 384, 840, 500, N, N, 181.797557582, 5_4, 0x6, PAL_LIKE),
MODE(26250, 512, 384, 840, 625, N, N, 145.438046066, 1_1, 0x1, PAL_LIKE),
MODE(20140, 512, 384, 800, 420, N, N, 213.257083791, 5_4, 0x4, NTSC_LIKE),
{}
};
-struct ch7006_mode *ch7006_lookup_mode(struct drm_encoder *encoder,
- const struct drm_display_mode *drm_mode)
+const struct ch7006_mode *ch7006_lookup_mode(struct drm_encoder *encoder,
+ const struct drm_display_mode *drm_mode)
{
struct ch7006_priv *priv = to_ch7006_priv(encoder);
- struct ch7006_mode *mode;
+ const struct ch7006_mode *mode;
for (mode = ch7006_modes; mode->mode.clock; mode++) {
struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
struct ch7006_priv *priv = to_ch7006_priv(encoder);
uint8_t *regs = priv->state.regs;
- struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
+ const struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
int gain;
int black_level;
struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
struct ch7006_priv *priv = to_ch7006_priv(encoder);
struct ch7006_state *state = &priv->state;
- struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
- struct ch7006_mode *mode = priv->mode;
+ const struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
+ const struct ch7006_mode *mode = priv->mode;
uint32_t subc_inc;
subc_inc = round_fixed((mode->subc_coeff >> 8)
struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
struct ch7006_priv *priv = to_ch7006_priv(encoder);
uint8_t *regs = priv->state.regs;
- struct ch7006_mode *mode = priv->mode;
+ const struct ch7006_mode *mode = priv->mode;
int n, best_n = 0;
int m, best_m = 0;
int freq, best_freq = 0;
struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
struct ch7006_priv *priv = to_ch7006_priv(encoder);
struct ch7006_state *state = &priv->state;
- struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
- struct ch7006_mode *ch_mode = priv->mode;
- struct drm_display_mode *mode = &ch_mode->mode;
+ const struct ch7006_tv_norm_info *norm = &ch7006_tv_norms[priv->norm];
+ const struct ch7006_mode *ch_mode = priv->mode;
+ const struct drm_display_mode *mode = &ch_mode->mode;
uint8_t *regs = state->regs;
int flicker, contrast, hpos, vpos;
uint64_t scale, aspect;
struct ch7006_priv {
struct ch7006_encoder_params params;
- struct ch7006_mode *mode;
+ const struct ch7006_mode *mode;
struct ch7006_state state;
struct ch7006_state saved_state;
extern char *ch7006_tv_norm;
extern int ch7006_scale;
-extern char *ch7006_tv_norm_names[];
-extern struct ch7006_tv_norm_info ch7006_tv_norms[];
-extern struct ch7006_mode ch7006_modes[];
+extern const char * const ch7006_tv_norm_names[];
+extern const struct ch7006_tv_norm_info ch7006_tv_norms[];
+extern const struct ch7006_mode ch7006_modes[];
-struct ch7006_mode *ch7006_lookup_mode(struct drm_encoder *encoder,
- const struct drm_display_mode *drm_mode);
+const struct ch7006_mode *ch7006_lookup_mode(struct drm_encoder *encoder,
+ const struct drm_display_mode *drm_mode);
void ch7006_setup_levels(struct drm_encoder *encoder);
void ch7006_setup_subcarrier(struct drm_encoder *encoder);
#include <drm/drmP.h>
#include <drm/drm_crtc_helper.h>
-#include <drm/drm_encoder_slave.h>
#include <drm/drm_edid.h>
#include <drm/drm_of.h>
#include <drm/i2c/tda998x.h>
struct i2c_client *cec;
struct i2c_client *hdmi;
struct mutex mutex;
- struct delayed_work dwork;
- uint16_t rev;
- uint8_t current_page;
+ u16 rev;
+ u8 current_page;
int dpms;
bool is_hdmi_sink;
u8 vip_cntrl_0;
wait_queue_head_t wq_edid;
volatile int wq_edid_wait;
- struct drm_encoder *encoder;
+
+ struct work_struct detect_work;
+ struct timer_list edid_delay_timer;
+ wait_queue_head_t edid_delay_waitq;
+ bool edid_delay_active;
+
+ struct drm_encoder encoder;
+ struct drm_connector connector;
};
-#define to_tda998x_priv(x) ((struct tda998x_priv *)to_encoder_slave(x)->slave_priv)
+#define conn_to_tda998x_priv(x) \
+ container_of(x, struct tda998x_priv, connector)
+
+#define enc_to_tda998x_priv(x) \
+ container_of(x, struct tda998x_priv, encoder)
/* The TDA9988 series of devices use a paged register scheme.. to simplify
* things we encode the page # in upper bits of the register #. To read/
# define CEC_FRO_IM_CLK_CTRL_FRO_DIV (1 << 0)
#define REG_CEC_RXSHPDINTENA 0xfc /* read/write */
#define REG_CEC_RXSHPDINT 0xfd /* read */
+# define CEC_RXSHPDINT_RXSENS BIT(0)
+# define CEC_RXSHPDINT_HPD BIT(1)
#define REG_CEC_RXSHPDLEV 0xfe /* read */
# define CEC_RXSHPDLEV_RXSENS (1 << 0)
# define CEC_RXSHPDLEV_HPD (1 << 1)
#define TDA19988 0x0301
static void
-cec_write(struct tda998x_priv *priv, uint16_t addr, uint8_t val)
+cec_write(struct tda998x_priv *priv, u16 addr, u8 val)
{
struct i2c_client *client = priv->cec;
- uint8_t buf[] = {addr, val};
+ u8 buf[] = {addr, val};
int ret;
ret = i2c_master_send(client, buf, sizeof(buf));
dev_err(&client->dev, "Error %d writing to cec:0x%x\n", ret, addr);
}
-static uint8_t
-cec_read(struct tda998x_priv *priv, uint8_t addr)
+static u8
+cec_read(struct tda998x_priv *priv, u8 addr)
{
struct i2c_client *client = priv->cec;
- uint8_t val;
+ u8 val;
int ret;
ret = i2c_master_send(client, &addr, sizeof(addr));
}
static int
-set_page(struct tda998x_priv *priv, uint16_t reg)
+set_page(struct tda998x_priv *priv, u16 reg)
{
if (REG2PAGE(reg) != priv->current_page) {
struct i2c_client *client = priv->hdmi;
- uint8_t buf[] = {
+ u8 buf[] = {
REG_CURPAGE, REG2PAGE(reg)
};
int ret = i2c_master_send(client, buf, sizeof(buf));
}
static int
-reg_read_range(struct tda998x_priv *priv, uint16_t reg, char *buf, int cnt)
+reg_read_range(struct tda998x_priv *priv, u16 reg, char *buf, int cnt)
{
struct i2c_client *client = priv->hdmi;
- uint8_t addr = REG2ADDR(reg);
+ u8 addr = REG2ADDR(reg);
int ret;
mutex_lock(&priv->mutex);
}
static void
-reg_write_range(struct tda998x_priv *priv, uint16_t reg, uint8_t *p, int cnt)
+reg_write_range(struct tda998x_priv *priv, u16 reg, u8 *p, int cnt)
{
struct i2c_client *client = priv->hdmi;
- uint8_t buf[cnt+1];
+ u8 buf[cnt+1];
int ret;
buf[0] = REG2ADDR(reg);
}
static int
-reg_read(struct tda998x_priv *priv, uint16_t reg)
+reg_read(struct tda998x_priv *priv, u16 reg)
{
- uint8_t val = 0;
+ u8 val = 0;
int ret;
ret = reg_read_range(priv, reg, &val, sizeof(val));
}
static void
-reg_write(struct tda998x_priv *priv, uint16_t reg, uint8_t val)
+reg_write(struct tda998x_priv *priv, u16 reg, u8 val)
{
struct i2c_client *client = priv->hdmi;
- uint8_t buf[] = {REG2ADDR(reg), val};
+ u8 buf[] = {REG2ADDR(reg), val};
int ret;
mutex_lock(&priv->mutex);
}
static void
-reg_write16(struct tda998x_priv *priv, uint16_t reg, uint16_t val)
+reg_write16(struct tda998x_priv *priv, u16 reg, u16 val)
{
struct i2c_client *client = priv->hdmi;
- uint8_t buf[] = {REG2ADDR(reg), val >> 8, val};
+ u8 buf[] = {REG2ADDR(reg), val >> 8, val};
int ret;
mutex_lock(&priv->mutex);
}
static void
-reg_set(struct tda998x_priv *priv, uint16_t reg, uint8_t val)
+reg_set(struct tda998x_priv *priv, u16 reg, u8 val)
{
int old_val;
}
static void
-reg_clear(struct tda998x_priv *priv, uint16_t reg, uint8_t val)
+reg_clear(struct tda998x_priv *priv, u16 reg, u8 val)
{
int old_val;
reg_write(priv, REG_MUX_VP_VIP_OUT, 0x24);
}
-/* handle HDMI connect/disconnect */
-static void tda998x_hpd(struct work_struct *work)
+/*
+ * The TDA998x has a problem when trying to read the EDID close to a
+ * HPD assertion: it needs a delay of 100ms to avoid timing out while
+ * trying to read EDID data.
+ *
+ * However, tda998x_encoder_get_modes() may be called at any moment
+ * after tda998x_connector_detect() indicates that we are connected, so
+ * we need to delay probing modes in tda998x_encoder_get_modes() after
+ * we have seen a HPD inactive->active transition. This code implements
+ * that delay.
+ */
+static void tda998x_edid_delay_done(unsigned long data)
+{
+ struct tda998x_priv *priv = (struct tda998x_priv *)data;
+
+ priv->edid_delay_active = false;
+ wake_up(&priv->edid_delay_waitq);
+ schedule_work(&priv->detect_work);
+}
+
+static void tda998x_edid_delay_start(struct tda998x_priv *priv)
+{
+ priv->edid_delay_active = true;
+ mod_timer(&priv->edid_delay_timer, jiffies + HZ/10);
+}
+
+static int tda998x_edid_delay_wait(struct tda998x_priv *priv)
+{
+ return wait_event_killable(priv->edid_delay_waitq, !priv->edid_delay_active);
+}
+
+/*
+ * We need to run the KMS hotplug event helper outside of our threaded
+ * interrupt routine as this can call back into our get_modes method,
+ * which will want to make use of interrupts.
+ */
+static void tda998x_detect_work(struct work_struct *work)
{
- struct delayed_work *dwork = to_delayed_work(work);
struct tda998x_priv *priv =
- container_of(dwork, struct tda998x_priv, dwork);
+ container_of(work, struct tda998x_priv, detect_work);
+ struct drm_device *dev = priv->encoder.dev;
- if (priv->encoder && priv->encoder->dev)
- drm_kms_helper_hotplug_event(priv->encoder->dev);
+ if (dev)
+ drm_kms_helper_hotplug_event(dev);
}
/*
{
struct tda998x_priv *priv = data;
u8 sta, cec, lvl, flag0, flag1, flag2;
+ bool handled = false;
- if (!priv)
- return IRQ_HANDLED;
sta = cec_read(priv, REG_CEC_INTSTATUS);
cec = cec_read(priv, REG_CEC_RXSHPDINT);
lvl = cec_read(priv, REG_CEC_RXSHPDLEV);
DRM_DEBUG_DRIVER(
"tda irq sta %02x cec %02x lvl %02x f0 %02x f1 %02x f2 %02x\n",
sta, cec, lvl, flag0, flag1, flag2);
+
+ if (cec & CEC_RXSHPDINT_HPD) {
+ if (lvl & CEC_RXSHPDLEV_HPD)
+ tda998x_edid_delay_start(priv);
+ else
+ schedule_work(&priv->detect_work);
+
+ handled = true;
+ }
+
if ((flag2 & INT_FLAGS_2_EDID_BLK_RD) && priv->wq_edid_wait) {
priv->wq_edid_wait = 0;
wake_up(&priv->wq_edid);
- } else if (cec != 0) { /* HPD change */
- schedule_delayed_work(&priv->dwork, HZ/10);
+ handled = true;
}
- return IRQ_HANDLED;
-}
-static uint8_t tda998x_cksum(uint8_t *buf, size_t bytes)
-{
- int sum = 0;
-
- while (bytes--)
- sum -= *buf++;
- return sum;
+ return IRQ_RETVAL(handled);
}
-#define HB(x) (x)
-#define PB(x) (HB(2) + 1 + (x))
-
static void
-tda998x_write_if(struct tda998x_priv *priv, uint8_t bit, uint16_t addr,
- uint8_t *buf, size_t size)
+tda998x_write_if(struct tda998x_priv *priv, u8 bit, u16 addr,
+ union hdmi_infoframe *frame)
{
+ u8 buf[32];
+ ssize_t len;
+
+ len = hdmi_infoframe_pack(frame, buf, sizeof(buf));
+ if (len < 0) {
+ dev_err(&priv->hdmi->dev,
+ "hdmi_infoframe_pack() type=0x%02x failed: %zd\n",
+ frame->any.type, len);
+ return;
+ }
+
reg_clear(priv, REG_DIP_IF_FLAGS, bit);
- reg_write_range(priv, addr, buf, size);
+ reg_write_range(priv, addr, buf, len);
reg_set(priv, REG_DIP_IF_FLAGS, bit);
}
static void
tda998x_write_aif(struct tda998x_priv *priv, struct tda998x_encoder_params *p)
{
- u8 buf[PB(HDMI_AUDIO_INFOFRAME_SIZE) + 1];
+ union hdmi_infoframe frame;
+
+ hdmi_audio_infoframe_init(&frame.audio);
- memset(buf, 0, sizeof(buf));
- buf[HB(0)] = HDMI_INFOFRAME_TYPE_AUDIO;
- buf[HB(1)] = 0x01;
- buf[HB(2)] = HDMI_AUDIO_INFOFRAME_SIZE;
- buf[PB(1)] = p->audio_frame[1] & 0x07; /* CC */
- buf[PB(2)] = p->audio_frame[2] & 0x1c; /* SF */
- buf[PB(4)] = p->audio_frame[4];
- buf[PB(5)] = p->audio_frame[5] & 0xf8; /* DM_INH + LSV */
+ frame.audio.channels = p->audio_frame[1] & 0x07;
+ frame.audio.channel_allocation = p->audio_frame[4];
+ frame.audio.level_shift_value = (p->audio_frame[5] & 0x78) >> 3;
+ frame.audio.downmix_inhibit = (p->audio_frame[5] & 0x80) >> 7;
- buf[PB(0)] = tda998x_cksum(buf, sizeof(buf));
+ /*
+ * L-PCM and IEC61937 compressed audio shall always set sample
+ * frequency to "refer to stream". For others, see the HDMI
+ * specification.
+ */
+ frame.audio.sample_frequency = (p->audio_frame[2] & 0x1c) >> 2;
- tda998x_write_if(priv, DIP_IF_FLAGS_IF4, REG_IF4_HB0, buf,
- sizeof(buf));
+ tda998x_write_if(priv, DIP_IF_FLAGS_IF4, REG_IF4_HB0, &frame);
}
static void
tda998x_write_avi(struct tda998x_priv *priv, struct drm_display_mode *mode)
{
- struct hdmi_avi_infoframe frame;
- u8 buf[HDMI_INFOFRAME_HEADER_SIZE + HDMI_AVI_INFOFRAME_SIZE];
- ssize_t len;
+ union hdmi_infoframe frame;
- drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+ drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+ frame.avi.quantization_range = HDMI_QUANTIZATION_RANGE_FULL;
- frame.quantization_range = HDMI_QUANTIZATION_RANGE_FULL;
-
- len = hdmi_avi_infoframe_pack(&frame, buf, sizeof(buf));
- if (len < 0) {
- dev_err(&priv->hdmi->dev,
- "hdmi_avi_infoframe_pack() failed: %zd\n", len);
- return;
- }
-
- tda998x_write_if(priv, DIP_IF_FLAGS_IF2, REG_IF2_HB0, buf, len);
+ tda998x_write_if(priv, DIP_IF_FLAGS_IF2, REG_IF2_HB0, &frame);
}
static void tda998x_audio_mute(struct tda998x_priv *priv, bool on)
tda998x_configure_audio(struct tda998x_priv *priv,
struct drm_display_mode *mode, struct tda998x_encoder_params *p)
{
- uint8_t buf[6], clksel_aip, clksel_fs, cts_n, adiv;
- uint32_t n;
+ u8 buf[6], clksel_aip, clksel_fs, cts_n, adiv;
+ u32 n;
/* Enable audio ports */
reg_write(priv, REG_ENA_AP, p->audio_cfg);
priv->params = *p;
}
-static void tda998x_encoder_dpms(struct tda998x_priv *priv, int mode)
+static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
{
+ struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
+
/* we only care about on or off: */
if (mode != DRM_MODE_DPMS_ON)
mode = DRM_MODE_DPMS_OFF;
return true;
}
-static int tda998x_encoder_mode_valid(struct tda998x_priv *priv,
- struct drm_display_mode *mode)
+static int tda998x_connector_mode_valid(struct drm_connector *connector,
+ struct drm_display_mode *mode)
{
if (mode->clock > 150000)
return MODE_CLOCK_HIGH;
}
static void
-tda998x_encoder_mode_set(struct tda998x_priv *priv,
+tda998x_encoder_mode_set(struct drm_encoder *encoder,
struct drm_display_mode *mode,
struct drm_display_mode *adjusted_mode)
{
- uint16_t ref_pix, ref_line, n_pix, n_line;
- uint16_t hs_pix_s, hs_pix_e;
- uint16_t vs1_pix_s, vs1_pix_e, vs1_line_s, vs1_line_e;
- uint16_t vs2_pix_s, vs2_pix_e, vs2_line_s, vs2_line_e;
- uint16_t vwin1_line_s, vwin1_line_e;
- uint16_t vwin2_line_s, vwin2_line_e;
- uint16_t de_pix_s, de_pix_e;
- uint8_t reg, div, rep;
+ struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
+ u16 ref_pix, ref_line, n_pix, n_line;
+ u16 hs_pix_s, hs_pix_e;
+ u16 vs1_pix_s, vs1_pix_e, vs1_line_s, vs1_line_e;
+ u16 vs2_pix_s, vs2_pix_e, vs2_line_s, vs2_line_e;
+ u16 vwin1_line_s, vwin1_line_e;
+ u16 vwin2_line_s, vwin2_line_e;
+ u16 de_pix_s, de_pix_e;
+ u8 reg, div, rep;
/*
* Internally TDA998x is using ITU-R BT.656 style sync but
}
static enum drm_connector_status
-tda998x_encoder_detect(struct tda998x_priv *priv)
+tda998x_connector_detect(struct drm_connector *connector, bool force)
{
- uint8_t val = cec_read(priv, REG_CEC_RXSHPDLEV);
+ struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+ u8 val = cec_read(priv, REG_CEC_RXSHPDLEV);
return (val & CEC_RXSHPDLEV_HPD) ? connector_status_connected :
connector_status_disconnected;
static int read_edid_block(void *data, u8 *buf, unsigned int blk, size_t length)
{
struct tda998x_priv *priv = data;
- uint8_t offset, segptr;
+ u8 offset, segptr;
int ret, i;
offset = (blk & 1) ? 128 : 0;
return 0;
}
-static int
-tda998x_encoder_get_modes(struct tda998x_priv *priv,
- struct drm_connector *connector)
+static int tda998x_connector_get_modes(struct drm_connector *connector)
{
+ struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
struct edid *edid;
int n;
+ /*
+ * If we get killed while waiting for the HPD timeout, return
+ * no modes found: we are not in a restartable path, so we
+ * can't handle signals gracefully.
+ */
+ if (tda998x_edid_delay_wait(priv))
+ return 0;
+
if (priv->rev == TDA19988)
reg_clear(priv, REG_TX4, TX4_PD_RAM);
DRM_CONNECTOR_POLL_DISCONNECT;
}
-static int
-tda998x_encoder_set_property(struct drm_encoder *encoder,
- struct drm_connector *connector,
- struct drm_property *property,
- uint64_t val)
-{
- DBG("");
- return 0;
-}
-
static void tda998x_destroy(struct tda998x_priv *priv)
{
/* disable all IRQs and free the IRQ handler */
cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
- if (priv->hdmi->irq) {
- free_irq(priv->hdmi->irq, priv);
- cancel_delayed_work_sync(&priv->dwork);
- }
-
- i2c_unregister_device(priv->cec);
-}
-
-/* Slave encoder support */
-
-static void
-tda998x_encoder_slave_set_config(struct drm_encoder *encoder, void *params)
-{
- tda998x_encoder_set_config(to_tda998x_priv(encoder), params);
-}
-static void tda998x_encoder_slave_destroy(struct drm_encoder *encoder)
-{
- struct tda998x_priv *priv = to_tda998x_priv(encoder);
-
- tda998x_destroy(priv);
- drm_i2c_encoder_destroy(encoder);
- kfree(priv);
-}
-
-static void tda998x_encoder_slave_dpms(struct drm_encoder *encoder, int mode)
-{
- tda998x_encoder_dpms(to_tda998x_priv(encoder), mode);
-}
-
-static int tda998x_encoder_slave_mode_valid(struct drm_encoder *encoder,
- struct drm_display_mode *mode)
-{
- return tda998x_encoder_mode_valid(to_tda998x_priv(encoder), mode);
-}
+ if (priv->hdmi->irq)
+ free_irq(priv->hdmi->irq, priv);
-static void
-tda998x_encoder_slave_mode_set(struct drm_encoder *encoder,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
-{
- tda998x_encoder_mode_set(to_tda998x_priv(encoder), mode, adjusted_mode);
-}
+ del_timer_sync(&priv->edid_delay_timer);
+ cancel_work_sync(&priv->detect_work);
-static enum drm_connector_status
-tda998x_encoder_slave_detect(struct drm_encoder *encoder,
- struct drm_connector *connector)
-{
- return tda998x_encoder_detect(to_tda998x_priv(encoder));
-}
-
-static int tda998x_encoder_slave_get_modes(struct drm_encoder *encoder,
- struct drm_connector *connector)
-{
- return tda998x_encoder_get_modes(to_tda998x_priv(encoder), connector);
-}
-
-static int
-tda998x_encoder_slave_create_resources(struct drm_encoder *encoder,
- struct drm_connector *connector)
-{
- tda998x_encoder_set_polling(to_tda998x_priv(encoder), connector);
- return 0;
+ i2c_unregister_device(priv->cec);
}
-static struct drm_encoder_slave_funcs tda998x_encoder_slave_funcs = {
- .set_config = tda998x_encoder_slave_set_config,
- .destroy = tda998x_encoder_slave_destroy,
- .dpms = tda998x_encoder_slave_dpms,
- .save = tda998x_encoder_save,
- .restore = tda998x_encoder_restore,
- .mode_fixup = tda998x_encoder_mode_fixup,
- .mode_valid = tda998x_encoder_slave_mode_valid,
- .mode_set = tda998x_encoder_slave_mode_set,
- .detect = tda998x_encoder_slave_detect,
- .get_modes = tda998x_encoder_slave_get_modes,
- .create_resources = tda998x_encoder_slave_create_resources,
- .set_property = tda998x_encoder_set_property,
-};
-
/* I2C driver functions */
static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
priv->dpms = DRM_MODE_DPMS_OFF;
mutex_init(&priv->mutex); /* protect the page access */
+ init_waitqueue_head(&priv->edid_delay_waitq);
+ setup_timer(&priv->edid_delay_timer, tda998x_edid_delay_done,
+ (unsigned long)priv);
+ INIT_WORK(&priv->detect_work, tda998x_detect_work);
/* wake up the device: */
cec_write(priv, REG_CEC_ENAMODS,
/* init read EDID waitqueue and HDP work */
init_waitqueue_head(&priv->wq_edid);
- INIT_DELAYED_WORK(&priv->dwork, tda998x_hpd);
/* clear pending interrupts */
reg_read(priv, REG_INT_FLAGS_0);
return -ENXIO;
}
-static int tda998x_encoder_init(struct i2c_client *client,
- struct drm_device *dev,
- struct drm_encoder_slave *encoder_slave)
-{
- struct tda998x_priv *priv;
- int ret;
-
- priv = kzalloc(sizeof(*priv), GFP_KERNEL);
- if (!priv)
- return -ENOMEM;
-
- priv->encoder = &encoder_slave->base;
-
- ret = tda998x_create(client, priv);
- if (ret) {
- kfree(priv);
- return ret;
- }
-
- encoder_slave->slave_priv = priv;
- encoder_slave->slave_funcs = &tda998x_encoder_slave_funcs;
-
- return 0;
-}
-
-struct tda998x_priv2 {
- struct tda998x_priv base;
- struct drm_encoder encoder;
- struct drm_connector connector;
-};
-
-#define conn_to_tda998x_priv2(x) \
- container_of(x, struct tda998x_priv2, connector);
-
-#define enc_to_tda998x_priv2(x) \
- container_of(x, struct tda998x_priv2, encoder);
-
-static void tda998x_encoder2_dpms(struct drm_encoder *encoder, int mode)
-{
- struct tda998x_priv2 *priv = enc_to_tda998x_priv2(encoder);
-
- tda998x_encoder_dpms(&priv->base, mode);
-}
-
static void tda998x_encoder_prepare(struct drm_encoder *encoder)
{
- tda998x_encoder2_dpms(encoder, DRM_MODE_DPMS_OFF);
+ tda998x_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
}
static void tda998x_encoder_commit(struct drm_encoder *encoder)
{
- tda998x_encoder2_dpms(encoder, DRM_MODE_DPMS_ON);
-}
-
-static void tda998x_encoder2_mode_set(struct drm_encoder *encoder,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
-{
- struct tda998x_priv2 *priv = enc_to_tda998x_priv2(encoder);
-
- tda998x_encoder_mode_set(&priv->base, mode, adjusted_mode);
+ tda998x_encoder_dpms(encoder, DRM_MODE_DPMS_ON);
}
static const struct drm_encoder_helper_funcs tda998x_encoder_helper_funcs = {
- .dpms = tda998x_encoder2_dpms,
+ .dpms = tda998x_encoder_dpms,
.save = tda998x_encoder_save,
.restore = tda998x_encoder_restore,
.mode_fixup = tda998x_encoder_mode_fixup,
.prepare = tda998x_encoder_prepare,
.commit = tda998x_encoder_commit,
- .mode_set = tda998x_encoder2_mode_set,
+ .mode_set = tda998x_encoder_mode_set,
};
static void tda998x_encoder_destroy(struct drm_encoder *encoder)
{
- struct tda998x_priv2 *priv = enc_to_tda998x_priv2(encoder);
+ struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
- tda998x_destroy(&priv->base);
+ tda998x_destroy(priv);
drm_encoder_cleanup(encoder);
}
.destroy = tda998x_encoder_destroy,
};
-static int tda998x_connector_get_modes(struct drm_connector *connector)
-{
- struct tda998x_priv2 *priv = conn_to_tda998x_priv2(connector);
-
- return tda998x_encoder_get_modes(&priv->base, connector);
-}
-
-static int tda998x_connector_mode_valid(struct drm_connector *connector,
- struct drm_display_mode *mode)
-{
- struct tda998x_priv2 *priv = conn_to_tda998x_priv2(connector);
-
- return tda998x_encoder_mode_valid(&priv->base, mode);
-}
-
static struct drm_encoder *
tda998x_connector_best_encoder(struct drm_connector *connector)
{
- struct tda998x_priv2 *priv = conn_to_tda998x_priv2(connector);
+ struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
return &priv->encoder;
}
.best_encoder = tda998x_connector_best_encoder,
};
-static enum drm_connector_status
-tda998x_connector_detect(struct drm_connector *connector, bool force)
-{
- struct tda998x_priv2 *priv = conn_to_tda998x_priv2(connector);
-
- return tda998x_encoder_detect(&priv->base);
-}
-
static void tda998x_connector_destroy(struct drm_connector *connector)
{
drm_connector_unregister(connector);
struct tda998x_encoder_params *params = dev->platform_data;
struct i2c_client *client = to_i2c_client(dev);
struct drm_device *drm = data;
- struct tda998x_priv2 *priv;
- uint32_t crtcs = 0;
+ struct tda998x_priv *priv;
+ u32 crtcs = 0;
int ret;
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
crtcs = 1 << 0;
}
- priv->base.encoder = &priv->encoder;
priv->connector.interlace_allowed = 1;
priv->encoder.possible_crtcs = crtcs;
- ret = tda998x_create(client, &priv->base);
+ ret = tda998x_create(client, priv);
if (ret)
return ret;
if (!dev->of_node && params)
- tda998x_encoder_set_config(&priv->base, params);
+ tda998x_encoder_set_config(priv, params);
- tda998x_encoder_set_polling(&priv->base, &priv->connector);
+ tda998x_encoder_set_polling(priv, &priv->connector);
drm_encoder_helper_add(&priv->encoder, &tda998x_encoder_helper_funcs);
ret = drm_encoder_init(drm, &priv->encoder, &tda998x_encoder_funcs,
err_connector:
drm_encoder_cleanup(&priv->encoder);
err_encoder:
- tda998x_destroy(&priv->base);
+ tda998x_destroy(priv);
return ret;
}
static void tda998x_unbind(struct device *dev, struct device *master,
void *data)
{
- struct tda998x_priv2 *priv = dev_get_drvdata(dev);
+ struct tda998x_priv *priv = dev_get_drvdata(dev);
drm_connector_cleanup(&priv->connector);
drm_encoder_cleanup(&priv->encoder);
- tda998x_destroy(&priv->base);
+ tda998x_destroy(priv);
}
static const struct component_ops tda998x_ops = {
};
MODULE_DEVICE_TABLE(i2c, tda998x_ids);
-static struct drm_i2c_encoder_driver tda998x_driver = {
- .i2c_driver = {
- .probe = tda998x_probe,
- .remove = tda998x_remove,
- .driver = {
- .name = "tda998x",
- .of_match_table = of_match_ptr(tda998x_dt_ids),
- },
- .id_table = tda998x_ids,
+static struct i2c_driver tda998x_driver = {
+ .probe = tda998x_probe,
+ .remove = tda998x_remove,
+ .driver = {
+ .name = "tda998x",
+ .of_match_table = of_match_ptr(tda998x_dt_ids),
},
- .encoder_init = tda998x_encoder_init,
+ .id_table = tda998x_ids,
};
-/* Module initialization */
-
-static int __init
-tda998x_init(void)
-{
- DBG("");
- return drm_i2c_encoder_register(THIS_MODULE, &tda998x_driver);
-}
-
-static void __exit
-tda998x_exit(void)
-{
- DBG("");
- drm_i2c_encoder_unregister(&tda998x_driver);
-}
+module_i2c_driver(tda998x_driver);
MODULE_AUTHOR("Rob Clark <robdclark@gmail.com");
MODULE_DESCRIPTION("NXP Semiconductors TDA998X HDMI Encoder");
MODULE_LICENSE("GPL");
-
-module_init(tda998x_init);
-module_exit(tda998x_exit);
intel_ringbuffer.o \
intel_uncore.o
+# general-purpose microcontroller (GuC) support
+i915-y += intel_guc_loader.o \
+ i915_guc_submission.o
+
# autogenerated null render state
i915-y += intel_renderstate_gen6.o \
intel_renderstate_gen7.o \
* after this function is called.
*/
void (*mode_set)(struct intel_dvo_device *dvo,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode);
+ const struct drm_display_mode *mode,
+ const struct drm_display_mode *adjusted_mode);
/*
* Probe for a connected output, and return detect_status.
}
static void ch7017_mode_set(struct intel_dvo_device *dvo,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *mode,
+ const struct drm_display_mode *adjusted_mode)
{
uint8_t lvds_pll_feedback_div, lvds_pll_vco_control;
uint8_t outputs_enable, lvds_control_2, lvds_power_down;
}
static void ch7xxx_mode_set(struct intel_dvo_device *dvo,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *mode,
+ const struct drm_display_mode *adjusted_mode)
{
uint8_t tvco, tpcp, tpd, tlpf, idf;
}
static void ivch_mode_set(struct intel_dvo_device *dvo,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *mode,
+ const struct drm_display_mode *adjusted_mode)
{
struct ivch_priv *priv = dvo->dev_priv;
uint16_t vr40 = 0;
vr40 = (VR40_STALL_ENABLE | VR40_VERTICAL_INTERP_ENABLE |
VR40_HORIZONTAL_INTERP_ENABLE);
- if (mode->hdisplay != adjusted_mode->hdisplay ||
- mode->vdisplay != adjusted_mode->vdisplay) {
+ if (mode->hdisplay != adjusted_mode->crtc_hdisplay ||
+ mode->vdisplay != adjusted_mode->crtc_vdisplay) {
uint16_t x_ratio, y_ratio;
vr01 |= VR01_PANEL_FIT_ENABLE;
vr40 |= VR40_CLOCK_GATING_ENABLE;
x_ratio = (((mode->hdisplay - 1) << 16) /
- (adjusted_mode->hdisplay - 1)) >> 2;
+ (adjusted_mode->crtc_hdisplay - 1)) >> 2;
y_ratio = (((mode->vdisplay - 1) << 16) /
- (adjusted_mode->vdisplay - 1)) >> 2;
+ (adjusted_mode->crtc_vdisplay - 1)) >> 2;
ivch_write(dvo, VR42, x_ratio);
ivch_write(dvo, VR41, y_ratio);
} else {
}
static void ns2501_mode_set(struct intel_dvo_device *dvo,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *mode,
+ const struct drm_display_mode *adjusted_mode)
{
const struct ns2501_configuration *conf;
struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv);
}
static void sil164_mode_set(struct intel_dvo_device *dvo,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *mode,
+ const struct drm_display_mode *adjusted_mode)
{
/* As long as the basics are set up, since we don't have clock
* dependencies in the mode setup, we can just leave the
}
static void tfp410_mode_set(struct intel_dvo_device *dvo,
- struct drm_display_mode *mode,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *mode,
+ const struct drm_display_mode *adjusted_mode)
{
/* As long as the basics are set up, since we don't have clock dependencies
* in the mode setup, we can just leave the registers alone and everything
#define CMD(op, opm, f, lm, fl, ...) \
{ \
.flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \
- .cmd = { (op), (opm) }, \
+ .cmd = { (op), (opm) }, \
.length = { (lm) }, \
__VA_ARGS__ \
}
CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ),
CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W,
.reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ),
- CMD( MI_STORE_REGISTER_MEM(1), SMI, !F, 0xFF, W | B,
+ CMD( MI_STORE_REGISTER_MEM, SMI, F, 3, W | B,
.reg = { .offset = 1, .mask = 0x007FFFFC },
.bits = {{
.offset = 0,
.mask = MI_GLOBAL_GTT,
.expected = 0,
}}, ),
- CMD( MI_LOAD_REGISTER_MEM(1), SMI, !F, 0xFF, W | B,
+ CMD( MI_LOAD_REGISTER_MEM, SMI, F, 3, W | B,
.reg = { .offset = 1, .mask = 0x007FFFFC },
.bits = {{
.offset = 0,
REG32(GEN7_3DPRIM_INSTANCE_COUNT),
REG32(GEN7_3DPRIM_START_INSTANCE),
REG32(GEN7_3DPRIM_BASE_VERTEX),
+ REG32(GEN7_GPGPU_DISPATCHDIMX),
+ REG32(GEN7_GPGPU_DISPATCHDIMY),
+ REG32(GEN7_GPGPU_DISPATCHDIMZ),
REG64(GEN7_SO_NUM_PRIMS_WRITTEN(0)),
REG64(GEN7_SO_NUM_PRIMS_WRITTEN(1)),
REG64(GEN7_SO_NUM_PRIMS_WRITTEN(2)),
* only MI_LOAD_REGISTER_IMM commands.
*/
if (reg_addr == OACONTROL) {
- if (desc->cmd.value == MI_LOAD_REGISTER_MEM(1)) {
+ if (desc->cmd.value == MI_LOAD_REGISTER_MEM) {
DRM_DEBUG_DRIVER("CMD: Rejected LRM to OACONTROL\n");
return false;
}
* allowed mask/value pair given in the whitelist entry.
*/
if (reg->mask) {
- if (desc->cmd.value == MI_LOAD_REGISTER_MEM(1)) {
+ if (desc->cmd.value == MI_LOAD_REGISTER_MEM) {
DRM_DEBUG_DRIVER("CMD: Rejected LRM to masked register 0x%08X\n",
reg_addr);
return false;
* 2. Allow access to the MI_PREDICATE_SRC0 and
* MI_PREDICATE_SRC1 registers.
* 3. Allow access to the GPGPU_THREADS_DISPATCHED register.
+ * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3.
+ * 5. GPGPU dispatch compute indirect registers.
*/
- return 3;
+ return 5;
}
PINNED_LIST,
};
-static const char *yesno(int v)
-{
- return v ? "yes" : "no";
-}
-
/* As the drm_debugfs_init() routines are called before dev->dev_private is
* allocated we need to hook into the minor for release. */
static int
struct drm_i915_gem_object *b =
container_of(B, struct drm_i915_gem_object, obj_exec_link);
- return a->stolen->start - b->stolen->start;
+ if (a->stolen->start < b->stolen->start)
+ return -1;
+ if (a->stolen->start > b->stolen->start)
+ return 1;
+ return 0;
}
static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
if (ret)
return ret;
- seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start);
seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
for (i = 0; i < dev_priv->num_fence_regs; i++) {
struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
seq_puts(m, "no P-state info available\n");
}
+ seq_printf(m, "Current CD clock frequency: %d kHz\n", dev_priv->cdclk_freq);
+ seq_printf(m, "Max CD clock frequency: %d kHz\n", dev_priv->max_cdclk_freq);
+ seq_printf(m, "Max pixel clock frequency: %d kHz\n", dev_priv->max_dotclk_freq);
+
out:
intel_runtime_pm_put(dev_priv);
return ret;
intel_runtime_pm_put(dev_priv);
mutex_unlock(&dev->struct_mutex);
- seq_printf(m, "HD boost: %s\n", (rgvmodectl & MEMMODE_BOOST_EN) ?
- "yes" : "no");
+ seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN));
seq_printf(m, "Boost freq: %d\n",
(rgvmodectl & MEMMODE_BOOST_FREQ_MASK) >>
MEMMODE_BOOST_FREQ_SHIFT);
seq_printf(m, "HW control enabled: %s\n",
- rgvmodectl & MEMMODE_HWIDLE_EN ? "yes" : "no");
+ yesno(rgvmodectl & MEMMODE_HWIDLE_EN));
seq_printf(m, "SW control enabled: %s\n",
- rgvmodectl & MEMMODE_SWMODE_EN ? "yes" : "no");
+ yesno(rgvmodectl & MEMMODE_SWMODE_EN));
seq_printf(m, "Gated voltage change: %s\n",
- rgvmodectl & MEMMODE_RCLK_GATE ? "yes" : "no");
+ yesno(rgvmodectl & MEMMODE_RCLK_GATE));
seq_printf(m, "Starting frequency: P%d\n",
(rgvmodectl & MEMMODE_FSTART_MASK) >> MEMMODE_FSTART_SHIFT);
seq_printf(m, "Max P-state: P%d\n",
seq_printf(m, "RS1 VID: %d\n", (crstandvid & 0x3f));
seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f));
seq_printf(m, "Render standby enabled: %s\n",
- (rstdbyctl & RCX_SW_EXIT) ? "no" : "yes");
+ yesno(!(rstdbyctl & RCX_SW_EXIT)));
seq_puts(m, "Current RS state: ");
switch (rstdbyctl & RSX_STATUS_MASK) {
case RSX_STATUS_ON:
goto out;
if (opregion->header) {
- memcpy_fromio(data, opregion->header, OPREGION_SIZE);
+ memcpy(data, opregion->header, OPREGION_SIZE);
seq_write(m, data, OPREGION_SIZE);
}
return;
}
- page = i915_gem_object_get_page(ctx_obj, 1);
+ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
if (!WARN_ON(page == NULL)) {
reg_state = kmap_atomic(page);
seq_printf(m, "%s\n", ring->name);
- status = I915_READ(RING_EXECLIST_STATUS(ring));
- ctx_id = I915_READ(RING_EXECLIST_STATUS(ring) + 4);
+ status = I915_READ(RING_EXECLIST_STATUS_LO(ring));
+ ctx_id = I915_READ(RING_EXECLIST_STATUS_HI(ring));
seq_printf(m, "\tExeclist status: 0x%08X, context: %u\n",
status, ctx_id);
read_pointer, write_pointer);
for (i = 0; i < 6; i++) {
- status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 8*i);
- ctx_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 8*i + 4);
+ status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, i));
+ ctx_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(ring, i));
seq_printf(m, "\tStatus buffer %d: 0x%08X, context: %u\n",
i, status, ctx_id);
for_each_ring(ring, dev_priv, unused) {
seq_printf(m, "%s\n", ring->name);
for (i = 0; i < 4; i++) {
- u32 offset = 0x270 + i * 8;
- u64 pdp = I915_READ(ring->mmio_base + offset + 4);
+ u64 pdp = I915_READ(GEN8_RING_PDP_UDW(ring, i));
pdp <<= 32;
- pdp |= I915_READ(ring->mmio_base + offset);
+ pdp |= I915_READ(GEN8_RING_PDP_LDW(ring, i));
seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp);
}
}
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_engine_cs *ring;
- struct drm_file *file;
int i;
if (INTEL_INFO(dev)->gen == 6)
ppgtt->debug_dump(ppgtt, m);
}
- list_for_each_entry_reverse(file, &dev->filelist, lhead) {
- struct drm_i915_file_private *file_priv = file->driver_priv;
-
- seq_printf(m, "proc: %s\n",
- get_pid_task(file->pid, PIDTYPE_PID)->comm);
- idr_for_each(&file_priv->context_idr, per_file_ctx, m);
- }
seq_printf(m, "ECOCHK: 0x%08x\n", I915_READ(GAM_ECOCHK));
}
struct drm_info_node *node = m->private;
struct drm_device *dev = node->minor->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_file *file;
int ret = mutex_lock_interruptible(&dev->struct_mutex);
if (ret)
else if (INTEL_INFO(dev)->gen >= 6)
gen6_ppgtt_info(m, dev);
+ list_for_each_entry_reverse(file, &dev->filelist, lhead) {
+ struct drm_i915_file_private *file_priv = file->driver_priv;
+ struct task_struct *task;
+
+ task = get_pid_task(file->pid, PIDTYPE_PID);
+ if (!task) {
+ ret = -ESRCH;
+ goto out_put;
+ }
+ seq_printf(m, "\nproc: %s\n", task->comm);
+ put_task_struct(task);
+ idr_for_each(&file_priv->context_idr, per_file_ctx,
+ (void *)(unsigned long)m);
+ }
+
+out_put:
intel_runtime_pm_put(dev_priv);
mutex_unlock(&dev->struct_mutex);
- return 0;
+ return ret;
}
static int count_irq_waiters(struct drm_i915_private *i915)
return 0;
}
+static int i915_guc_load_status_info(struct seq_file *m, void *data)
+{
+ struct drm_info_node *node = m->private;
+ struct drm_i915_private *dev_priv = node->minor->dev->dev_private;
+ struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
+ u32 tmp, i;
+
+ if (!HAS_GUC_UCODE(dev_priv->dev))
+ return 0;
+
+ seq_printf(m, "GuC firmware status:\n");
+ seq_printf(m, "\tpath: %s\n",
+ guc_fw->guc_fw_path);
+ seq_printf(m, "\tfetch: %s\n",
+ intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status));
+ seq_printf(m, "\tload: %s\n",
+ intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
+ seq_printf(m, "\tversion wanted: %d.%d\n",
+ guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted);
+ seq_printf(m, "\tversion found: %d.%d\n",
+ guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found);
+
+ tmp = I915_READ(GUC_STATUS);
+
+ seq_printf(m, "\nGuC status 0x%08x:\n", tmp);
+ seq_printf(m, "\tBootrom status = 0x%x\n",
+ (tmp & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT);
+ seq_printf(m, "\tuKernel status = 0x%x\n",
+ (tmp & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT);
+ seq_printf(m, "\tMIA Core status = 0x%x\n",
+ (tmp & GS_MIA_MASK) >> GS_MIA_SHIFT);
+ seq_puts(m, "\nScratch registers:\n");
+ for (i = 0; i < 16; i++)
+ seq_printf(m, "\t%2d: \t0x%x\n", i, I915_READ(SOFT_SCRATCH(i)));
+
+ return 0;
+}
+
+static void i915_guc_client_info(struct seq_file *m,
+ struct drm_i915_private *dev_priv,
+ struct i915_guc_client *client)
+{
+ struct intel_engine_cs *ring;
+ uint64_t tot = 0;
+ uint32_t i;
+
+ seq_printf(m, "\tPriority %d, GuC ctx index: %u, PD offset 0x%x\n",
+ client->priority, client->ctx_index, client->proc_desc_offset);
+ seq_printf(m, "\tDoorbell id %d, offset: 0x%x, cookie 0x%x\n",
+ client->doorbell_id, client->doorbell_offset, client->cookie);
+ seq_printf(m, "\tWQ size %d, offset: 0x%x, tail %d\n",
+ client->wq_size, client->wq_offset, client->wq_tail);
+
+ seq_printf(m, "\tFailed to queue: %u\n", client->q_fail);
+ seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail);
+ seq_printf(m, "\tLast submission result: %d\n", client->retcode);
+
+ for_each_ring(ring, dev_priv, i) {
+ seq_printf(m, "\tSubmissions: %llu %s\n",
+ client->submissions[i],
+ ring->name);
+ tot += client->submissions[i];
+ }
+ seq_printf(m, "\tTotal: %llu\n", tot);
+}
+
+static int i915_guc_info(struct seq_file *m, void *data)
+{
+ struct drm_info_node *node = m->private;
+ struct drm_device *dev = node->minor->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc guc;
+ struct i915_guc_client client = {};
+ struct intel_engine_cs *ring;
+ enum intel_ring_id i;
+ u64 total = 0;
+
+ if (!HAS_GUC_SCHED(dev_priv->dev))
+ return 0;
+
+ /* Take a local copy of the GuC data, so we can dump it at leisure */
+ spin_lock(&dev_priv->guc.host2guc_lock);
+ guc = dev_priv->guc;
+ if (guc.execbuf_client) {
+ spin_lock(&guc.execbuf_client->wq_lock);
+ client = *guc.execbuf_client;
+ spin_unlock(&guc.execbuf_client->wq_lock);
+ }
+ spin_unlock(&dev_priv->guc.host2guc_lock);
+
+ seq_printf(m, "GuC total action count: %llu\n", guc.action_count);
+ seq_printf(m, "GuC action failure count: %u\n", guc.action_fail);
+ seq_printf(m, "GuC last action command: 0x%x\n", guc.action_cmd);
+ seq_printf(m, "GuC last action status: 0x%x\n", guc.action_status);
+ seq_printf(m, "GuC last action error code: %d\n", guc.action_err);
+
+ seq_printf(m, "\nGuC submissions:\n");
+ for_each_ring(ring, dev_priv, i) {
+ seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x %9d\n",
+ ring->name, guc.submissions[i],
+ guc.last_seqno[i], guc.last_seqno[i]);
+ total += guc.submissions[i];
+ }
+ seq_printf(m, "\t%s: %llu\n", "Total", total);
+
+ seq_printf(m, "\nGuC execbuf client @ %p:\n", guc.execbuf_client);
+ i915_guc_client_info(m, dev_priv, &client);
+
+ /* Add more as required ... */
+
+ return 0;
+}
+
+static int i915_guc_log_dump(struct seq_file *m, void *data)
+{
+ struct drm_info_node *node = m->private;
+ struct drm_device *dev = node->minor->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_gem_object *log_obj = dev_priv->guc.log_obj;
+ u32 *log;
+ int i = 0, pg;
+
+ if (!log_obj)
+ return 0;
+
+ for (pg = 0; pg < log_obj->base.size / PAGE_SIZE; pg++) {
+ log = kmap_atomic(i915_gem_object_get_page(log_obj, pg));
+
+ for (i = 0; i < PAGE_SIZE / sizeof(u32); i += 4)
+ seq_printf(m, "0x%08x 0x%08x 0x%08x 0x%08x\n",
+ *(log + i), *(log + i + 1),
+ *(log + i + 2), *(log + i + 3));
+
+ kunmap_atomic(log);
+ }
+
+ seq_putc(m, '\n');
+
+ return 0;
+}
+
static int i915_edp_psr_status(struct seq_file *m, void *data)
{
struct drm_info_node *node = m->private;
struct drm_device *dev = node->minor->dev;
struct drm_crtc *crtc = &intel_crtc->base;
struct intel_encoder *intel_encoder;
+ struct drm_plane_state *plane_state = crtc->primary->state;
+ struct drm_framebuffer *fb = plane_state->fb;
- if (crtc->primary->fb)
+ if (fb)
seq_printf(m, "\tfb: %d, pos: %dx%d, size: %dx%d\n",
- crtc->primary->fb->base.id, crtc->x, crtc->y,
- crtc->primary->fb->width, crtc->primary->fb->height);
+ fb->base.id, plane_state->src_x >> 16,
+ plane_state->src_y >> 16, fb->width, fb->height);
else
seq_puts(m, "\tprimary plane disabled\n");
for_each_encoder_on_crtc(dev, crtc, intel_encoder)
struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base);
seq_printf(m, "\tDPCD rev: %x\n", intel_dp->dpcd[DP_DPCD_REV]);
- seq_printf(m, "\taudio support: %s\n", intel_dp->has_audio ? "yes" :
- "no");
+ seq_printf(m, "\taudio support: %s\n", yesno(intel_dp->has_audio));
if (intel_encoder->type == INTEL_OUTPUT_EDP)
intel_panel_info(m, &intel_connector->panel);
}
struct intel_encoder *intel_encoder = intel_connector->encoder;
struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
- seq_printf(m, "\taudio support: %s\n", intel_hdmi->has_audio ? "yes" :
- "no");
+ seq_printf(m, "\taudio support: %s\n", yesno(intel_hdmi->has_audio));
}
static void intel_lvds_info(struct seq_file *m,
u32 state;
if (IS_845G(dev) || IS_I865G(dev))
- state = I915_READ(_CURACNTR) & CURSOR_ENABLE;
+ state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
else
state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
skl_ddb_entry_size(entry));
}
- entry = &ddb->cursor[pipe];
+ entry = &ddb->plane[pipe][PLANE_CURSOR];
seq_printf(m, " %-13s%8u%8u%8u\n", "Cursor", entry->start,
entry->end, skl_ddb_entry_size(entry));
}
struct sseu_dev_status *stat)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- const int ss_max = 2;
+ int ss_max = 2;
int ss;
u32 sig1[ss_max], sig2[ss_max];
}
}
+static void broadwell_sseu_device_status(struct drm_device *dev,
+ struct sseu_dev_status *stat)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int s;
+ u32 slice_info = I915_READ(GEN8_GT_SLICE_INFO);
+
+ stat->slice_total = hweight32(slice_info & GEN8_LSLICESTAT_MASK);
+
+ if (stat->slice_total) {
+ stat->subslice_per_slice = INTEL_INFO(dev)->subslice_per_slice;
+ stat->subslice_total = stat->slice_total *
+ stat->subslice_per_slice;
+ stat->eu_per_subslice = INTEL_INFO(dev)->eu_per_subslice;
+ stat->eu_total = stat->eu_per_subslice * stat->subslice_total;
+
+ /* subtract fused off EU(s) from enabled slice(s) */
+ for (s = 0; s < stat->slice_total; s++) {
+ u8 subslice_7eu = INTEL_INFO(dev)->subslice_7eu[s];
+
+ stat->eu_total -= hweight8(subslice_7eu);
+ }
+ }
+}
+
static int i915_sseu_status(struct seq_file *m, void *unused)
{
struct drm_info_node *node = (struct drm_info_node *) m->private;
struct drm_device *dev = node->minor->dev;
struct sseu_dev_status stat;
- if ((INTEL_INFO(dev)->gen < 8) || IS_BROADWELL(dev))
+ if (INTEL_INFO(dev)->gen < 8)
return -ENODEV;
seq_puts(m, "SSEU Device Info\n");
memset(&stat, 0, sizeof(stat));
if (IS_CHERRYVIEW(dev)) {
cherryview_sseu_device_status(dev, &stat);
+ } else if (IS_BROADWELL(dev)) {
+ broadwell_sseu_device_status(dev, &stat);
} else if (INTEL_INFO(dev)->gen >= 9) {
gen9_sseu_device_status(dev, &stat);
}
{"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
{"i915_gem_hws_vebox", i915_hws_info, 0, (void *)VECS},
{"i915_gem_batch_pool", i915_gem_batch_pool_info, 0},
+ {"i915_guc_info", i915_guc_info, 0},
+ {"i915_guc_load_status", i915_guc_load_status_info, 0},
+ {"i915_guc_log_dump", i915_guc_log_dump, 0},
{"i915_frequency_info", i915_frequency_info, 0},
{"i915_hangcheck_info", i915_hangcheck_info, 0},
{"i915_drpc_info", i915_drpc_info, 0},
value = 1;
break;
case I915_PARAM_NUM_FENCES_AVAIL:
- value = dev_priv->num_fence_regs - dev_priv->fence_reg_start;
+ value = dev_priv->num_fence_regs;
break;
case I915_PARAM_HAS_OVERLAY:
value = dev_priv->overlay ? 1 : 0;
return 0;
}
-static int i915_setparam(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- drm_i915_setparam_t *param = data;
-
- switch (param->param) {
- case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
- case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
- case I915_SETPARAM_ALLOW_BATCHBUFFER:
- /* Reject all old ums/dri params. */
- return -ENODEV;
-
- case I915_SETPARAM_NUM_USED_FENCES:
- if (param->value > dev_priv->num_fence_regs ||
- param->value < 0)
- return -EINVAL;
- /* Userspace can use first N regs */
- dev_priv->fence_reg_start = param->value;
- break;
- default:
- DRM_DEBUG_DRIVER("unknown parameter %d\n",
- param->param);
- return -EINVAL;
- }
-
- return 0;
-}
-
static int i915_get_bridge_dev(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
/* i915 resume handler doesn't set to D0 */
pci_set_power_state(dev->pdev, PCI_D0);
- i915_resume_legacy(dev);
+ i915_resume_switcheroo(dev);
dev->switch_power_state = DRM_SWITCH_POWER_ON;
} else {
pr_err("switched off\n");
dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
- i915_suspend_legacy(dev, pmm);
+ i915_suspend_switcheroo(dev, pmm);
dev->switch_power_state = DRM_SWITCH_POWER_OFF;
}
}
* working irqs for e.g. gmbus and dp aux transfers. */
intel_modeset_init(dev);
+ intel_guc_ucode_init(dev);
+
ret = i915_gem_init(dev);
if (ret)
goto cleanup_irq;
i915_gem_context_fini(dev);
mutex_unlock(&dev->struct_mutex);
cleanup_irq:
+ intel_guc_ucode_fini(dev);
drm_irq_uninstall(dev);
cleanup_gem_stolen:
i915_gem_cleanup_stolen(dev);
u32 fuse2, s_enable, ss_disable, eu_disable;
u8 eu_mask = 0xff;
- /*
- * BXT has a single slice. BXT also has at most 6 EU per subslice,
- * and therefore only the lowest 6 bits of the 8-bit EU disable
- * fields are valid.
- */
- if (IS_BROXTON(dev)) {
- s_max = 1;
- eu_max = 6;
- eu_mask = 0x3f;
- }
-
info = (struct intel_device_info *)&dev_priv->info;
fuse2 = I915_READ(GEN8_FUSE2);
s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >>
info->has_eu_pg = (info->eu_per_subslice > 2);
}
+static void broadwell_sseu_info_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_device_info *info;
+ const int s_max = 3, ss_max = 3, eu_max = 8;
+ int s, ss;
+ u32 fuse2, eu_disable[s_max], s_enable, ss_disable;
+
+ fuse2 = I915_READ(GEN8_FUSE2);
+ s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
+ ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT;
+
+ eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK;
+ eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) |
+ ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) <<
+ (32 - GEN8_EU_DIS0_S1_SHIFT));
+ eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) |
+ ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) <<
+ (32 - GEN8_EU_DIS1_S2_SHIFT));
+
+
+ info = (struct intel_device_info *)&dev_priv->info;
+ info->slice_total = hweight32(s_enable);
+
+ /*
+ * The subslice disable field is global, i.e. it applies
+ * to each of the enabled slices.
+ */
+ info->subslice_per_slice = ss_max - hweight32(ss_disable);
+ info->subslice_total = info->slice_total * info->subslice_per_slice;
+
+ /*
+ * Iterate through enabled slices and subslices to
+ * count the total enabled EU.
+ */
+ for (s = 0; s < s_max; s++) {
+ if (!(s_enable & (0x1 << s)))
+ /* skip disabled slice */
+ continue;
+
+ for (ss = 0; ss < ss_max; ss++) {
+ u32 n_disabled;
+
+ if (ss_disable & (0x1 << ss))
+ /* skip disabled subslice */
+ continue;
+
+ n_disabled = hweight8(eu_disable[s] >> (ss * eu_max));
+
+ /*
+ * Record which subslices have 7 EUs.
+ */
+ if (eu_max - n_disabled == 7)
+ info->subslice_7eu[s] |= 1 << ss;
+
+ info->eu_total += eu_max - n_disabled;
+ }
+ }
+
+ /*
+ * BDW is expected to always have a uniform distribution of EU across
+ * subslices with the exception that any one EU in any one subslice may
+ * be fused off for die recovery.
+ */
+ info->eu_per_subslice = info->subslice_total ?
+ DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0;
+
+ /*
+ * BDW supports slice power gating on devices with more than
+ * one slice.
+ */
+ info->has_slice_pg = (info->slice_total > 1);
+ info->has_subslice_pg = 0;
+ info->has_eu_pg = 0;
+}
+
/*
* Determine various intel_device_info fields at runtime.
*
/* Initialize slice/subslice/EU info */
if (IS_CHERRYVIEW(dev))
cherryview_sseu_info_init(dev);
+ else if (IS_BROADWELL(dev))
+ broadwell_sseu_info_init(dev);
else if (INTEL_INFO(dev)->gen >= 9)
gen9_sseu_info_init(dev);
info->has_eu_pg ? "y" : "n");
}
+static void intel_init_dpio(struct drm_i915_private *dev_priv)
+{
+ if (!IS_VALLEYVIEW(dev_priv))
+ return;
+
+ /*
+ * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
+ * CHV x1 PHY (DP/HDMI D)
+ * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
+ */
+ if (IS_CHERRYVIEW(dev_priv)) {
+ DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
+ DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
+ } else {
+ DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
+ }
+}
+
/**
* i915_driver_load - setup chip and create an initial config
* @dev: DRM device
mutex_init(&dev_priv->sb_lock);
mutex_init(&dev_priv->modeset_restore_lock);
mutex_init(&dev_priv->csr_lock);
+ mutex_init(&dev_priv->av_mutex);
intel_pm_setup(dev);
intel_setup_gmbus(dev);
intel_opregion_setup(dev);
- intel_setup_bios(dev);
-
i915_gem_load(dev);
/* On the 945G/GM, the chipset reports the MSI capability on the
intel_device_info_runtime_init(dev);
+ intel_init_dpio(dev_priv);
+
if (INTEL_INFO(dev)->num_pipes) {
ret = drm_vblank_init(dev, INTEL_INFO(dev)->num_pipes);
if (ret)
put_bridge:
pci_dev_put(dev_priv->bridge_dev);
free_priv:
- if (dev_priv->requests)
- kmem_cache_destroy(dev_priv->requests);
- if (dev_priv->vmas)
- kmem_cache_destroy(dev_priv->vmas);
- if (dev_priv->objects)
- kmem_cache_destroy(dev_priv->objects);
+ kmem_cache_destroy(dev_priv->requests);
+ kmem_cache_destroy(dev_priv->vmas);
+ kmem_cache_destroy(dev_priv->objects);
kfree(dev_priv);
return ret;
}
dev_priv->vbt.child_dev = NULL;
dev_priv->vbt.child_dev_num = 0;
}
+ kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
+ dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
+ kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
+ dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
vga_switcheroo_unregister_client(dev->pdev);
vga_client_register(dev->pdev, NULL, NULL, NULL);
/* Flush any outstanding unpin_work. */
flush_workqueue(dev_priv->wq);
+ intel_guc_ucode_fini(dev);
mutex_lock(&dev->struct_mutex);
i915_gem_cleanup_ringbuffer(dev);
i915_gem_context_fini(dev);
if (dev_priv->regs != NULL)
pci_iounmap(dev->pdev, dev_priv->regs);
- if (dev_priv->requests)
- kmem_cache_destroy(dev_priv->requests);
- if (dev_priv->vmas)
- kmem_cache_destroy(dev_priv->vmas);
- if (dev_priv->objects)
- kmem_cache_destroy(dev_priv->objects);
-
+ kmem_cache_destroy(dev_priv->requests);
+ kmem_cache_destroy(dev_priv->vmas);
+ kmem_cache_destroy(dev_priv->objects);
pci_dev_put(dev_priv->bridge_dev);
kfree(dev_priv);
DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH),
DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_get_reset_stats_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
+ DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_get_reset_stats_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
};
int i915_max_ioctl = ARRAY_SIZE(i915_ioctls);
.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
.has_llc = 1,
.has_ddi = 1,
+ .has_fpga_dbg = 1,
.has_fbc = 1,
GEN_DEFAULT_PIPEOFFSETS,
IVB_CURSOR_OFFSETS,
.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
.has_llc = 1,
.has_ddi = 1,
+ .has_fpga_dbg = 1,
.has_fbc = 1,
GEN_DEFAULT_PIPEOFFSETS,
IVB_CURSOR_OFFSETS,
.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
.num_pipes = 3,
.has_ddi = 1,
+ .has_fpga_dbg = 1,
.has_fbc = 1,
GEN_DEFAULT_PIPEOFFSETS,
IVB_CURSOR_OFFSETS,
MODULE_DEVICE_TABLE(pci, pciidlist);
+static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
+{
+ enum intel_pch ret = PCH_NOP;
+
+ /*
+ * In a virtualized passthrough environment we can be in a
+ * setup where the ISA bridge is not able to be passed through.
+ * In this case, a south bridge can be emulated and we have to
+ * make an educated guess as to which PCH is really there.
+ */
+
+ if (IS_GEN5(dev)) {
+ ret = PCH_IBX;
+ DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
+ } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
+ ret = PCH_CPT;
+ DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
+ } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+ ret = PCH_LPT;
+ DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
+ } else if (IS_SKYLAKE(dev)) {
+ ret = PCH_SPT;
+ DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
+ }
+
+ return ret;
+}
+
void intel_detect_pch(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
dev_priv->pch_type = PCH_SPT;
DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
WARN_ON(!IS_SKYLAKE(dev));
+ } else if (id == INTEL_PCH_P2X_DEVICE_ID_TYPE) {
+ dev_priv->pch_type = intel_virt_detect_pch(dev);
} else
continue;
return error;
}
+ intel_guc_suspend(dev);
+
intel_suspend_gt_powersave(dev);
/*
return 0;
}
-int i915_suspend_legacy(struct drm_device *dev, pm_message_t state)
+int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
{
int error;
}
mutex_unlock(&dev->struct_mutex);
+ intel_guc_resume(dev);
+
intel_modeset_init_hw(dev);
spin_lock_irq(&dev_priv->irq_lock);
return ret;
}
-int i915_resume_legacy(struct drm_device *dev)
+int i915_resume_switcheroo(struct drm_device *dev)
{
int ret;
{
/* Enabling DC6 is not a hard requirement to enter runtime D3 */
- /*
- * This is to ensure that CSR isn't identified as loaded before
- * CSR-loading program is called during runtime-resume.
- */
- intel_csr_load_status_set(dev_priv, FW_UNINITIALIZED);
-
skl_uninit_cdclk(dev_priv);
return 0;
s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1);
for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
- s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS_BASE + i * 4);
+ s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
s->pm_ier = I915_READ(GEN6_PMIER);
for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
- s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH_BASE + i * 4);
+ s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
/* GT SA CZ domain, 0x100000-0x138124 */
s->tilectl = I915_READ(TILECTL);
I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
- I915_WRITE(GEN7_LRA_LIMITS_BASE + i * 4, s->lra_limits[i]);
+ I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
I915_WRITE(GEN6_PMIER, s->pm_ier);
for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
- I915_WRITE(GEN7_GT_SCRATCH_BASE + i * 4, s->gt_scratch[i]);
+ I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
/* GT SA CZ domain, 0x100000-0x138124 */
I915_WRITE(TILECTL, s->tilectl);
i915_gem_release_all_mmaps(dev_priv);
mutex_unlock(&dev->struct_mutex);
+ intel_guc_suspend(dev);
+
intel_suspend_gt_powersave(dev);
intel_runtime_pm_disable_interrupts(dev_priv);
intel_opregion_notify_adapter(dev, PCI_D0);
dev_priv->pm.suspended = false;
+ intel_guc_resume(dev);
+
if (IS_GEN6(dev_priv))
intel_init_pch_refclk(dev);
gen6_update_ring_freq(dev);
intel_runtime_pm_enable_interrupts(dev_priv);
+
+ /*
+ * On VLV/CHV display interrupts are part of the display
+ * power well, so hpd is reinitialized from there. For
+ * everyone else do it here.
+ */
+ if (!IS_VALLEYVIEW(dev_priv))
+ intel_hpd_init(dev_priv);
+
intel_enable_gt_powersave(dev);
if (ret)
*/
.driver_features =
DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
- DRIVER_RENDER,
+ DRIVER_RENDER | DRIVER_MODESET,
.load = i915_driver_load,
.unload = i915_driver_unload,
.open = i915_driver_open,
.postclose = i915_driver_postclose,
.set_busid = drm_pci_set_busid,
- /* Used in place of i915_pm_ops for non-DRIVER_MODESET */
- .suspend = i915_suspend_legacy,
- .resume = i915_resume_legacy,
-
#if defined(CONFIG_DEBUG_FS)
.debugfs_init = i915_debugfs_init,
.debugfs_cleanup = i915_debugfs_cleanup,
* either the i915.modeset prarameter or by the
* vga_text_mode_force boot option.
*/
- driver.driver_features |= DRIVER_MODESET;
if (i915.modeset == 0)
driver.driver_features &= ~DRIVER_MODESET;
#endif
if (!(driver.driver_features & DRIVER_MODESET)) {
- driver.get_vblank_timestamp = NULL;
/* Silently fail loading to not upset userspace. */
DRM_DEBUG_DRIVER("KMS and UMS disabled.\n");
return 0;
}
- /*
- * FIXME: Note that we're lying to the DRM core here so that we can get access
- * to the atomic ioctl and the atomic properties. Only plane operations on
- * a single CRTC will actually work.
- */
- if (driver.driver_features & DRIVER_MODESET)
+ if (i915.nuclear_pageflip)
driver.driver_features |= DRIVER_ATOMIC;
return drm_pci_init(&driver, &i915_pci_driver);
#include <linux/intel-iommu.h>
#include <linux/kref.h>
#include <linux/pm_qos.h>
+#include "intel_guc.h"
/* General customization:
*/
#define DRIVER_NAME "i915"
#define DRIVER_DESC "Intel Graphics"
-#define DRIVER_DATE "20150731"
+#define DRIVER_DATE "20151010"
#undef WARN_ON
/* Many gcc seem to no see through this and fall over :( */
BUILD_BUG_ON(__i915_warn_cond); \
WARN(__i915_warn_cond, "WARN_ON(" #x ")"); })
#else
-#define WARN_ON(x) WARN((x), "WARN_ON(" #x ")")
+#define WARN_ON(x) WARN((x), "WARN_ON(%s)", #x )
#endif
#undef WARN_ON_ONCE
-#define WARN_ON_ONCE(x) WARN_ONCE((x), "WARN_ON_ONCE(" #x ")")
+#define WARN_ON_ONCE(x) WARN_ONCE((x), "WARN_ON_ONCE(%s)", #x )
#define MISSING_CASE(x) WARN(1, "Missing switch case (%lu) in %s\n", \
(long) (x), __func__);
unlikely(__ret_warn_on); \
})
+static inline const char *yesno(bool v)
+{
+ return v ? "yes" : "no";
+}
+
enum pipe {
INVALID_PIPE = -1,
PIPE_A = 0,
#define transcoder_name(t) ((t) + 'A')
/*
- * This is the maximum (across all platforms) number of planes (primary +
- * sprites) that can be active at the same time on one pipe.
- *
- * This value doesn't count the cursor plane.
+ * I915_MAX_PLANES in the enum below is the maximum (across all platforms)
+ * number of planes per CRTC. Not all platforms really have this many planes,
+ * which means some arrays of size I915_MAX_PLANES may have unused entries
+ * between the topmost sprite plane and the cursor plane.
*/
-#define I915_MAX_PLANES 4
-
enum plane {
PLANE_A = 0,
PLANE_B,
PLANE_C,
+ PLANE_CURSOR,
+ I915_MAX_PLANES,
};
#define plane_name(p) ((p) + 'A')
struct opregion_asle;
struct intel_opregion {
- struct opregion_header __iomem *header;
- struct opregion_acpi __iomem *acpi;
- struct opregion_swsci __iomem *swsci;
+ struct opregion_header *header;
+ struct opregion_acpi *acpi;
+ struct opregion_swsci *swsci;
u32 swsci_gbda_sub_functions;
u32 swsci_sbcb_sub_functions;
- struct opregion_asle __iomem *asle;
- void __iomem *vbt;
- u32 __iomem *lid_state;
+ struct opregion_asle *asle;
+ void *vbt;
+ u32 *lid_state;
struct work_struct asle_work;
};
#define OPREGION_SIZE (8*1024)
struct drm_i915_error_object {
int page_count;
- u32 gtt_offset;
+ u64 gtt_offset;
u32 *pages[0];
} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
u32 size;
u32 name;
u32 rseqno[I915_NUM_RINGS], wseqno;
- u32 gtt_offset;
+ u64 gtt_offset;
u32 read_domains;
u32 write_domain;
s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
void (*crtc_disable)(struct drm_crtc *crtc);
void (*audio_codec_enable)(struct drm_connector *connector,
struct intel_encoder *encoder,
- struct drm_display_mode *mode);
+ const struct drm_display_mode *adjusted_mode);
void (*audio_codec_disable)(struct intel_encoder *encoder);
void (*fdi_link_train)(struct drm_crtc *crtc);
void (*init_clock_gating)(struct drm_device *dev);
/* render clock increase/decrease */
/* display clock increase/decrease */
/* pll clock increase/decrease */
-
- int (*setup_backlight)(struct intel_connector *connector, enum pipe pipe);
- uint32_t (*get_backlight)(struct intel_connector *connector);
- void (*set_backlight)(struct intel_connector *connector,
- uint32_t level);
- void (*disable_backlight)(struct intel_connector *connector);
- void (*enable_backlight)(struct intel_connector *connector);
};
enum forcewake_domain_id {
} legacy_hw_ctx;
/* Execlists */
- bool rcs_initialized;
struct {
struct drm_i915_gem_object *state;
struct intel_ringbuffer *ringbuf;
FBC_CHIP_DEFAULT, /* disabled by default on this chip */
FBC_ROTATION, /* rotation is not supported */
FBC_IN_DBG_MASTER, /* kernel debugger is active */
+ FBC_BAD_STRIDE, /* stride is not supported */
+ FBC_PIXEL_RATE, /* pixel rate is too big */
+ FBC_PIXEL_FORMAT /* pixel format is invalid */
} no_fbc_reason;
bool (*fbc_enabled)(struct drm_i915_private *dev_priv);
u32 saveMI_ARB_STATE;
u32 saveSWF0[16];
u32 saveSWF1[16];
- u32 saveSWF2[3];
+ u32 saveSWF3[3];
uint64_t saveFENCE[I915_MAX_NUM_FENCES];
u32 savePCH_PORT_HOTPLUG;
u16 saveGCDGMBUS;
u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */
u8 rp1_freq; /* "less than" RP0 power/freqency */
u8 rp0_freq; /* Non-overclocked max frequency. */
- u32 cz_freq;
u8 up_threshold; /* Current %busy required to uplock */
u8 down_threshold; /* Current %busy required to downclock */
struct skl_ddb_allocation {
struct skl_ddb_entry pipe[I915_MAX_PIPES];
struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */
- struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* y-plane */
- struct skl_ddb_entry cursor[I915_MAX_PIPES];
+ struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES];
};
struct skl_wm_values {
struct skl_ddb_allocation ddb;
uint32_t wm_linetime[I915_MAX_PIPES];
uint32_t plane[I915_MAX_PIPES][I915_MAX_PLANES][8];
- uint32_t cursor[I915_MAX_PIPES][8];
uint32_t plane_trans[I915_MAX_PIPES][I915_MAX_PLANES];
- uint32_t cursor_trans[I915_MAX_PIPES];
};
struct skl_wm_level {
bool plane_en[I915_MAX_PLANES];
- bool cursor_en;
uint16_t plane_res_b[I915_MAX_PLANES];
uint8_t plane_res_l[I915_MAX_PLANES];
- uint16_t cursor_res_b;
- uint8_t cursor_res_l;
};
/*
struct drm_file *file;
uint32_t dispatch_flags;
uint32_t args_batch_start_offset;
- uint32_t batch_obj_vm_offset;
+ uint64_t batch_obj_vm_offset;
struct intel_engine_cs *ring;
struct drm_i915_gem_object *batch_obj;
struct intel_context *ctx;
struct i915_virtual_gpu vgpu;
+ struct intel_guc guc;
+
struct intel_csr csr;
/* Display CSR-related protection */
struct mutex pps_mutex;
struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */
- int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */
int num_fence_regs; /* 8 on pre-965, 16 otherwise */
unsigned int fsb_freq, mem_freq, is_ddr3;
unsigned int skl_boot_cdclk;
unsigned int cdclk_freq, max_cdclk_freq;
+ unsigned int max_dotclk_freq;
unsigned int hpll_freq;
+ unsigned int czclk_freq;
/**
* wq - Driver workqueue for GEM.
/* hda/i915 audio component */
struct i915_audio_component *audio_component;
bool audio_component_registered;
+ /**
+ * av_mutex - mutex for audio/video sync
+ *
+ */
+ struct mutex av_mutex;
uint32_t hw_context_size;
struct list_head context_list;
bool edp_low_vswing;
+ /* perform PHY state sanity checks? */
+ bool chv_phy_assert[2];
+
/*
* NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
* will be rejected. Instead look for a better place.
return to_i915(dev_get_drvdata(dev));
}
+static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
+{
+ return container_of(guc, struct drm_i915_private, guc);
+}
+
/* Iterate over initialised rings */
#define for_each_ring(ring__, dev_priv__, i__) \
for ((i__) = 0; (i__) < I915_NUM_RINGS; (i__)++) \
/*
* Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is
- * considered to be the frontbuffer for the given plane interface-vise. This
+ * considered to be the frontbuffer for the given plane interface-wise. This
* doesn't mean that the hw necessarily already scans it out, but that any
* rendering (by the cpu or gpu) will land in the frontbuffer eventually.
*
* We have one bit per pipe and per scanout plane type.
*/
-#define INTEL_FRONTBUFFER_BITS_PER_PIPE 4
+#define INTEL_MAX_SPRITE_BITS_PER_PIPE 5
+#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8
#define INTEL_FRONTBUFFER_BITS \
(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES)
#define INTEL_FRONTBUFFER_PRIMARY(pipe) \
(1 << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
#define INTEL_FRONTBUFFER_CURSOR(pipe) \
- (1 << (1 +(INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
-#define INTEL_FRONTBUFFER_SPRITE(pipe) \
- (1 << (2 +(INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
+ (1 << (1 + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
+#define INTEL_FRONTBUFFER_SPRITE(pipe, plane) \
+ (1 << (2 + plane + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
#define INTEL_FRONTBUFFER_OVERLAY(pipe) \
- (1 << (3 +(INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
+ (1 << (2 + INTEL_MAX_SPRITE_BITS_PER_PIPE + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \
- (0xf << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
+ (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
struct drm_i915_gem_object {
struct drm_gem_object base;
#define IS_SKL_ULX(dev) (INTEL_DEVID(dev) == 0x190E || \
INTEL_DEVID(dev) == 0x1915 || \
INTEL_DEVID(dev) == 0x191E)
+#define IS_SKL_GT3(dev) (IS_SKYLAKE(dev) && \
+ (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
+#define IS_SKL_GT4(dev) (IS_SKYLAKE(dev) && \
+ (INTEL_DEVID(dev) & 0x00F0) == 0x0030)
+
#define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary)
#define SKL_REVID_A0 (0x0)
#define BXT_REVID_A0 (0x0)
#define BXT_REVID_B0 (0x3)
-#define BXT_REVID_C0 (0x6)
+#define BXT_REVID_C0 (0x9)
/*
* The genX designation typically refers to the render engine, so render
#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 6)
#define HAS_LOGICAL_RING_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 8)
#define USES_PPGTT(dev) (i915.enable_ppgtt)
-#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt == 2)
+#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt >= 2)
+#define USES_FULL_48BIT_PPGTT(dev) (i915.enable_ppgtt == 3)
#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay)
#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical)
#define HAS_RC6(dev) (INTEL_INFO(dev)->gen >= 6)
#define HAS_RC6p(dev) (INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev))
-#define HAS_CSR(dev) (IS_SKYLAKE(dev))
+#define HAS_CSR(dev) (IS_GEN9(dev))
+
+#define HAS_GUC_UCODE(dev) (IS_GEN9(dev))
+#define HAS_GUC_SCHED(dev) (IS_GEN9(dev))
#define HAS_RESOURCE_STREAMER(dev) (IS_HASWELL(dev) || \
INTEL_INFO(dev)->gen >= 8)
#define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE 0x9c00
#define INTEL_PCH_SPT_DEVICE_ID_TYPE 0xA100
#define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE 0x9D00
+#define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100
#define INTEL_PCH_TYPE(dev) (__I915__(dev)->pch_type)
#define HAS_PCH_SPT(dev) (INTEL_PCH_TYPE(dev) == PCH_SPT)
#define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT)
+#define HAS_PCH_LPT_LP(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
#define HAS_PCH_NOP(dev) (INTEL_PCH_TYPE(dev) == PCH_NOP)
extern const struct drm_ioctl_desc i915_ioctls[];
extern int i915_max_ioctl;
-extern int i915_suspend_legacy(struct drm_device *dev, pm_message_t state);
-extern int i915_resume_legacy(struct drm_device *dev);
+extern int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state);
+extern int i915_resume_switcheroo(struct drm_device *dev);
/* i915_params.c */
struct i915_params {
int enable_cmd_parser;
/* leave bools at the end to not create holes */
bool enable_hangcheck;
- bool fastboot;
bool prefault_disable;
bool load_detect_test;
bool reset;
int use_mmio_flip;
int mmio_debug;
bool verbose_state_checks;
+ bool nuclear_pageflip;
int edp_vswing;
};
extern struct i915_params i915 __read_mostly;
void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv);
void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv);
+void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
+ uint32_t mask,
+ uint32_t bits);
void
ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask);
void
size_t size);
struct drm_i915_gem_object *i915_gem_object_create_from_data(
struct drm_device *dev, const void *data, size_t size);
-void i915_init_vm(struct drm_i915_private *dev_priv,
- struct i915_address_space *vm);
void i915_gem_free_object(struct drm_gem_object *obj);
void i915_gem_vma_destroy(struct i915_vma *vma);
#define PIN_OFFSET_BIAS (1<<3)
#define PIN_USER (1<<4)
#define PIN_UPDATE (1<<5)
+#define PIN_ZONE_4G (1<<6)
+#define PIN_HIGH (1<<7)
#define PIN_OFFSET_MASK (~4095)
int __must_check
i915_gem_object_pin(struct drm_i915_gem_object *obj,
int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
u32 flags);
int __must_check i915_vma_unbind(struct i915_vma *vma);
+/*
+ * BEWARE: Do not use the function below unless you can _absolutely_
+ * _guarantee_ VMA in question is _not in use_ anywhere.
+ */
+int __must_check __i915_vma_unbind_no_wait(struct i915_vma *vma);
int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
struct drm_gem_object *gem_obj, int flags);
-unsigned long
-i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
- const struct i915_ggtt_view *view);
-unsigned long
-i915_gem_obj_offset(struct drm_i915_gem_object *o,
- struct i915_address_space *vm);
-static inline unsigned long
+u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
+ const struct i915_ggtt_view *view);
+u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
+ struct i915_address_space *vm);
+static inline u64
i915_gem_obj_ggtt_offset(struct drm_i915_gem_object *o)
{
return i915_gem_obj_ggtt_offset_view(o, &i915_ggtt_view_normal);
unsigned long end,
unsigned flags);
int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
-int i915_gem_evict_everything(struct drm_device *dev);
/* belongs in i915_gem_gtt.h */
static inline void i915_gem_chipset_flush(struct drm_device *dev)
int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
struct drm_mm_node *node, u64 size,
unsigned alignment);
+int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
+ struct drm_mm_node *node, u64 size,
+ unsigned alignment, u64 start,
+ u64 end);
void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
struct drm_mm_node *node);
int i915_gem_init_stolen(struct drm_device *dev);
/* i915_gem_shrinker.c */
unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
- long target,
+ unsigned long target,
unsigned flags);
#define I915_SHRINK_PURGEABLE 0x1
#define I915_SHRINK_UNBOUND 0x2
#define I915_SHRINK_BOUND 0x4
+#define I915_SHRINK_ACTIVE 0x8
unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
void i915_gem_shrinker_init(struct drm_i915_private *dev_priv);
if (!needs_clflush_after &&
obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
if (i915_gem_clflush_object(obj, obj->pin_display))
- i915_gem_chipset_flush(dev);
+ needs_clflush_after = true;
}
}
if (needs_clflush_after)
i915_gem_chipset_flush(dev);
+ else
+ obj->cache_dirty = true;
intel_fb_obj_flush(obj, false, ORIGIN_CPU);
return ret;
/**
* i915_gem_fault - fault a page into the GTT
- * vma: VMA in question
- * vmf: fault info
+ * @vma: VMA in question
+ * @vmf: fault info
*
* The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
* from userspace. The fault handler takes care of binding the object to
old_write_domain);
}
-int i915_vma_unbind(struct i915_vma *vma)
+static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
{
struct drm_i915_gem_object *obj = vma->obj;
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
BUG_ON(obj->pages == NULL);
- ret = i915_gem_object_wait_rendering(obj, false);
- if (ret)
- return ret;
- /* Continue on if we fail due to EIO, the GPU is hung so we
- * should be safe and we need to cleanup or else we might
- * cause memory corruption through use-after-free.
- */
+ if (wait) {
+ ret = i915_gem_object_wait_rendering(obj, false);
+ if (ret)
+ return ret;
+ }
if (i915_is_ggtt(vma->vm) &&
vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
return 0;
}
+int i915_vma_unbind(struct i915_vma *vma)
+{
+ return __i915_vma_unbind(vma, true);
+}
+
+int __i915_vma_unbind_no_wait(struct i915_vma *vma)
+{
+ return __i915_vma_unbind(vma, false);
+}
+
int i915_gpu_idle(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
{
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- u32 size, fence_size, fence_alignment, unfenced_alignment;
- u64 start =
- flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
- u64 end =
- flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
+ u32 fence_alignment, unfenced_alignment;
+ u32 search_flag, alloc_flag;
+ u64 start, end;
+ u64 size, fence_size;
struct i915_vma *vma;
int ret;
size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
}
+ start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
+ end = vm->total;
+ if (flags & PIN_MAPPABLE)
+ end = min_t(u64, end, dev_priv->gtt.mappable_end);
+ if (flags & PIN_ZONE_4G)
+ end = min_t(u64, end, (1ULL << 32));
+
if (alignment == 0)
alignment = flags & PIN_MAPPABLE ? fence_alignment :
unfenced_alignment;
* attempt to find space.
*/
if (size > end) {
- DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%u > %s aperture=%llu\n",
+ DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
ggtt_view ? ggtt_view->type : 0,
size,
flags & PIN_MAPPABLE ? "mappable" : "total",
if (IS_ERR(vma))
goto err_unpin;
+ if (flags & PIN_HIGH) {
+ search_flag = DRM_MM_SEARCH_BELOW;
+ alloc_flag = DRM_MM_CREATE_TOP;
+ } else {
+ search_flag = DRM_MM_SEARCH_DEFAULT;
+ alloc_flag = DRM_MM_CREATE_DEFAULT;
+ }
+
search_free:
ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
size, alignment,
obj->cache_level,
start, end,
- DRM_MM_SEARCH_DEFAULT,
- DRM_MM_CREATE_DEFAULT);
+ search_flag,
+ alloc_flag);
if (ret) {
ret = i915_gem_evict_something(dev, vm, size, alignment,
obj->cache_level,
return 0;
}
+/**
+ * Changes the cache-level of an object across all VMA.
+ *
+ * After this function returns, the object will be in the new cache-level
+ * across all GTT and the contents of the backing storage will be coherent,
+ * with respect to the new cache-level. In order to keep the backing storage
+ * coherent for all users, we only allow a single cache level to be set
+ * globally on the object and prevent it from being changed whilst the
+ * hardware is reading from the object. That is if the object is currently
+ * on the scanout it will be set to uncached (or equivalent display
+ * cache coherency) and all non-MOCS GPU access will also be uncached so
+ * that all direct access to the scanout remains coherent.
+ */
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
enum i915_cache_level cache_level)
{
struct drm_device *dev = obj->base.dev;
struct i915_vma *vma, *next;
- int ret;
+ bool bound = false;
+ int ret = 0;
if (obj->cache_level == cache_level)
- return 0;
-
- if (i915_gem_obj_is_pinned(obj)) {
- DRM_DEBUG("can not change the cache level of pinned objects\n");
- return -EBUSY;
- }
+ goto out;
+ /* Inspect the list of currently bound VMA and unbind any that would
+ * be invalid given the new cache-level. This is principally to
+ * catch the issue of the CS prefetch crossing page boundaries and
+ * reading an invalid PTE on older architectures.
+ */
list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
+ if (!drm_mm_node_allocated(&vma->node))
+ continue;
+
+ if (vma->pin_count) {
+ DRM_DEBUG("can not change the cache level of pinned objects\n");
+ return -EBUSY;
+ }
+
if (!i915_gem_valid_gtt_space(vma, cache_level)) {
ret = i915_vma_unbind(vma);
if (ret)
return ret;
- }
+ } else
+ bound = true;
}
- if (i915_gem_obj_bound_any(obj)) {
+ /* We can reuse the existing drm_mm nodes but need to change the
+ * cache-level on the PTE. We could simply unbind them all and
+ * rebind with the correct cache-level on next use. However since
+ * we already have a valid slot, dma mapping, pages etc, we may as
+ * rewrite the PTE in the belief that doing so tramples upon less
+ * state and so involves less work.
+ */
+ if (bound) {
+ /* Before we change the PTE, the GPU must not be accessing it.
+ * If we wait upon the object, we know that all the bound
+ * VMA are no longer active.
+ */
ret = i915_gem_object_wait_rendering(obj, false);
if (ret)
return ret;
- i915_gem_object_finish_gtt(obj);
-
- /* Before SandyBridge, you could not use tiling or fence
- * registers with snooped memory, so relinquish any fences
- * currently pointing to our region in the aperture.
- */
- if (INTEL_INFO(dev)->gen < 6) {
+ if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
+ /* Access to snoopable pages through the GTT is
+ * incoherent and on some machines causes a hard
+ * lockup. Relinquish the CPU mmaping to force
+ * userspace to refault in the pages and we can
+ * then double check if the GTT mapping is still
+ * valid for that pointer access.
+ */
+ i915_gem_release_mmap(obj);
+
+ /* As we no longer need a fence for GTT access,
+ * we can relinquish it now (and so prevent having
+ * to steal a fence from someone else on the next
+ * fence request). Note GPU activity would have
+ * dropped the fence as all snoopable access is
+ * supposed to be linear.
+ */
ret = i915_gem_object_put_fence(obj);
if (ret)
return ret;
+ } else {
+ /* We either have incoherent backing store and
+ * so no GTT access or the architecture is fully
+ * coherent. In such cases, existing GTT mmaps
+ * ignore the cache bit in the PTE and we can
+ * rewrite it without confusing the GPU or having
+ * to force userspace to fault back in its mmaps.
+ */
}
- list_for_each_entry(vma, &obj->vma_list, vma_link)
- if (drm_mm_node_allocated(&vma->node)) {
- ret = i915_vma_bind(vma, cache_level,
- PIN_UPDATE);
- if (ret)
- return ret;
- }
+ list_for_each_entry(vma, &obj->vma_list, vma_link) {
+ if (!drm_mm_node_allocated(&vma->node))
+ continue;
+
+ ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
+ if (ret)
+ return ret;
+ }
}
list_for_each_entry(vma, &obj->vma_list, vma_link)
vma->node.color = cache_level;
obj->cache_level = cache_level;
+out:
+ /* Flush the dirty CPU caches to the backing storage so that the
+ * object is now coherent at its new cache level (with respect
+ * to the access domain).
+ */
if (obj->cache_dirty &&
obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
cpu_write_needs_clflush(obj)) {
level = I915_CACHE_NONE;
break;
case I915_CACHING_CACHED:
+ /*
+ * Due to a HW issue on BXT A stepping, GPU stores via a
+ * snooped mapping may leave stale data in a corresponding CPU
+ * cacheline, whereas normally such cachelines would get
+ * invalidated.
+ */
+ if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)
+ return -ENODEV;
+
level = I915_CACHE_LLC;
break;
case I915_CACHING_DISPLAY:
return -EBUSY;
if (i915_vma_misplaced(vma, alignment, flags)) {
- unsigned long offset;
- offset = ggtt_view ? i915_gem_obj_ggtt_offset_view(obj, ggtt_view) :
- i915_gem_obj_offset(obj, vm);
WARN(vma->pin_count,
"bo is already pinned in %s with incorrect alignment:"
- " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
+ " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
" obj->map_and_fenceable=%d\n",
ggtt_view ? "ggtt" : "ppgtt",
- offset,
+ upper_32_bits(vma->node.start),
+ lower_32_bits(vma->node.start),
alignment,
!!(flags & PIN_MAPPABLE),
obj->map_and_fenceable);
BUG();
}
-static bool
-intel_enable_blt(struct drm_device *dev)
-{
- if (!HAS_BLT(dev))
- return false;
-
- /* The blitter was dysfunctional on early prototypes */
- if (IS_GEN6(dev) && dev->pdev->revision < 8) {
- DRM_INFO("BLT not supported on this pre-production hardware;"
- " graphics performance will be degraded.\n");
- return false;
- }
-
- return true;
-}
-
static void init_unused_ring(struct drm_device *dev, u32 base)
{
struct drm_i915_private *dev_priv = dev->dev_private;
goto cleanup_render_ring;
}
- if (intel_enable_blt(dev)) {
+ if (HAS_BLT(dev)) {
ret = intel_init_blt_ring_buffer(dev);
if (ret)
goto cleanup_bsd_ring;
goto cleanup_vebox_ring;
}
- ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
- if (ret)
- goto cleanup_bsd2_ring;
-
return 0;
-cleanup_bsd2_ring:
- intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]);
cleanup_vebox_ring:
intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
cleanup_blt_ring:
goto out;
}
+ /* We can't enable contexts until all firmware is loaded */
+ if (HAS_GUC_UCODE(dev)) {
+ ret = intel_guc_ucode_load(dev);
+ if (ret) {
+ /*
+ * If we got an error and GuC submission is enabled, map
+ * the error to -EIO so the GPU will be declared wedged.
+ * OTOH, if we didn't intend to use the GuC anyway, just
+ * discard the error and carry on.
+ */
+ DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret,
+ i915.enable_guc_submission ? "" :
+ " (ignored)");
+ ret = i915.enable_guc_submission ? -EIO : 0;
+ if (ret)
+ goto out;
+ }
+ }
+
+ /*
+ * Increment the next seqno by 0x100 so we have a visible break
+ * on re-initialisation
+ */
+ ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
+ if (ret)
+ goto out;
+
/* Now it is safe to go back round and do everything else: */
for_each_ring(ring, dev_priv, i) {
struct drm_i915_gem_request *req;
INIT_LIST_HEAD(&ring->request_list);
}
-void i915_init_vm(struct drm_i915_private *dev_priv,
- struct i915_address_space *vm)
-{
- if (!i915_is_ggtt(vm))
- drm_mm_init(&vm->mm, vm->start, vm->total);
- vm->dev = dev_priv->dev;
- INIT_LIST_HEAD(&vm->active_list);
- INIT_LIST_HEAD(&vm->inactive_list);
- INIT_LIST_HEAD(&vm->global_link);
- list_add_tail(&vm->global_link, &dev_priv->vm_list);
-}
-
void
i915_gem_load(struct drm_device *dev)
{
NULL);
INIT_LIST_HEAD(&dev_priv->vm_list);
- i915_init_vm(dev_priv, &dev_priv->gtt.base);
-
INIT_LIST_HEAD(&dev_priv->context_list);
INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
INIT_LIST_HEAD(&dev_priv->mm.bound_list);
dev_priv->num_fence_regs =
I915_READ(vgtif_reg(avail_rs.fence_num));
+ /*
+ * Set initial sequence number for requests.
+ * Using this number allows the wraparound to happen early,
+ * catching any obvious problems.
+ */
+ dev_priv->next_seqno = ((u32)~0 - 0x1100);
+ dev_priv->last_seqno = ((u32)~0 - 0x1101);
+
/* Initialize fence registers to zero */
INIT_LIST_HEAD(&dev_priv->mm.fence_list);
i915_gem_restore_fences(dev);
/**
* i915_gem_track_fb - update frontbuffer tracking
- * old: current GEM buffer for the frontbuffer slots
- * new: new GEM buffer for the frontbuffer slots
- * frontbuffer_bits: bitmask of frontbuffer slots
+ * @old: current GEM buffer for the frontbuffer slots
+ * @new: new GEM buffer for the frontbuffer slots
+ * @frontbuffer_bits: bitmask of frontbuffer slots
*
* This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
* from @old and setting them in @new. Both @old and @new can be NULL.
}
/* All the new VM stuff */
-unsigned long
-i915_gem_obj_offset(struct drm_i915_gem_object *o,
- struct i915_address_space *vm)
+u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
+ struct i915_address_space *vm)
{
struct drm_i915_private *dev_priv = o->base.dev->dev_private;
struct i915_vma *vma;
return -1;
}
-unsigned long
-i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
- const struct i915_ggtt_view *view)
+u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
+ const struct i915_ggtt_view *view)
{
struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
struct i915_vma *vma;
return ret;
}
+static void i915_gem_context_clean(struct intel_context *ctx)
+{
+ struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
+ struct i915_vma *vma, *next;
+
+ if (!ppgtt)
+ return;
+
+ WARN_ON(!list_empty(&ppgtt->base.active_list));
+
+ list_for_each_entry_safe(vma, next, &ppgtt->base.inactive_list,
+ mm_list) {
+ if (WARN_ON(__i915_vma_unbind_no_wait(vma)))
+ break;
+ }
+}
+
void i915_gem_context_free(struct kref *ctx_ref)
{
struct intel_context *ctx = container_of(ctx_ref, typeof(*ctx), ref);
if (i915.enable_execlists)
intel_lr_context_free(ctx);
+ /*
+ * This context is going away and we need to remove all VMAs still
+ * around. This is to handle imported shared objects for which
+ * destructor did not run when their handles were closed.
+ */
+ i915_gem_context_clean(ctx);
+
i915_ppgtt_put(ctx->ppgtt);
if (ctx->legacy_hw_ctx.rcs_state)
if (WARN_ON(dev_priv->ring[RCS].default_context))
return 0;
+ if (intel_vgpu_active(dev) && HAS_LOGICAL_RING_CONTEXTS(dev)) {
+ if (!i915.enable_execlists) {
+ DRM_INFO("Only EXECLIST mode is supported in vgpu.\n");
+ return -EINVAL;
+ }
+ }
+
if (i915.enable_execlists) {
/* NB: intentionally left blank. We will allocate our own
* backing objects as we need them, thank you very much */
return 0;
}
-
-/**
- * i915_gem_evict_everything - Try to evict all objects
- * @dev: Device to evict objects for
- *
- * This functions tries to evict all gem objects from all address spaces. Used
- * by the shrinker as a last-ditch effort and for suspend, before releasing the
- * backing storage of all unbound objects.
- */
-int
-i915_gem_evict_everything(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct i915_address_space *vm, *v;
- bool lists_empty = true;
- int ret;
-
- list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
- lists_empty = (list_empty(&vm->inactive_list) &&
- list_empty(&vm->active_list));
- if (!lists_empty)
- lists_empty = false;
- }
-
- if (lists_empty)
- return -ENOSPC;
-
- trace_i915_gem_evict_everything(dev);
-
- /* The gpu_idle will flush everything in the write domain to the
- * active list. Then we must move everything off the active list
- * with retire requests.
- */
- ret = i915_gpu_idle(dev);
- if (ret)
- return ret;
-
- i915_gem_retire_requests(dev);
-
- /* Having flushed everything, unbind() should never raise an error */
- list_for_each_entry_safe(vm, v, &dev_priv->vm_list, global_link)
- WARN_ON(i915_gem_evict_vm(vm, false));
-
- return 0;
-}
flags |= PIN_GLOBAL;
if (!drm_mm_node_allocated(&vma->node)) {
+ /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
+ * limit address to the first 4GBs for unflagged objects.
+ */
+ if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
+ flags |= PIN_ZONE_4G;
if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
flags |= PIN_GLOBAL | PIN_MAPPABLE;
if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+ if ((flags & PIN_MAPPABLE) == 0)
+ flags |= PIN_HIGH;
}
ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
return !only_mappable_for_reloc(entry->flags);
+ if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
+ (vma->node.start + vma->node.size - 1) >> 32)
+ return true;
+
return false;
}
if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
return false;
- return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
+ /* Kernel clipping was a DRI1 misfeature */
+ if (exec->num_cliprects || exec->cliprects_ptr)
+ return false;
+
+ if (exec->DR4 == 0xffffffff) {
+ DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
+ exec->DR4 = 0;
+ }
+ if (exec->DR1 || exec->DR4)
+ return false;
+
+ if ((exec->batch_start_offset | exec->batch_len) & 0x7)
+ return false;
+
+ return true;
}
static int
}
if (i915.enable_execlists && !ctx->engine[ring->id].state) {
- int ret = intel_lr_context_deferred_create(ctx, ring);
+ int ret = intel_lr_context_deferred_alloc(ctx, ring);
if (ret) {
DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);
return ERR_PTR(ret);
return 0;
}
-static int
-i915_emit_box(struct drm_i915_gem_request *req,
- struct drm_clip_rect *box,
- int DR1, int DR4)
-{
- struct intel_engine_cs *ring = req->ring;
- int ret;
-
- if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
- box->y2 <= 0 || box->x2 <= 0) {
- DRM_ERROR("Bad box %d,%d..%d,%d\n",
- box->x1, box->y1, box->x2, box->y2);
- return -EINVAL;
- }
-
- if (INTEL_INFO(ring->dev)->gen >= 4) {
- ret = intel_ring_begin(req, 4);
- if (ret)
- return ret;
-
- intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO_I965);
- intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16);
- intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16);
- intel_ring_emit(ring, DR4);
- } else {
- ret = intel_ring_begin(req, 6);
- if (ret)
- return ret;
-
- intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO);
- intel_ring_emit(ring, DR1);
- intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16);
- intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16);
- intel_ring_emit(ring, DR4);
- intel_ring_emit(ring, 0);
- }
- intel_ring_advance(ring);
-
- return 0;
-}
-
static struct drm_i915_gem_object*
i915_gem_execbuffer_parse(struct intel_engine_cs *ring,
struct drm_i915_gem_exec_object2 *shadow_exec_entry,
struct drm_i915_gem_execbuffer2 *args,
struct list_head *vmas)
{
- struct drm_clip_rect *cliprects = NULL;
struct drm_device *dev = params->dev;
struct intel_engine_cs *ring = params->ring;
struct drm_i915_private *dev_priv = dev->dev_private;
u64 exec_start, exec_len;
int instp_mode;
u32 instp_mask;
- int i, ret = 0;
-
- if (args->num_cliprects != 0) {
- if (ring != &dev_priv->ring[RCS]) {
- DRM_DEBUG("clip rectangles are only valid with the render ring\n");
- return -EINVAL;
- }
-
- if (INTEL_INFO(dev)->gen >= 5) {
- DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
- return -EINVAL;
- }
-
- if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
- DRM_DEBUG("execbuf with %u cliprects\n",
- args->num_cliprects);
- return -EINVAL;
- }
-
- cliprects = kcalloc(args->num_cliprects,
- sizeof(*cliprects),
- GFP_KERNEL);
- if (cliprects == NULL) {
- ret = -ENOMEM;
- goto error;
- }
-
- if (copy_from_user(cliprects,
- to_user_ptr(args->cliprects_ptr),
- sizeof(*cliprects)*args->num_cliprects)) {
- ret = -EFAULT;
- goto error;
- }
- } else {
- if (args->DR4 == 0xffffffff) {
- DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
- args->DR4 = 0;
- }
-
- if (args->DR1 || args->DR4 || args->cliprects_ptr) {
- DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
- return -EINVAL;
- }
- }
+ int ret;
ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
if (ret)
- goto error;
+ return ret;
ret = i915_switch_context(params->request);
if (ret)
- goto error;
+ return ret;
WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<<ring->id),
"%s didn't clear reload\n", ring->name);
case I915_EXEC_CONSTANTS_REL_SURFACE:
if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
- ret = -EINVAL;
- goto error;
+ return -EINVAL;
}
if (instp_mode != dev_priv->relative_constants_mode) {
if (INTEL_INFO(dev)->gen < 4) {
DRM_DEBUG("no rel constants on pre-gen4\n");
- ret = -EINVAL;
- goto error;
+ return -EINVAL;
}
if (INTEL_INFO(dev)->gen > 5 &&
instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
- ret = -EINVAL;
- goto error;
+ return -EINVAL;
}
/* The HW changed the meaning on this bit on gen6 */
break;
default:
DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
- ret = -EINVAL;
- goto error;
+ return -EINVAL;
}
if (ring == &dev_priv->ring[RCS] &&
- instp_mode != dev_priv->relative_constants_mode) {
+ instp_mode != dev_priv->relative_constants_mode) {
ret = intel_ring_begin(params->request, 4);
if (ret)
- goto error;
+ return ret;
intel_ring_emit(ring, MI_NOOP);
intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
ret = i915_reset_gen7_sol_offsets(dev, params->request);
if (ret)
- goto error;
+ return ret;
}
exec_len = args->batch_len;
exec_start = params->batch_obj_vm_offset +
params->args_batch_start_offset;
- if (cliprects) {
- for (i = 0; i < args->num_cliprects; i++) {
- ret = i915_emit_box(params->request, &cliprects[i],
- args->DR1, args->DR4);
- if (ret)
- goto error;
-
- ret = ring->dispatch_execbuffer(params->request,
- exec_start, exec_len,
- params->dispatch_flags);
- if (ret)
- goto error;
- }
- } else {
- ret = ring->dispatch_execbuffer(params->request,
- exec_start, exec_len,
- params->dispatch_flags);
- if (ret)
- return ret;
- }
+ ret = ring->dispatch_execbuffer(params->request,
+ exec_start, exec_len,
+ params->dispatch_flags);
+ if (ret)
+ return ret;
trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
i915_gem_execbuffer_move_to_active(vmas, params->request);
i915_gem_execbuffer_retire_commands(params);
-error:
- kfree(cliprects);
- return ret;
+ return 0;
}
/**
struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- int fence_reg;
+ int fence_reg_lo, fence_reg_hi;
int fence_pitch_shift;
if (INTEL_INFO(dev)->gen >= 6) {
- fence_reg = FENCE_REG_SANDYBRIDGE_0;
- fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
+ fence_reg_lo = FENCE_REG_GEN6_LO(reg);
+ fence_reg_hi = FENCE_REG_GEN6_HI(reg);
+ fence_pitch_shift = GEN6_FENCE_PITCH_SHIFT;
} else {
- fence_reg = FENCE_REG_965_0;
+ fence_reg_lo = FENCE_REG_965_LO(reg);
+ fence_reg_hi = FENCE_REG_965_HI(reg);
fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
}
- fence_reg += reg * 8;
-
/* To w/a incoherency with non-atomic 64-bit register updates,
* we split the 64-bit update into two 32-bit writes. In order
* for a partial fence not to be evaluated between writes, we
* For extra levels of paranoia, we make sure each step lands
* before applying the next step.
*/
- I915_WRITE(fence_reg, 0);
- POSTING_READ(fence_reg);
+ I915_WRITE(fence_reg_lo, 0);
+ POSTING_READ(fence_reg_lo);
if (obj) {
u32 size = i915_gem_obj_ggtt_size(obj);
val |= 1 << I965_FENCE_TILING_Y_SHIFT;
val |= I965_FENCE_REG_VALID;
- I915_WRITE(fence_reg + 4, val >> 32);
- POSTING_READ(fence_reg + 4);
+ I915_WRITE(fence_reg_hi, val >> 32);
+ POSTING_READ(fence_reg_hi);
- I915_WRITE(fence_reg + 0, val);
- POSTING_READ(fence_reg);
+ I915_WRITE(fence_reg_lo, val);
+ POSTING_READ(fence_reg_lo);
} else {
- I915_WRITE(fence_reg + 4, 0);
- POSTING_READ(fence_reg + 4);
+ I915_WRITE(fence_reg_hi, 0);
+ POSTING_READ(fence_reg_hi);
}
}
WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
(size & -size) != size ||
(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
- "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
+ "object 0x%08llx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
} else
val = 0;
- if (reg < 8)
- reg = FENCE_REG_830_0 + reg * 4;
- else
- reg = FENCE_REG_945_8 + (reg - 8) * 4;
-
- I915_WRITE(reg, val);
- POSTING_READ(reg);
+ I915_WRITE(FENCE_REG(reg), val);
+ POSTING_READ(FENCE_REG(reg));
}
static void i830_write_fence_reg(struct drm_device *dev, int reg,
WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
(size & -size) != size ||
(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
- "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
+ "object 0x%08llx not 512K or pot-size 0x%08x aligned\n",
i915_gem_obj_ggtt_offset(obj), size);
pitch_val = obj->stride / 128;
} else
val = 0;
- I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
- POSTING_READ(FENCE_REG_830_0 + reg * 4);
+ I915_WRITE(FENCE_REG(reg), val);
+ POSTING_READ(FENCE_REG(reg));
}
inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
/* First try to find a free reg */
avail = NULL;
- for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
+ for (i = 0; i < dev_priv->num_fence_regs; i++) {
reg = &dev_priv->fence_regs[i];
if (!reg->obj)
return reg;
return pde;
}
+#define gen8_pdpe_encode gen8_pde_encode
+#define gen8_pml4e_encode gen8_pde_encode
+
static gen6_pte_t snb_pte_encode(dma_addr_t addr,
enum i915_cache_level level,
bool valid, u32 unused)
fill_px(vm->dev, pd, scratch_pde);
}
+static int __pdp_init(struct drm_device *dev,
+ struct i915_page_directory_pointer *pdp)
+{
+ size_t pdpes = I915_PDPES_PER_PDP(dev);
+
+ pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
+ sizeof(unsigned long),
+ GFP_KERNEL);
+ if (!pdp->used_pdpes)
+ return -ENOMEM;
+
+ pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
+ GFP_KERNEL);
+ if (!pdp->page_directory) {
+ kfree(pdp->used_pdpes);
+ /* the PDP might be the statically allocated top level. Keep it
+ * as clean as possible */
+ pdp->used_pdpes = NULL;
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void __pdp_fini(struct i915_page_directory_pointer *pdp)
+{
+ kfree(pdp->used_pdpes);
+ kfree(pdp->page_directory);
+ pdp->page_directory = NULL;
+}
+
+static struct
+i915_page_directory_pointer *alloc_pdp(struct drm_device *dev)
+{
+ struct i915_page_directory_pointer *pdp;
+ int ret = -ENOMEM;
+
+ WARN_ON(!USES_FULL_48BIT_PPGTT(dev));
+
+ pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
+ if (!pdp)
+ return ERR_PTR(-ENOMEM);
+
+ ret = __pdp_init(dev, pdp);
+ if (ret)
+ goto fail_bitmap;
+
+ ret = setup_px(dev, pdp);
+ if (ret)
+ goto fail_page_m;
+
+ return pdp;
+
+fail_page_m:
+ __pdp_fini(pdp);
+fail_bitmap:
+ kfree(pdp);
+
+ return ERR_PTR(ret);
+}
+
+static void free_pdp(struct drm_device *dev,
+ struct i915_page_directory_pointer *pdp)
+{
+ __pdp_fini(pdp);
+ if (USES_FULL_48BIT_PPGTT(dev)) {
+ cleanup_px(dev, pdp);
+ kfree(pdp);
+ }
+}
+
+static void gen8_initialize_pdp(struct i915_address_space *vm,
+ struct i915_page_directory_pointer *pdp)
+{
+ gen8_ppgtt_pdpe_t scratch_pdpe;
+
+ scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
+
+ fill_px(vm->dev, pdp, scratch_pdpe);
+}
+
+static void gen8_initialize_pml4(struct i915_address_space *vm,
+ struct i915_pml4 *pml4)
+{
+ gen8_ppgtt_pml4e_t scratch_pml4e;
+
+ scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
+ I915_CACHE_LLC);
+
+ fill_px(vm->dev, pml4, scratch_pml4e);
+}
+
+static void
+gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
+ struct i915_page_directory_pointer *pdp,
+ struct i915_page_directory *pd,
+ int index)
+{
+ gen8_ppgtt_pdpe_t *page_directorypo;
+
+ if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
+ return;
+
+ page_directorypo = kmap_px(pdp);
+ page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
+ kunmap_px(ppgtt, page_directorypo);
+}
+
+static void
+gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
+ struct i915_pml4 *pml4,
+ struct i915_page_directory_pointer *pdp,
+ int index)
+{
+ gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
+
+ WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev));
+ pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
+ kunmap_px(ppgtt, pagemap);
+}
+
/* Broadwell Page Directory Pointer Descriptors */
static int gen8_write_pdp(struct drm_i915_gem_request *req,
unsigned entry,
return 0;
}
-static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
- struct drm_i915_gem_request *req)
+static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt,
+ struct drm_i915_gem_request *req)
{
int i, ret;
return 0;
}
-static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
- uint64_t start,
- uint64_t length,
- bool use_scratch)
+static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
+ struct drm_i915_gem_request *req)
+{
+ return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
+}
+
+static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm,
+ struct i915_page_directory_pointer *pdp,
+ uint64_t start,
+ uint64_t length,
+ gen8_pte_t scratch_pte)
{
struct i915_hw_ppgtt *ppgtt =
container_of(vm, struct i915_hw_ppgtt, base);
- gen8_pte_t *pt_vaddr, scratch_pte;
- unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
- unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
- unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
+ gen8_pte_t *pt_vaddr;
+ unsigned pdpe = gen8_pdpe_index(start);
+ unsigned pde = gen8_pde_index(start);
+ unsigned pte = gen8_pte_index(start);
unsigned num_entries = length >> PAGE_SHIFT;
unsigned last_pte, i;
- scratch_pte = gen8_pte_encode(px_dma(ppgtt->base.scratch_page),
- I915_CACHE_LLC, use_scratch);
+ if (WARN_ON(!pdp))
+ return;
while (num_entries) {
struct i915_page_directory *pd;
struct i915_page_table *pt;
- if (WARN_ON(!ppgtt->pdp.page_directory[pdpe]))
+ if (WARN_ON(!pdp->page_directory[pdpe]))
break;
- pd = ppgtt->pdp.page_directory[pdpe];
+ pd = pdp->page_directory[pdpe];
if (WARN_ON(!pd->page_table[pde]))
break;
pte = 0;
if (++pde == I915_PDES) {
- pdpe++;
+ if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
+ break;
pde = 0;
}
}
}
-static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
- struct sg_table *pages,
- uint64_t start,
- enum i915_cache_level cache_level, u32 unused)
+static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
+ uint64_t start,
+ uint64_t length,
+ bool use_scratch)
+{
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+ gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+ I915_CACHE_LLC, use_scratch);
+
+ if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
+ gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length,
+ scratch_pte);
+ } else {
+ uint64_t templ4, pml4e;
+ struct i915_page_directory_pointer *pdp;
+
+ gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
+ gen8_ppgtt_clear_pte_range(vm, pdp, start, length,
+ scratch_pte);
+ }
+ }
+}
+
+static void
+gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
+ struct i915_page_directory_pointer *pdp,
+ struct sg_page_iter *sg_iter,
+ uint64_t start,
+ enum i915_cache_level cache_level)
{
struct i915_hw_ppgtt *ppgtt =
container_of(vm, struct i915_hw_ppgtt, base);
gen8_pte_t *pt_vaddr;
- unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
- unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
- unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
- struct sg_page_iter sg_iter;
+ unsigned pdpe = gen8_pdpe_index(start);
+ unsigned pde = gen8_pde_index(start);
+ unsigned pte = gen8_pte_index(start);
pt_vaddr = NULL;
- for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
- if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
- break;
-
+ while (__sg_page_iter_next(sg_iter)) {
if (pt_vaddr == NULL) {
- struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe];
+ struct i915_page_directory *pd = pdp->page_directory[pdpe];
struct i915_page_table *pt = pd->page_table[pde];
pt_vaddr = kmap_px(pt);
}
pt_vaddr[pte] =
- gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
+ gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
cache_level, true);
if (++pte == GEN8_PTES) {
kunmap_px(ppgtt, pt_vaddr);
pt_vaddr = NULL;
if (++pde == I915_PDES) {
- pdpe++;
+ if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
+ break;
pde = 0;
}
pte = 0;
kunmap_px(ppgtt, pt_vaddr);
}
+static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
+ struct sg_table *pages,
+ uint64_t start,
+ enum i915_cache_level cache_level,
+ u32 unused)
+{
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+ struct sg_page_iter sg_iter;
+
+ __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
+
+ if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
+ gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
+ cache_level);
+ } else {
+ struct i915_page_directory_pointer *pdp;
+ uint64_t templ4, pml4e;
+ uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT;
+
+ gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
+ gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,
+ start, cache_level);
+ }
+ }
+}
+
static void gen8_free_page_tables(struct drm_device *dev,
struct i915_page_directory *pd)
{
return PTR_ERR(vm->scratch_pd);
}
+ if (USES_FULL_48BIT_PPGTT(dev)) {
+ vm->scratch_pdp = alloc_pdp(dev);
+ if (IS_ERR(vm->scratch_pdp)) {
+ free_pd(dev, vm->scratch_pd);
+ free_pt(dev, vm->scratch_pt);
+ free_scratch_page(dev, vm->scratch_page);
+ return PTR_ERR(vm->scratch_pdp);
+ }
+ }
+
gen8_initialize_pt(vm, vm->scratch_pt);
gen8_initialize_pd(vm, vm->scratch_pd);
+ if (USES_FULL_48BIT_PPGTT(dev))
+ gen8_initialize_pdp(vm, vm->scratch_pdp);
+
+ return 0;
+}
+
+static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
+{
+ enum vgt_g2v_type msg;
+ struct drm_device *dev = ppgtt->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ unsigned int offset = vgtif_reg(pdp0_lo);
+ int i;
+
+ if (USES_FULL_48BIT_PPGTT(dev)) {
+ u64 daddr = px_dma(&ppgtt->pml4);
+
+ I915_WRITE(offset, lower_32_bits(daddr));
+ I915_WRITE(offset + 4, upper_32_bits(daddr));
+
+ msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
+ VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
+ } else {
+ for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
+ u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
+
+ I915_WRITE(offset, lower_32_bits(daddr));
+ I915_WRITE(offset + 4, upper_32_bits(daddr));
+
+ offset += 8;
+ }
+
+ msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
+ VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
+ }
+
+ I915_WRITE(vgtif_reg(g2v_notify), msg);
return 0;
}
{
struct drm_device *dev = vm->dev;
+ if (USES_FULL_48BIT_PPGTT(dev))
+ free_pdp(dev, vm->scratch_pdp);
free_pd(dev, vm->scratch_pd);
free_pt(dev, vm->scratch_pt);
free_scratch_page(dev, vm->scratch_page);
}
-static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
+static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev,
+ struct i915_page_directory_pointer *pdp)
{
- struct i915_hw_ppgtt *ppgtt =
- container_of(vm, struct i915_hw_ppgtt, base);
int i;
- for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) {
- if (WARN_ON(!ppgtt->pdp.page_directory[i]))
+ for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) {
+ if (WARN_ON(!pdp->page_directory[i]))
continue;
- gen8_free_page_tables(ppgtt->base.dev,
- ppgtt->pdp.page_directory[i]);
- free_pd(ppgtt->base.dev, ppgtt->pdp.page_directory[i]);
+ gen8_free_page_tables(dev, pdp->page_directory[i]);
+ free_pd(dev, pdp->page_directory[i]);
}
+ free_pdp(dev, pdp);
+}
+
+static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
+{
+ int i;
+
+ for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
+ if (WARN_ON(!ppgtt->pml4.pdps[i]))
+ continue;
+
+ gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]);
+ }
+
+ cleanup_px(ppgtt->base.dev, &ppgtt->pml4);
+}
+
+static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
+{
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+
+ if (intel_vgpu_active(vm->dev))
+ gen8_ppgtt_notify_vgt(ppgtt, false);
+
+ if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
+ gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp);
+ else
+ gen8_ppgtt_cleanup_4lvl(ppgtt);
+
gen8_free_scratch(vm);
}
/**
* gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
- * @ppgtt: Master ppgtt structure.
- * @pd: Page directory for this address range.
+ * @vm: Master vm structure.
+ * @pd: Page directory for this address range.
* @start: Starting virtual address to begin allocations.
- * @length Size of the allocations.
+ * @length: Size of the allocations.
* @new_pts: Bitmap set by function with new allocations. Likely used by the
* caller to free on error.
*
*
* Return: 0 if success; negative error code otherwise.
*/
-static int gen8_ppgtt_alloc_pagetabs(struct i915_hw_ppgtt *ppgtt,
+static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
struct i915_page_directory *pd,
uint64_t start,
uint64_t length,
unsigned long *new_pts)
{
- struct drm_device *dev = ppgtt->base.dev;
+ struct drm_device *dev = vm->dev;
struct i915_page_table *pt;
uint64_t temp;
uint32_t pde;
gen8_for_each_pde(pt, pd, start, length, temp, pde) {
/* Don't reallocate page tables */
- if (pt) {
+ if (test_bit(pde, pd->used_pdes)) {
/* Scratch is never allocated this way */
- WARN_ON(pt == ppgtt->base.scratch_pt);
+ WARN_ON(pt == vm->scratch_pt);
continue;
}
if (IS_ERR(pt))
goto unwind_out;
- gen8_initialize_pt(&ppgtt->base, pt);
+ gen8_initialize_pt(vm, pt);
pd->page_table[pde] = pt;
__set_bit(pde, new_pts);
+ trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
}
return 0;
/**
* gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
- * @ppgtt: Master ppgtt structure.
+ * @vm: Master vm structure.
* @pdp: Page directory pointer for this address range.
* @start: Starting virtual address to begin allocations.
- * @length Size of the allocations.
- * @new_pds Bitmap set by function with new allocations. Likely used by the
+ * @length: Size of the allocations.
+ * @new_pds: Bitmap set by function with new allocations. Likely used by the
* caller to free on error.
*
* Allocate the required number of page directories starting at the pde index of
*
* Return: 0 if success; negative error code otherwise.
*/
-static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
- struct i915_page_directory_pointer *pdp,
- uint64_t start,
- uint64_t length,
- unsigned long *new_pds)
+static int
+gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
+ struct i915_page_directory_pointer *pdp,
+ uint64_t start,
+ uint64_t length,
+ unsigned long *new_pds)
{
- struct drm_device *dev = ppgtt->base.dev;
+ struct drm_device *dev = vm->dev;
struct i915_page_directory *pd;
uint64_t temp;
uint32_t pdpe;
+ uint32_t pdpes = I915_PDPES_PER_PDP(dev);
- WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES));
+ WARN_ON(!bitmap_empty(new_pds, pdpes));
gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
- if (pd)
+ if (test_bit(pdpe, pdp->used_pdpes))
continue;
pd = alloc_pd(dev);
if (IS_ERR(pd))
goto unwind_out;
- gen8_initialize_pd(&ppgtt->base, pd);
+ gen8_initialize_pd(vm, pd);
pdp->page_directory[pdpe] = pd;
__set_bit(pdpe, new_pds);
+ trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
}
return 0;
unwind_out:
- for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES)
+ for_each_set_bit(pdpe, new_pds, pdpes)
free_pd(dev, pdp->page_directory[pdpe]);
return -ENOMEM;
}
-static void
-free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
+/**
+ * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range.
+ * @vm: Master vm structure.
+ * @pml4: Page map level 4 for this address range.
+ * @start: Starting virtual address to begin allocations.
+ * @length: Size of the allocations.
+ * @new_pdps: Bitmap set by function with new allocations. Likely used by the
+ * caller to free on error.
+ *
+ * Allocate the required number of page directory pointers. Extremely similar to
+ * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs().
+ * The main difference is here we are limited by the pml4 boundary (instead of
+ * the page directory pointer).
+ *
+ * Return: 0 if success; negative error code otherwise.
+ */
+static int
+gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
+ struct i915_pml4 *pml4,
+ uint64_t start,
+ uint64_t length,
+ unsigned long *new_pdps)
{
- int i;
+ struct drm_device *dev = vm->dev;
+ struct i915_page_directory_pointer *pdp;
+ uint64_t temp;
+ uint32_t pml4e;
+
+ WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
+
+ gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+ if (!test_bit(pml4e, pml4->used_pml4es)) {
+ pdp = alloc_pdp(dev);
+ if (IS_ERR(pdp))
+ goto unwind_out;
+
+ gen8_initialize_pdp(vm, pdp);
+ pml4->pdps[pml4e] = pdp;
+ __set_bit(pml4e, new_pdps);
+ trace_i915_page_directory_pointer_entry_alloc(vm,
+ pml4e,
+ start,
+ GEN8_PML4E_SHIFT);
+ }
+ }
- for (i = 0; i < GEN8_LEGACY_PDPES; i++)
- kfree(new_pts[i]);
+ return 0;
+
+unwind_out:
+ for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
+ free_pdp(dev, pml4->pdps[pml4e]);
+
+ return -ENOMEM;
+}
+
+static void
+free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts)
+{
kfree(new_pts);
kfree(new_pds);
}
*/
static
int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
- unsigned long ***new_pts)
+ unsigned long **new_pts,
+ uint32_t pdpes)
{
- int i;
unsigned long *pds;
- unsigned long **pts;
+ unsigned long *pts;
- pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL);
+ pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY);
if (!pds)
return -ENOMEM;
- pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL);
- if (!pts) {
- kfree(pds);
- return -ENOMEM;
- }
-
- for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
- pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES),
- sizeof(unsigned long), GFP_KERNEL);
- if (!pts[i])
- goto err_out;
- }
+ pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long),
+ GFP_TEMPORARY);
+ if (!pts)
+ goto err_out;
*new_pds = pds;
*new_pts = pts;
ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
}
-static int gen8_alloc_va_range(struct i915_address_space *vm,
- uint64_t start,
- uint64_t length)
+static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
+ struct i915_page_directory_pointer *pdp,
+ uint64_t start,
+ uint64_t length)
{
struct i915_hw_ppgtt *ppgtt =
container_of(vm, struct i915_hw_ppgtt, base);
- unsigned long *new_page_dirs, **new_page_tables;
+ unsigned long *new_page_dirs, *new_page_tables;
+ struct drm_device *dev = vm->dev;
struct i915_page_directory *pd;
const uint64_t orig_start = start;
const uint64_t orig_length = length;
uint64_t temp;
uint32_t pdpe;
+ uint32_t pdpes = I915_PDPES_PER_PDP(dev);
int ret;
/* Wrap is never okay since we can only represent 48b, and we don't
if (WARN_ON(start + length < start))
return -ENODEV;
- if (WARN_ON(start + length > ppgtt->base.total))
+ if (WARN_ON(start + length > vm->total))
return -ENODEV;
- ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables);
+ ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
if (ret)
return ret;
/* Do the allocations first so we can easily bail out */
- ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length,
- new_page_dirs);
+ ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
+ new_page_dirs);
if (ret) {
free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
return ret;
}
/* For every page directory referenced, allocate page tables */
- gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
- ret = gen8_ppgtt_alloc_pagetabs(ppgtt, pd, start, length,
- new_page_tables[pdpe]);
+ gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
+ ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
+ new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
if (ret)
goto err_out;
}
/* Allocations have completed successfully, so set the bitmaps, and do
* the mappings. */
- gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
+ gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
gen8_pde_t *const page_directory = kmap_px(pd);
struct i915_page_table *pt;
- uint64_t pd_len = gen8_clamp_pd(start, length);
+ uint64_t pd_len = length;
uint64_t pd_start = start;
uint32_t pde;
/* Map the PDE to the page table */
page_directory[pde] = gen8_pde_encode(px_dma(pt),
I915_CACHE_LLC);
+ trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
+ gen8_pte_index(start),
+ gen8_pte_count(start, length),
+ GEN8_PTES);
/* NB: We haven't yet mapped ptes to pages. At this
* point we're still relying on insert_entries() */
}
kunmap_px(ppgtt, page_directory);
-
- __set_bit(pdpe, ppgtt->pdp.used_pdpes);
+ __set_bit(pdpe, pdp->used_pdpes);
+ gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);
}
free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
err_out:
while (pdpe--) {
- for_each_set_bit(temp, new_page_tables[pdpe], I915_PDES)
- free_pt(vm->dev, ppgtt->pdp.page_directory[pdpe]->page_table[temp]);
+ for_each_set_bit(temp, new_page_tables + pdpe *
+ BITS_TO_LONGS(I915_PDES), I915_PDES)
+ free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]);
}
- for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES)
- free_pd(vm->dev, ppgtt->pdp.page_directory[pdpe]);
+ for_each_set_bit(pdpe, new_page_dirs, pdpes)
+ free_pd(dev, pdp->page_directory[pdpe]);
free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
mark_tlbs_dirty(ppgtt);
return ret;
}
+static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
+ struct i915_pml4 *pml4,
+ uint64_t start,
+ uint64_t length)
+{
+ DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+ struct i915_page_directory_pointer *pdp;
+ uint64_t temp, pml4e;
+ int ret = 0;
+
+ /* Do the pml4 allocations first, so we don't need to track the newly
+ * allocated tables below the pdp */
+ bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
+
+ /* The pagedirectory and pagetable allocations are done in the shared 3
+ * and 4 level code. Just allocate the pdps.
+ */
+ ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length,
+ new_pdps);
+ if (ret)
+ return ret;
+
+ WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2,
+ "The allocation has spanned more than 512GB. "
+ "It is highly likely this is incorrect.");
+
+ gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+ WARN_ON(!pdp);
+
+ ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
+ if (ret)
+ goto err_out;
+
+ gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e);
+ }
+
+ bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
+ GEN8_PML4ES_PER_PML4);
+
+ return 0;
+
+err_out:
+ for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
+ gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]);
+
+ return ret;
+}
+
+static int gen8_alloc_va_range(struct i915_address_space *vm,
+ uint64_t start, uint64_t length)
+{
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+
+ if (USES_FULL_48BIT_PPGTT(vm->dev))
+ return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
+ else
+ return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
+}
+
+static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
+ uint64_t start, uint64_t length,
+ gen8_pte_t scratch_pte,
+ struct seq_file *m)
+{
+ struct i915_page_directory *pd;
+ uint64_t temp;
+ uint32_t pdpe;
+
+ gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
+ struct i915_page_table *pt;
+ uint64_t pd_len = length;
+ uint64_t pd_start = start;
+ uint32_t pde;
+
+ if (!test_bit(pdpe, pdp->used_pdpes))
+ continue;
+
+ seq_printf(m, "\tPDPE #%d\n", pdpe);
+ gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
+ uint32_t pte;
+ gen8_pte_t *pt_vaddr;
+
+ if (!test_bit(pde, pd->used_pdes))
+ continue;
+
+ pt_vaddr = kmap_px(pt);
+ for (pte = 0; pte < GEN8_PTES; pte += 4) {
+ uint64_t va =
+ (pdpe << GEN8_PDPE_SHIFT) |
+ (pde << GEN8_PDE_SHIFT) |
+ (pte << GEN8_PTE_SHIFT);
+ int i;
+ bool found = false;
+
+ for (i = 0; i < 4; i++)
+ if (pt_vaddr[pte + i] != scratch_pte)
+ found = true;
+ if (!found)
+ continue;
+
+ seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
+ for (i = 0; i < 4; i++) {
+ if (pt_vaddr[pte + i] != scratch_pte)
+ seq_printf(m, " %llx", pt_vaddr[pte + i]);
+ else
+ seq_puts(m, " SCRATCH ");
+ }
+ seq_puts(m, "\n");
+ }
+ /* don't use kunmap_px, it could trigger
+ * an unnecessary flush.
+ */
+ kunmap_atomic(pt_vaddr);
+ }
+ }
+}
+
+static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
+{
+ struct i915_address_space *vm = &ppgtt->base;
+ uint64_t start = ppgtt->base.start;
+ uint64_t length = ppgtt->base.total;
+ gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+ I915_CACHE_LLC, true);
+
+ if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
+ gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
+ } else {
+ uint64_t templ4, pml4e;
+ struct i915_pml4 *pml4 = &ppgtt->pml4;
+ struct i915_page_directory_pointer *pdp;
+
+ gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) {
+ if (!test_bit(pml4e, pml4->used_pml4es))
+ continue;
+
+ seq_printf(m, " PML4E #%llu\n", pml4e);
+ gen8_dump_pdp(pdp, start, length, scratch_pte, m);
+ }
+ }
+}
+
+static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
+{
+ unsigned long *new_page_dirs, *new_page_tables;
+ uint32_t pdpes = I915_PDPES_PER_PDP(dev);
+ int ret;
+
+ /* We allocate temp bitmap for page tables for no gain
+ * but as this is for init only, lets keep the things simple
+ */
+ ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
+ if (ret)
+ return ret;
+
+ /* Allocate for all pdps regardless of how the ppgtt
+ * was defined.
+ */
+ ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,
+ 0, 1ULL << 32,
+ new_page_dirs);
+ if (!ret)
+ *ppgtt->pdp.used_pdpes = *new_page_dirs;
+
+ free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
+
+ return ret;
+}
+
/*
* GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
* with a net effect resembling a 2-level page table in normal x86 terms. Each
return ret;
ppgtt->base.start = 0;
- ppgtt->base.total = 1ULL << 32;
- if (IS_ENABLED(CONFIG_X86_32))
- /* While we have a proliferation of size_t variables
- * we cannot represent the full ppgtt size on 32bit,
- * so limit it to the same size as the GGTT (currently
- * 2GiB).
- */
- ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total;
ppgtt->base.cleanup = gen8_ppgtt_cleanup;
ppgtt->base.allocate_va_range = gen8_alloc_va_range;
ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
ppgtt->base.clear_range = gen8_ppgtt_clear_range;
ppgtt->base.unbind_vma = ppgtt_unbind_vma;
ppgtt->base.bind_vma = ppgtt_bind_vma;
+ ppgtt->debug_dump = gen8_dump_ppgtt;
- ppgtt->switch_mm = gen8_mm_switch;
+ if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
+ ret = setup_px(ppgtt->base.dev, &ppgtt->pml4);
+ if (ret)
+ goto free_scratch;
+
+ gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
+
+ ppgtt->base.total = 1ULL << 48;
+ ppgtt->switch_mm = gen8_48b_mm_switch;
+ } else {
+ ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp);
+ if (ret)
+ goto free_scratch;
+
+ ppgtt->base.total = 1ULL << 32;
+ ppgtt->switch_mm = gen8_legacy_mm_switch;
+ trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base,
+ 0, 0,
+ GEN8_PML4E_SHIFT);
+
+ if (intel_vgpu_active(ppgtt->base.dev)) {
+ ret = gen8_preallocate_top_level_pdps(ppgtt);
+ if (ret)
+ goto free_scratch;
+ }
+ }
+
+ if (intel_vgpu_active(ppgtt->base.dev))
+ gen8_ppgtt_notify_vgt(ppgtt, true);
return 0;
+
+free_scratch:
+ gen8_free_scratch(&ppgtt->base);
+ return ret;
}
static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
int j;
for_each_ring(ring, dev_priv, j) {
+ u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0;
I915_WRITE(RING_MODE_GEN7(ring),
- _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
+ _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
}
}
return gen8_ppgtt_init(ppgtt);
}
+static void i915_address_space_init(struct i915_address_space *vm,
+ struct drm_i915_private *dev_priv)
+{
+ drm_mm_init(&vm->mm, vm->start, vm->total);
+ vm->dev = dev_priv->dev;
+ INIT_LIST_HEAD(&vm->active_list);
+ INIT_LIST_HEAD(&vm->inactive_list);
+ list_add_tail(&vm->global_link, &dev_priv->vm_list);
+}
+
int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
{
struct drm_i915_private *dev_priv = dev->dev_private;
ret = __hw_ppgtt_init(dev, ppgtt);
if (ret == 0) {
kref_init(&ppgtt->ref);
- drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
- ppgtt->base.total);
- i915_init_vm(dev_priv, &ppgtt->base);
+ i915_address_space_init(&ppgtt->base, dev_priv);
}
return ret;
static int ggtt_bind_vma(struct i915_vma *vma,
enum i915_cache_level cache_level,
u32 flags)
+{
+ struct drm_i915_gem_object *obj = vma->obj;
+ u32 pte_flags = 0;
+ int ret;
+
+ ret = i915_get_ggtt_vma_pages(vma);
+ if (ret)
+ return ret;
+
+ /* Currently applicable only to VLV */
+ if (obj->gt_ro)
+ pte_flags |= PTE_READ_ONLY;
+
+ vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages,
+ vma->node.start,
+ cache_level, pte_flags);
+
+ /*
+ * Without aliasing PPGTT there's no difference between
+ * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
+ * upgrade to both bound if we bind either to avoid double-binding.
+ */
+ vma->bound |= GLOBAL_BIND | LOCAL_BIND;
+
+ return 0;
+}
+
+static int aliasing_gtt_bind_vma(struct i915_vma *vma,
+ enum i915_cache_level cache_level,
+ u32 flags)
{
struct drm_device *dev = vma->vm->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
pte_flags |= PTE_READ_ONLY;
- if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
+ if (flags & GLOBAL_BIND) {
vma->vm->insert_entries(vma->vm, pages,
vma->node.start,
cache_level, pte_flags);
-
- /* Note the inconsistency here is due to absence of the
- * aliasing ppgtt on gen4 and earlier. Though we always
- * request PIN_USER for execbuffer (translated to LOCAL_BIND),
- * without the appgtt, we cannot honour that request and so
- * must substitute it with a global binding. Since we do this
- * behind the upper layers back, we need to explicitly set
- * the bound flag ourselves.
- */
- vma->bound |= GLOBAL_BIND;
-
}
- if (dev_priv->mm.aliasing_ppgtt && flags & LOCAL_BIND) {
+ if (flags & LOCAL_BIND) {
struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
appgtt->base.insert_entries(&appgtt->base, pages,
vma->node.start,
}
static int i915_gem_setup_global_gtt(struct drm_device *dev,
- unsigned long start,
- unsigned long mappable_end,
- unsigned long end)
+ u64 start,
+ u64 mappable_end,
+ u64 end)
{
/* Let GEM Manage all of the aperture.
*
BUG_ON(mappable_end > end);
- /* Subtract the guard page ... */
- drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
+ ggtt_vm->start = start;
- dev_priv->gtt.base.start = start;
- dev_priv->gtt.base.total = end - start;
+ /* Subtract the guard page before address space initialization to
+ * shrink the range used by drm_mm */
+ ggtt_vm->total = end - start - PAGE_SIZE;
+ i915_address_space_init(ggtt_vm, dev_priv);
+ ggtt_vm->total += PAGE_SIZE;
if (intel_vgpu_active(dev)) {
ret = intel_vgt_balloon(dev);
}
if (!HAS_LLC(dev))
- dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
+ ggtt_vm->mm.color_adjust = i915_gtt_color_adjust;
/* Mark any preallocated objects as occupied */
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
- DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
+ DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n",
i915_gem_obj_ggtt_offset(obj), obj->base.size);
WARN_ON(i915_gem_obj_ggtt_bound(obj));
return ret;
}
vma->bound |= GLOBAL_BIND;
+ list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list);
}
/* Clear any non-preallocated blocks */
true);
dev_priv->mm.aliasing_ppgtt = ppgtt;
+ WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma);
+ dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma;
}
return 0;
/* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
* write would work. */
- I915_WRITE(GEN8_PRIVATE_PAT, pat);
- I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
+ I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
+ I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
}
static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
GEN8_PPAT(6, CHV_PPAT_SNOOP) |
GEN8_PPAT(7, CHV_PPAT_SNOOP);
- I915_WRITE(GEN8_PRIVATE_PAT, pat);
- I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
+ I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
+ I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
}
static int gen8_gmch_probe(struct drm_device *dev,
}
-static void
-rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height,
- struct sg_table *st)
+static struct scatterlist *
+rotate_pages(dma_addr_t *in, unsigned int offset,
+ unsigned int width, unsigned int height,
+ struct sg_table *st, struct scatterlist *sg)
{
unsigned int column, row;
unsigned int src_idx;
- struct scatterlist *sg = st->sgl;
- st->nents = 0;
+ if (!sg) {
+ st->nents = 0;
+ sg = st->sgl;
+ }
for (column = 0; column < width; column++) {
src_idx = width * (height - 1) + column;
* The only thing we need are DMA addresses.
*/
sg_set_page(sg, NULL, PAGE_SIZE, 0);
- sg_dma_address(sg) = in[src_idx];
+ sg_dma_address(sg) = in[offset + src_idx];
sg_dma_len(sg) = PAGE_SIZE;
sg = sg_next(sg);
src_idx -= width;
}
}
+
+ return sg;
}
static struct sg_table *
{
struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
unsigned int size_pages = rot_info->size >> PAGE_SHIFT;
+ unsigned int size_pages_uv;
struct sg_page_iter sg_iter;
unsigned long i;
dma_addr_t *page_addr_list;
struct sg_table *st;
+ unsigned int uv_start_page;
+ struct scatterlist *sg;
int ret = -ENOMEM;
/* Allocate a temporary list of source pages for random access. */
if (!page_addr_list)
return ERR_PTR(ret);
+ /* Account for UV plane with NV12. */
+ if (rot_info->pixel_format == DRM_FORMAT_NV12)
+ size_pages_uv = rot_info->size_uv >> PAGE_SHIFT;
+ else
+ size_pages_uv = 0;
+
/* Allocate target SG list. */
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (!st)
goto err_st_alloc;
- ret = sg_alloc_table(st, size_pages, GFP_KERNEL);
+ ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL);
if (ret)
goto err_sg_alloc;
}
/* Rotate the pages. */
- rotate_pages(page_addr_list,
+ sg = rotate_pages(page_addr_list, 0,
rot_info->width_pages, rot_info->height_pages,
- st);
+ st, NULL);
+
+ /* Append the UV plane if NV12. */
+ if (rot_info->pixel_format == DRM_FORMAT_NV12) {
+ uv_start_page = size_pages;
+
+ /* Check for tile-row un-alignment. */
+ if (offset_in_page(rot_info->uv_offset))
+ uv_start_page--;
+
+ rot_info->uv_start_page = uv_start_page;
+
+ rotate_pages(page_addr_list, uv_start_page,
+ rot_info->width_pages_uv,
+ rot_info->height_pages_uv,
+ st, sg);
+ }
DRM_DEBUG_KMS(
- "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages).\n",
+ "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n",
obj->base.size, rot_info->pitch, rot_info->height,
rot_info->pixel_format, rot_info->width_pages,
- rot_info->height_pages, size_pages);
+ rot_info->height_pages, size_pages + size_pages_uv,
+ size_pages);
drm_free_large(page_addr_list);
drm_free_large(page_addr_list);
DRM_DEBUG_KMS(
- "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages)\n",
+ "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n",
obj->base.size, ret, rot_info->pitch, rot_info->height,
rot_info->pixel_format, rot_info->width_pages,
- rot_info->height_pages, size_pages);
+ rot_info->height_pages, size_pages + size_pages_uv,
+ size_pages);
return ERR_PTR(ret);
}
typedef uint32_t gen6_pte_t;
typedef uint64_t gen8_pte_t;
typedef uint64_t gen8_pde_t;
+typedef uint64_t gen8_ppgtt_pdpe_t;
+typedef uint64_t gen8_ppgtt_pml4e_t;
#define gtt_total_entries(gtt) ((gtt).base.total >> PAGE_SHIFT)
* PDPE | PDE | PTE | offset
* The difference as compared to normal x86 3 level page table is the PDPEs are
* programmed via register.
+ *
+ * GEN8 48b legacy style address is defined as a 4 level page table:
+ * 47:39 | 38:30 | 29:21 | 20:12 | 11:0
+ * PML4E | PDPE | PDE | PTE | offset
*/
+#define GEN8_PML4ES_PER_PML4 512
+#define GEN8_PML4E_SHIFT 39
+#define GEN8_PML4E_MASK (GEN8_PML4ES_PER_PML4 - 1)
#define GEN8_PDPE_SHIFT 30
-#define GEN8_PDPE_MASK 0x3
+/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
+ * tables */
+#define GEN8_PDPE_MASK 0x1ff
#define GEN8_PDE_SHIFT 21
#define GEN8_PDE_MASK 0x1ff
#define GEN8_PTE_SHIFT 12
#define GEN8_LEGACY_PDPES 4
#define GEN8_PTES I915_PTES(sizeof(gen8_pte_t))
+#define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
+ GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES)
+
#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD)
#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */
#define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */
struct intel_rotation_info {
unsigned int height;
unsigned int pitch;
+ unsigned int uv_offset;
uint32_t pixel_format;
uint64_t fb_modifier;
unsigned int width_pages, height_pages;
uint64_t size;
+ unsigned int width_pages_uv, height_pages_uv;
+ uint64_t size_uv;
+ unsigned int uv_start_page;
};
struct i915_ggtt_view {
union {
struct {
- unsigned long offset;
+ u64 offset;
unsigned int size;
} partial;
} params;
};
struct i915_page_directory_pointer {
- /* struct page *page; */
- DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);
- struct i915_page_directory *page_directory[GEN8_LEGACY_PDPES];
+ struct i915_page_dma base;
+
+ unsigned long *used_pdpes;
+ struct i915_page_directory **page_directory;
+};
+
+struct i915_pml4 {
+ struct i915_page_dma base;
+
+ DECLARE_BITMAP(used_pml4es, GEN8_PML4ES_PER_PML4);
+ struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
};
struct i915_address_space {
struct i915_page_scratch *scratch_page;
struct i915_page_table *scratch_pt;
struct i915_page_directory *scratch_pd;
+ struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */
/**
* List of objects currently involved in rendering.
struct i915_address_space base;
size_t stolen_size; /* Total size of stolen memory */
+ size_t stolen_usable_size; /* Total size minus BIOS reserved */
u64 mappable_end; /* End offset that we can CPU map */
struct io_mapping *mappable; /* Mapping to our CPU mappable region */
phys_addr_t mappable_base; /* PA of our GMADR */
struct drm_mm_node node;
unsigned long pd_dirty_rings;
union {
- struct i915_page_directory_pointer pdp;
- struct i915_page_directory pd;
+ struct i915_pml4 pml4; /* GEN8+ & 48b PPGTT */
+ struct i915_page_directory_pointer pdp; /* GEN8+ */
+ struct i915_page_directory pd; /* GEN6-7 */
};
struct drm_i915_file_private *file_priv;
*/
#define gen6_for_each_pde(pt, pd, start, length, temp, iter) \
for (iter = gen6_pde_index(start); \
- pt = (pd)->page_table[iter], length > 0 && iter < I915_PDES; \
+ length > 0 && iter < I915_PDES ? \
+ (pt = (pd)->page_table[iter]), 1 : 0; \
iter++, \
temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT) - start, \
temp = min_t(unsigned, temp, length), \
*/
#define gen8_for_each_pde(pt, pd, start, length, temp, iter) \
for (iter = gen8_pde_index(start); \
- pt = (pd)->page_table[iter], length > 0 && iter < I915_PDES; \
+ length > 0 && iter < I915_PDES ? \
+ (pt = (pd)->page_table[iter]), 1 : 0; \
iter++, \
temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT) - start, \
temp = min(temp, length), \
start += temp, length -= temp)
-#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter) \
- for (iter = gen8_pdpe_index(start); \
- pd = (pdp)->page_directory[iter], length > 0 && iter < GEN8_LEGACY_PDPES; \
+#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter) \
+ for (iter = gen8_pdpe_index(start); \
+ length > 0 && (iter < I915_PDPES_PER_PDP(dev)) ? \
+ (pd = (pdp)->page_directory[iter]), 1 : 0; \
iter++, \
temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT) - start, \
temp = min(temp, length), \
start += temp, length -= temp)
-/* Clamp length to the next page_directory boundary */
-static inline uint64_t gen8_clamp_pd(uint64_t start, uint64_t length)
-{
- uint64_t next_pd = ALIGN(start + 1, 1 << GEN8_PDPE_SHIFT);
-
- if (next_pd > (start + length))
- return length;
-
- return next_pd - start;
-}
+#define gen8_for_each_pml4e(pdp, pml4, start, length, temp, iter) \
+ for (iter = gen8_pml4e_index(start); \
+ length > 0 && iter < GEN8_PML4ES_PER_PML4 ? \
+ (pdp = (pml4)->pdps[iter]), 1 : 0; \
+ iter++, \
+ temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT) - start, \
+ temp = min(temp, length), \
+ start += temp, length -= temp)
static inline uint32_t gen8_pte_index(uint64_t address)
{
static inline uint32_t gen8_pml4e_index(uint64_t address)
{
- WARN_ON(1); /* For 64B */
- return 0;
+ return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK;
}
static inline size_t gen8_pte_count(uint64_t address, uint64_t length)
*/
unsigned long
i915_gem_shrink(struct drm_i915_private *dev_priv,
- long target, unsigned flags)
+ unsigned long target, unsigned flags)
{
const struct {
struct list_head *list;
}, *phase;
unsigned long count = 0;
+ trace_i915_gem_shrink(dev_priv, target, flags);
+ i915_gem_retire_requests(dev_priv->dev);
+
/*
* As we may completely rewrite the (un)bound list whilst unbinding
* (due to retiring requests) we have to strictly process only
obj->madv != I915_MADV_DONTNEED)
continue;
+ if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active)
+ continue;
+
drm_gem_object_reference(&obj->base);
/* For the unbound phase, this should be a no-op! */
list_splice(&still_in_list, phase->list);
}
+ i915_gem_retire_requests(dev_priv->dev);
+
return count;
}
/**
- * i915_gem_shrink - Shrink buffer object caches completely
+ * i915_gem_shrink_all - Shrink buffer object caches completely
* @dev_priv: i915 device
*
* This is a simple wraper around i915_gem_shrink() to aggressively shrink all
*/
unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
{
- i915_gem_evict_everything(dev_priv->dev);
- return i915_gem_shrink(dev_priv, LONG_MAX,
- I915_SHRINK_BOUND | I915_SHRINK_UNBOUND);
+ return i915_gem_shrink(dev_priv, -1UL,
+ I915_SHRINK_BOUND |
+ I915_SHRINK_UNBOUND |
+ I915_SHRINK_ACTIVE);
}
static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
count += obj->base.size >> PAGE_SHIFT;
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
- if (obj->pages_pin_count == num_vma_bound(obj))
+ if (!obj->active && obj->pages_pin_count == num_vma_bound(obj))
count += obj->base.size >> PAGE_SHIFT;
}
#include <drm/i915_drm.h>
#include "i915_drv.h"
+#define KB(x) ((x) * 1024)
+#define MB(x) (KB(x) * 1024)
+
/*
* The BIOS typically reserves some of the system's memory for the exclusive
* use of the integrated graphics. This memory is no longer available for
* for is a boon.
*/
-int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
- struct drm_mm_node *node, u64 size,
- unsigned alignment)
+int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
+ struct drm_mm_node *node, u64 size,
+ unsigned alignment, u64 start, u64 end)
{
int ret;
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return -ENODEV;
+ /* See the comment at the drm_mm_init() call for more about this check.
+ * WaSkipStolenMemoryFirstPage:bdw,chv (incomplete) */
+ if (INTEL_INFO(dev_priv)->gen == 8 && start < 4096)
+ start = 4096;
+
mutex_lock(&dev_priv->mm.stolen_lock);
- ret = drm_mm_insert_node(&dev_priv->mm.stolen, node, size, alignment,
- DRM_MM_SEARCH_DEFAULT);
+ ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node, size,
+ alignment, start, end,
+ DRM_MM_SEARCH_DEFAULT);
mutex_unlock(&dev_priv->mm.stolen_lock);
return ret;
}
+int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
+ struct drm_mm_node *node, u64 size,
+ unsigned alignment)
+{
+ return i915_gem_stolen_insert_node_in_range(dev_priv, node, size,
+ alignment, 0,
+ dev_priv->gtt.stolen_usable_size);
+}
+
void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
struct drm_mm_node *node)
{
/* Almost universally we can find the Graphics Base of Stolen Memory
* at offset 0x5c in the igfx configuration space. On a few (desktop)
* machines this is also mirrored in the bridge device at different
- * locations, or in the MCHBAR. On gen2, the layout is again slightly
- * different with the Graphics Segment immediately following Top of
- * Memory (or Top of Usable DRAM). Note it appears that TOUD is only
- * reported by 865g, so we just use the top of memory as determined
- * by the e820 probe.
+ * locations, or in the MCHBAR.
+ *
+ * On 865 we just check the TOUD register.
+ *
+ * On 830/845/85x the stolen memory base isn't available in any
+ * register. We need to calculate it as TOM-TSEG_SIZE-stolen_size.
*
- * XXX However gen2 requires an unavailable symbol.
*/
base = 0;
if (INTEL_INFO(dev)->gen >= 3) {
/* Read Graphics Base of Stolen Memory directly */
pci_read_config_dword(dev->pdev, 0x5c, &base);
base &= ~((1<<20) - 1);
- } else { /* GEN2 */
-#if 0
- /* Stolen is immediately above Top of Memory */
- base = max_low_pfn_mapped << PAGE_SHIFT;
-#endif
+ } else if (IS_I865G(dev)) {
+ u16 toud = 0;
+
+ /*
+ * FIXME is the graphics stolen memory region
+ * always at TOUD? Ie. is it always the last
+ * one to be allocated by the BIOS?
+ */
+ pci_bus_read_config_word(dev->pdev->bus, PCI_DEVFN(0, 0),
+ I865_TOUD, &toud);
+
+ base = toud << 16;
+ } else if (IS_I85X(dev)) {
+ u32 tseg_size = 0;
+ u32 tom;
+ u8 tmp;
+
+ pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+ I85X_ESMRAMC, &tmp);
+
+ if (tmp & TSEG_ENABLE)
+ tseg_size = MB(1);
+
+ pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 1),
+ I85X_DRB3, &tmp);
+ tom = tmp * MB(32);
+
+ base = tom - tseg_size - dev_priv->gtt.stolen_size;
+ } else if (IS_845G(dev)) {
+ u32 tseg_size = 0;
+ u32 tom;
+ u8 tmp;
+
+ pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+ I845_ESMRAMC, &tmp);
+
+ if (tmp & TSEG_ENABLE) {
+ switch (tmp & I845_TSEG_SIZE_MASK) {
+ case I845_TSEG_SIZE_512K:
+ tseg_size = KB(512);
+ break;
+ case I845_TSEG_SIZE_1M:
+ tseg_size = MB(1);
+ break;
+ }
+ }
+
+ pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+ I830_DRB3, &tmp);
+ tom = tmp * MB(32);
+
+ base = tom - tseg_size - dev_priv->gtt.stolen_size;
+ } else if (IS_I830(dev)) {
+ u32 tseg_size = 0;
+ u32 tom;
+ u8 tmp;
+
+ pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+ I830_ESMRAMC, &tmp);
+
+ if (tmp & TSEG_ENABLE) {
+ if (tmp & I830_TSEG_SIZE_1M)
+ tseg_size = MB(1);
+ else
+ tseg_size = KB(512);
+ }
+
+ pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+ I830_DRB3, &tmp);
+ tom = tmp * MB(32);
+
+ base = tom - tseg_size - dev_priv->gtt.stolen_size;
}
if (base == 0)
drm_mm_takedown(&dev_priv->mm.stolen);
}
+static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
+ unsigned long *base, unsigned long *size)
+{
+ uint32_t reg_val = I915_READ(IS_GM45(dev_priv) ?
+ CTG_STOLEN_RESERVED :
+ ELK_STOLEN_RESERVED);
+ unsigned long stolen_top = dev_priv->mm.stolen_base +
+ dev_priv->gtt.stolen_size;
+
+ *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
+
+ WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
+
+ /* On these platforms, the register doesn't have a size field, so the
+ * size is the distance between the base and the top of the stolen
+ * memory. We also have the genuine case where base is zero and there's
+ * nothing reserved. */
+ if (*base == 0)
+ *size = 0;
+ else
+ *size = stolen_top - *base;
+}
+
static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
unsigned long *base, unsigned long *size)
{
int i915_gem_init_stolen(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- unsigned long reserved_total, reserved_base, reserved_size;
+ unsigned long reserved_total, reserved_base = 0, reserved_size;
unsigned long stolen_top;
mutex_init(&dev_priv->mm.stolen_lock);
switch (INTEL_INFO(dev_priv)->gen) {
case 2:
case 3:
+ break;
case 4:
+ if (IS_G4X(dev))
+ g4x_get_stolen_reserved(dev_priv, &reserved_base,
+ &reserved_size);
+ break;
case 5:
/* Assume the gen6 maximum for the older platforms. */
reserved_size = 1024 * 1024;
dev_priv->gtt.stolen_size >> 10,
(dev_priv->gtt.stolen_size - reserved_total) >> 10);
- /* Basic memrange allocator for stolen space */
- drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->gtt.stolen_size -
- reserved_total);
+ dev_priv->gtt.stolen_usable_size = dev_priv->gtt.stolen_size -
+ reserved_total;
+
+ /*
+ * Basic memrange allocator for stolen space.
+ *
+ * TODO: Notice that some platforms require us to not use the first page
+ * of the stolen memory but their BIOSes may still put the framebuffer
+ * on the first page. So we don't reserve this page for now because of
+ * that. Our current solution is to just prevent new nodes from being
+ * inserted on the first page - see the check we have at
+ * i915_gem_stolen_insert_node_in_range(). We may want to fix the fbcon
+ * problem later.
+ */
+ drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->gtt.stolen_usable_size);
return 0;
}
vma = i915_gem_obj_lookup_or_create_vma(obj, ggtt);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
- goto err_out;
+ goto err;
}
/* To simplify the initialisation sequence between KMS and GTT,
ret = drm_mm_reserve_node(&ggtt->mm, &vma->node);
if (ret) {
DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
- goto err_vma;
+ goto err;
}
- }
- vma->bound |= GLOBAL_BIND;
+ vma->bound |= GLOBAL_BIND;
+ list_add_tail(&vma->mm_list, &ggtt->inactive_list);
+ }
list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
- list_add_tail(&vma->mm_list, &ggtt->inactive_list);
i915_gem_object_pin_pages(obj);
return obj;
-err_vma:
- i915_gem_vma_destroy(vma);
-err_out:
- i915_gem_stolen_remove_node(dev_priv, stolen);
- kfree(stolen);
+err:
drm_gem_object_unreference(&obj->base);
return NULL;
}
struct mmu_notifier mn;
struct rb_root objects;
struct list_head linear;
- unsigned long serial;
bool has_linear;
};
struct interval_tree_node it;
struct list_head link;
struct drm_i915_gem_object *obj;
+ struct work_struct work;
+ bool active;
bool is_linear;
};
-static unsigned long cancel_userptr(struct drm_i915_gem_object *obj)
+static void __cancel_userptr__worker(struct work_struct *work)
{
+ struct i915_mmu_object *mo = container_of(work, typeof(*mo), work);
+ struct drm_i915_gem_object *obj = mo->obj;
struct drm_device *dev = obj->base.dev;
- unsigned long end;
mutex_lock(&dev->struct_mutex);
/* Cancel any active worker and force us to re-evaluate gup */
dev_priv->mm.interruptible = was_interruptible;
}
- end = obj->userptr.ptr + obj->base.size;
-
drm_gem_object_unreference(&obj->base);
mutex_unlock(&dev->struct_mutex);
-
- return end;
}
-static void *invalidate_range__linear(struct i915_mmu_notifier *mn,
- struct mm_struct *mm,
- unsigned long start,
- unsigned long end)
+static unsigned long cancel_userptr(struct i915_mmu_object *mo)
{
- struct i915_mmu_object *mo;
- unsigned long serial;
-
-restart:
- serial = mn->serial;
- list_for_each_entry(mo, &mn->linear, link) {
- struct drm_i915_gem_object *obj;
-
- if (mo->it.last < start || mo->it.start > end)
- continue;
-
- obj = mo->obj;
-
- if (!kref_get_unless_zero(&obj->base.refcount))
- continue;
-
- spin_unlock(&mn->lock);
-
- cancel_userptr(obj);
-
- spin_lock(&mn->lock);
- if (serial != mn->serial)
- goto restart;
+ unsigned long end = mo->obj->userptr.ptr + mo->obj->base.size;
+
+ /* The mmu_object is released late when destroying the
+ * GEM object so it is entirely possible to gain a
+ * reference on an object in the process of being freed
+ * since our serialisation is via the spinlock and not
+ * the struct_mutex - and consequently use it after it
+ * is freed and then double free it.
+ */
+ if (mo->active && kref_get_unless_zero(&mo->obj->base.refcount)) {
+ schedule_work(&mo->work);
+ /* only schedule one work packet to avoid the refleak */
+ mo->active = false;
}
- return NULL;
+ return end;
}
static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
unsigned long start,
unsigned long end)
{
- struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn);
- struct interval_tree_node *it = NULL;
- unsigned long next = start;
- unsigned long serial = 0;
-
- end--; /* interval ranges are inclusive, but invalidate range is exclusive */
- while (next < end) {
- struct drm_i915_gem_object *obj = NULL;
-
- spin_lock(&mn->lock);
- if (mn->has_linear)
- it = invalidate_range__linear(mn, mm, start, end);
- else if (serial == mn->serial)
- it = interval_tree_iter_next(it, next, end);
- else
- it = interval_tree_iter_first(&mn->objects, start, end);
- if (it != NULL) {
- obj = container_of(it, struct i915_mmu_object, it)->obj;
-
- /* The mmu_object is released late when destroying the
- * GEM object so it is entirely possible to gain a
- * reference on an object in the process of being freed
- * since our serialisation is via the spinlock and not
- * the struct_mutex - and consequently use it after it
- * is freed and then double free it.
- */
- if (!kref_get_unless_zero(&obj->base.refcount)) {
- spin_unlock(&mn->lock);
- serial = 0;
+ struct i915_mmu_notifier *mn =
+ container_of(_mn, struct i915_mmu_notifier, mn);
+ struct i915_mmu_object *mo;
+
+ /* interval ranges are inclusive, but invalidate range is exclusive */
+ end--;
+
+ spin_lock(&mn->lock);
+ if (mn->has_linear) {
+ list_for_each_entry(mo, &mn->linear, link) {
+ if (mo->it.last < start || mo->it.start > end)
continue;
- }
- serial = mn->serial;
+ cancel_userptr(mo);
}
- spin_unlock(&mn->lock);
- if (obj == NULL)
- return;
+ } else {
+ struct interval_tree_node *it;
- next = cancel_userptr(obj);
+ it = interval_tree_iter_first(&mn->objects, start, end);
+ while (it) {
+ mo = container_of(it, struct i915_mmu_object, it);
+ start = cancel_userptr(mo);
+ it = interval_tree_iter_next(it, start, end);
+ }
}
+ spin_unlock(&mn->lock);
}
static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
spin_lock_init(&mn->lock);
mn->mn.ops = &i915_gem_userptr_notifier;
mn->objects = RB_ROOT;
- mn->serial = 1;
INIT_LIST_HEAD(&mn->linear);
mn->has_linear = false;
return mn;
}
-static void __i915_mmu_notifier_update_serial(struct i915_mmu_notifier *mn)
-{
- if (++mn->serial == 0)
- mn->serial = 1;
-}
-
static int
i915_mmu_notifier_add(struct drm_device *dev,
struct i915_mmu_notifier *mn,
} else
interval_tree_insert(&mo->it, &mn->objects);
- if (ret == 0) {
+ if (ret == 0)
list_add(&mo->link, &mn->linear);
- __i915_mmu_notifier_update_serial(mn);
- }
+
spin_unlock(&mn->lock);
mutex_unlock(&dev->struct_mutex);
mn->has_linear = i915_mmu_notifier_has_linear(mn);
else
interval_tree_remove(&mo->it, &mn->objects);
- __i915_mmu_notifier_update_serial(mn);
spin_unlock(&mn->lock);
}
mo->it.start = obj->userptr.ptr;
mo->it.last = mo->it.start + obj->base.size - 1;
mo->obj = obj;
+ INIT_WORK(&mo->work, __cancel_userptr__worker);
ret = i915_mmu_notifier_add(obj->base.dev, mn, mo);
if (ret) {
return ret;
}
+static int
+__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj,
+ bool value)
+{
+ int ret = 0;
+
+ /* During mm_invalidate_range we need to cancel any userptr that
+ * overlaps the range being invalidated. Doing so requires the
+ * struct_mutex, and that risks recursion. In order to cause
+ * recursion, the user must alias the userptr address space with
+ * a GTT mmapping (possible with a MAP_FIXED) - then when we have
+ * to invalidate that mmaping, mm_invalidate_range is called with
+ * the userptr address *and* the struct_mutex held. To prevent that
+ * we set a flag under the i915_mmu_notifier spinlock to indicate
+ * whether this object is valid.
+ */
+#if defined(CONFIG_MMU_NOTIFIER)
+ if (obj->userptr.mmu_object == NULL)
+ return 0;
+
+ spin_lock(&obj->userptr.mmu_object->mn->lock);
+ /* In order to serialise get_pages with an outstanding
+ * cancel_userptr, we must drop the struct_mutex and try again.
+ */
+ if (!value || !work_pending(&obj->userptr.mmu_object->work))
+ obj->userptr.mmu_object->active = value;
+ else
+ ret = -EAGAIN;
+ spin_unlock(&obj->userptr.mmu_object->mn->lock);
+#endif
+
+ return ret;
+}
+
static void
__i915_gem_userptr_get_pages_worker(struct work_struct *_work)
{
struct get_pages_work *work = container_of(_work, typeof(*work), work);
struct drm_i915_gem_object *obj = work->obj;
struct drm_device *dev = obj->base.dev;
- const int num_pages = obj->base.size >> PAGE_SHIFT;
+ const int npages = obj->base.size >> PAGE_SHIFT;
struct page **pvec;
int pinned, ret;
ret = -ENOMEM;
pinned = 0;
- pvec = kmalloc(num_pages*sizeof(struct page *),
+ pvec = kmalloc(npages*sizeof(struct page *),
GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
if (pvec == NULL)
- pvec = drm_malloc_ab(num_pages, sizeof(struct page *));
+ pvec = drm_malloc_ab(npages, sizeof(struct page *));
if (pvec != NULL) {
struct mm_struct *mm = obj->userptr.mm->mm;
down_read(&mm->mmap_sem);
- while (pinned < num_pages) {
+ while (pinned < npages) {
ret = get_user_pages(work->task, mm,
obj->userptr.ptr + pinned * PAGE_SIZE,
- num_pages - pinned,
+ npages - pinned,
!obj->userptr.read_only, 0,
pvec + pinned, NULL);
if (ret < 0)
}
mutex_lock(&dev->struct_mutex);
- if (obj->userptr.work != &work->work) {
- ret = 0;
- } else if (pinned == num_pages) {
- ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
- if (ret == 0) {
- list_add_tail(&obj->global_list, &to_i915(dev)->mm.unbound_list);
- obj->get_page.sg = obj->pages->sgl;
- obj->get_page.last = 0;
-
- pinned = 0;
+ if (obj->userptr.work == &work->work) {
+ if (pinned == npages) {
+ ret = __i915_gem_userptr_set_pages(obj, pvec, npages);
+ if (ret == 0) {
+ list_add_tail(&obj->global_list,
+ &to_i915(dev)->mm.unbound_list);
+ obj->get_page.sg = obj->pages->sgl;
+ obj->get_page.last = 0;
+ pinned = 0;
+ }
}
+ obj->userptr.work = ERR_PTR(ret);
+ if (ret)
+ __i915_gem_userptr_set_active(obj, false);
}
- obj->userptr.work = ERR_PTR(ret);
obj->userptr.workers--;
drm_gem_object_unreference(&obj->base);
mutex_unlock(&dev->struct_mutex);
kfree(work);
}
+static int
+__i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
+ bool *active)
+{
+ struct get_pages_work *work;
+
+ /* Spawn a worker so that we can acquire the
+ * user pages without holding our mutex. Access
+ * to the user pages requires mmap_sem, and we have
+ * a strict lock ordering of mmap_sem, struct_mutex -
+ * we already hold struct_mutex here and so cannot
+ * call gup without encountering a lock inversion.
+ *
+ * Userspace will keep on repeating the operation
+ * (thanks to EAGAIN) until either we hit the fast
+ * path or the worker completes. If the worker is
+ * cancelled or superseded, the task is still run
+ * but the results ignored. (This leads to
+ * complications that we may have a stray object
+ * refcount that we need to be wary of when
+ * checking for existing objects during creation.)
+ * If the worker encounters an error, it reports
+ * that error back to this function through
+ * obj->userptr.work = ERR_PTR.
+ */
+ if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS)
+ return -EAGAIN;
+
+ work = kmalloc(sizeof(*work), GFP_KERNEL);
+ if (work == NULL)
+ return -ENOMEM;
+
+ obj->userptr.work = &work->work;
+ obj->userptr.workers++;
+
+ work->obj = obj;
+ drm_gem_object_reference(&obj->base);
+
+ work->task = current;
+ get_task_struct(work->task);
+
+ INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
+ schedule_work(&work->work);
+
+ *active = true;
+ return -EAGAIN;
+}
+
static int
i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
{
const int num_pages = obj->base.size >> PAGE_SHIFT;
struct page **pvec;
int pinned, ret;
+ bool active;
/* If userspace should engineer that these pages are replaced in
* the vma between us binding this page into the GTT and completion
* to the vma (discard or cloning) which should prevent the more
* egregious cases from causing harm.
*/
+ if (IS_ERR(obj->userptr.work)) {
+ /* active flag will have been dropped already by the worker */
+ ret = PTR_ERR(obj->userptr.work);
+ obj->userptr.work = NULL;
+ return ret;
+ }
+ if (obj->userptr.work)
+ /* active flag should still be held for the pending work */
+ return -EAGAIN;
+
+ /* Let the mmu-notifier know that we have begun and need cancellation */
+ ret = __i915_gem_userptr_set_active(obj, true);
+ if (ret)
+ return ret;
pvec = NULL;
pinned = 0;
GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
if (pvec == NULL) {
pvec = drm_malloc_ab(num_pages, sizeof(struct page *));
- if (pvec == NULL)
+ if (pvec == NULL) {
+ __i915_gem_userptr_set_active(obj, false);
return -ENOMEM;
+ }
}
pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages,
!obj->userptr.read_only, pvec);
}
- if (pinned < num_pages) {
- if (pinned < 0) {
- ret = pinned;
- pinned = 0;
- } else {
- /* Spawn a worker so that we can acquire the
- * user pages without holding our mutex. Access
- * to the user pages requires mmap_sem, and we have
- * a strict lock ordering of mmap_sem, struct_mutex -
- * we already hold struct_mutex here and so cannot
- * call gup without encountering a lock inversion.
- *
- * Userspace will keep on repeating the operation
- * (thanks to EAGAIN) until either we hit the fast
- * path or the worker completes. If the worker is
- * cancelled or superseded, the task is still run
- * but the results ignored. (This leads to
- * complications that we may have a stray object
- * refcount that we need to be wary of when
- * checking for existing objects during creation.)
- * If the worker encounters an error, it reports
- * that error back to this function through
- * obj->userptr.work = ERR_PTR.
- */
- ret = -EAGAIN;
- if (obj->userptr.work == NULL &&
- obj->userptr.workers < I915_GEM_USERPTR_MAX_WORKERS) {
- struct get_pages_work *work;
-
- work = kmalloc(sizeof(*work), GFP_KERNEL);
- if (work != NULL) {
- obj->userptr.work = &work->work;
- obj->userptr.workers++;
-
- work->obj = obj;
- drm_gem_object_reference(&obj->base);
-
- work->task = current;
- get_task_struct(work->task);
-
- INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
- schedule_work(&work->work);
- } else
- ret = -ENOMEM;
- } else {
- if (IS_ERR(obj->userptr.work)) {
- ret = PTR_ERR(obj->userptr.work);
- obj->userptr.work = NULL;
- }
- }
- }
- } else {
+
+ active = false;
+ if (pinned < 0)
+ ret = pinned, pinned = 0;
+ else if (pinned < num_pages)
+ ret = __i915_gem_userptr_get_pages_schedule(obj, &active);
+ else
ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
- if (ret == 0) {
- obj->userptr.work = NULL;
- pinned = 0;
- }
+ if (ret) {
+ __i915_gem_userptr_set_active(obj, active);
+ release_pages(pvec, pinned, 0);
}
-
- release_pages(pvec, pinned, 0);
drm_free_large(pvec);
return ret;
}
struct sg_page_iter sg_iter;
BUG_ON(obj->userptr.work != NULL);
+ __i915_gem_userptr_set_active(obj, false);
if (obj->madv != I915_MADV_WILLNEED)
obj->dirty = 0;
int
i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_userptr *args = data;
struct drm_i915_gem_object *obj;
int ret;
if (offset_in_page(args->user_ptr | args->user_size))
return -EINVAL;
- if (args->user_size > dev_priv->gtt.base.total)
- return -E2BIG;
-
if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE,
(char __user *)(unsigned long)args->user_ptr, args->user_size))
return -EFAULT;
#include <generated/utsrelease.h>
#include "i915_drv.h"
-static const char *yesno(int v)
-{
- return v ? "yes" : "no";
-}
-
static const char *ring_str(int ring)
{
switch (ring) {
err_printf(m, " %s [%d]:\n", name, count);
while (count--) {
- err_printf(m, " %08x %8u %02x %02x [ ",
- err->gtt_offset,
+ err_printf(m, " %08x_%08x %8u %02x %02x [ ",
+ upper_32_bits(err->gtt_offset),
+ lower_32_bits(err->gtt_offset),
err->size,
err->read_domains,
err->write_domain);
err_printf(m, " (submitted by %s [%d])",
error->ring[i].comm,
error->ring[i].pid);
- err_printf(m, " --- gtt_offset = 0x%08x\n",
- obj->gtt_offset);
+ err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
+ upper_32_bits(obj->gtt_offset),
+ lower_32_bits(obj->gtt_offset));
print_error_obj(m, obj);
}
obj = error->ring[i].wa_batchbuffer;
if (obj) {
err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
- dev_priv->ring[i].name, obj->gtt_offset);
+ dev_priv->ring[i].name,
+ lower_32_bits(obj->gtt_offset));
print_error_obj(m, obj);
}
if ((obj = error->ring[i].ringbuffer)) {
err_printf(m, "%s --- ringbuffer = 0x%08x\n",
dev_priv->ring[i].name,
- obj->gtt_offset);
+ lower_32_bits(obj->gtt_offset));
print_error_obj(m, obj);
}
if ((obj = error->ring[i].hws_page)) {
- err_printf(m, "%s --- HW Status = 0x%08x\n",
- dev_priv->ring[i].name,
- obj->gtt_offset);
+ u64 hws_offset = obj->gtt_offset;
+ u32 *hws_page = &obj->pages[0][0];
+
+ if (i915.enable_execlists) {
+ hws_offset += LRC_PPHWSP_PN * PAGE_SIZE;
+ hws_page = &obj->pages[LRC_PPHWSP_PN][0];
+ }
+ err_printf(m, "%s --- HW Status = 0x%08llx\n",
+ dev_priv->ring[i].name, hws_offset);
offset = 0;
for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
err_printf(m, "[%04x] %08x %08x %08x %08x\n",
offset,
- obj->pages[0][elt],
- obj->pages[0][elt+1],
- obj->pages[0][elt+2],
- obj->pages[0][elt+3]);
+ hws_page[elt],
+ hws_page[elt+1],
+ hws_page[elt+2],
+ hws_page[elt+3]);
offset += 16;
}
}
if ((obj = error->ring[i].ctx)) {
err_printf(m, "%s --- HW Context = 0x%08x\n",
dev_priv->ring[i].name,
- obj->gtt_offset);
+ lower_32_bits(obj->gtt_offset));
print_error_obj(m, obj);
}
}
if ((obj = error->semaphore_obj)) {
- err_printf(m, "Semaphore page = 0x%08x\n", obj->gtt_offset);
+ err_printf(m, "Semaphore page = 0x%08x\n",
+ lower_32_bits(obj->gtt_offset));
for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
err_printf(m, "[%04x] %08x %08x %08x %08x\n",
elt * 4,
int num_pages;
bool use_ggtt;
int i = 0;
- u32 reloc_offset;
+ u64 reloc_offset;
if (src == NULL || src->pages == NULL)
return NULL;
int i;
if (IS_GEN3(dev) || IS_GEN2(dev)) {
- for (i = 0; i < 8; i++)
- error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
- if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
- for (i = 0; i < 8; i++)
- error->fence[i+8] = I915_READ(FENCE_REG_945_8 +
- (i * 4));
- } else if (IS_GEN5(dev) || IS_GEN4(dev))
- for (i = 0; i < 16; i++)
- error->fence[i] = I915_READ64(FENCE_REG_965_0 +
- (i * 8));
- else if (INTEL_INFO(dev)->gen >= 6)
for (i = 0; i < dev_priv->num_fence_regs; i++)
- error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 +
- (i * 8));
+ error->fence[i] = I915_READ(FENCE_REG(i));
+ } else if (IS_GEN5(dev) || IS_GEN4(dev)) {
+ for (i = 0; i < dev_priv->num_fence_regs; i++)
+ error->fence[i] = I915_READ64(FENCE_REG_965_LO(i));
+ } else if (INTEL_INFO(dev)->gen >= 6) {
+ for (i = 0; i < dev_priv->num_fence_regs; i++)
+ error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i));
+ }
}
ering->faddr = I915_READ(DMA_FADD_I8XX);
ering->ipeir = I915_READ(IPEIR);
ering->ipehr = I915_READ(IPEHR);
- ering->instdone = I915_READ(INSTDONE);
+ ering->instdone = I915_READ(GEN2_INSTDONE);
}
ering->waiting = waitqueue_active(&ring->irq_queue);
memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
if (IS_GEN2(dev) || IS_GEN3(dev))
- instdone[0] = I915_READ(INSTDONE);
+ instdone[0] = I915_READ(GEN2_INSTDONE);
else if (IS_GEN4(dev) || IS_GEN5(dev) || IS_GEN6(dev)) {
- instdone[0] = I915_READ(INSTDONE_I965);
- instdone[1] = I915_READ(INSTDONE1);
+ instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
+ instdone[1] = I915_READ(GEN4_INSTDONE1);
} else if (INTEL_INFO(dev)->gen >= 7) {
- instdone[0] = I915_READ(GEN7_INSTDONE_1);
+ instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
instdone[1] = I915_READ(GEN7_SC_INSTDONE);
instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
#define GS_UKERNEL_READY (0xF0 << GS_UKERNEL_SHIFT)
#define GS_MIA_SHIFT 16
#define GS_MIA_MASK (0x07 << GS_MIA_SHIFT)
-
-#define GUC_WOPCM_SIZE 0xc050
-#define GUC_WOPCM_SIZE_VALUE (0x80 << 12) /* 512KB */
-#define GUC_WOPCM_OFFSET 0x80000 /* 512KB */
+#define GS_MIA_CORE_STATE (1 << GS_MIA_SHIFT)
#define SOFT_SCRATCH(n) (0xc180 + ((n) * 4))
-#define UOS_RSA_SCRATCH_0 0xc200
+#define UOS_RSA_SCRATCH(i) (0xc200 + (i) * 4)
#define DMA_ADDR_0_LOW 0xc300
#define DMA_ADDR_0_HIGH 0xc304
#define DMA_ADDR_1_LOW 0xc308
#define UOS_MOVE (1<<4)
#define START_DMA (1<<0)
#define DMA_GUC_WOPCM_OFFSET 0xc340
+#define GUC_WOPCM_OFFSET_VALUE 0x80000 /* 512KB */
+#define GUC_MAX_IDLE_COUNT 0xC3E4
+
+#define GUC_WOPCM_SIZE 0xc050
+#define GUC_WOPCM_SIZE_VALUE (0x80 << 12) /* 512KB */
+
+/* GuC addresses below GUC_WOPCM_TOP don't map through the GTT */
+#define GUC_WOPCM_TOP (GUC_WOPCM_SIZE_VALUE)
#define GEN8_GT_PM_CONFIG 0x138140
+#define GEN9LP_GT_PM_CONFIG 0x138140
#define GEN9_GT_PM_CONFIG 0x13816c
-#define GEN8_GT_DOORBELL_ENABLE (1<<0)
+#define GT_DOORBELL_ENABLE (1<<0)
#define GEN8_GTCR 0x4274
#define GEN8_GTCR_INVALIDATE (1<<0)
GUC_ENABLE_READ_CACHE_LOGIC | \
GUC_ENABLE_MIA_CACHING | \
GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA | \
- GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA)
+ GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA | \
+ GUC_ENABLE_MIA_CLOCK_GATING)
#define HOST2GUC_INTERRUPT 0xc4c8
#define HOST2GUC_TRIGGER (1<<0)
--- /dev/null
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+#include <linux/firmware.h>
+#include <linux/circ_buf.h>
+#include "i915_drv.h"
+#include "intel_guc.h"
+
+/**
+ * DOC: GuC Client
+ *
+ * i915_guc_client:
+ * We use the term client to avoid confusion with contexts. A i915_guc_client is
+ * equivalent to GuC object guc_context_desc. This context descriptor is
+ * allocated from a pool of 1024 entries. Kernel driver will allocate doorbell
+ * and workqueue for it. Also the process descriptor (guc_process_desc), which
+ * is mapped to client space. So the client can write Work Item then ring the
+ * doorbell.
+ *
+ * To simplify the implementation, we allocate one gem object that contains all
+ * pages for doorbell, process descriptor and workqueue.
+ *
+ * The Scratch registers:
+ * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
+ * a value to the action register (SOFT_SCRATCH_0) along with any data. It then
+ * triggers an interrupt on the GuC via another register write (0xC4C8).
+ * Firmware writes a success/fail code back to the action register after
+ * processes the request. The kernel driver polls waiting for this update and
+ * then proceeds.
+ * See host2guc_action()
+ *
+ * Doorbells:
+ * Doorbells are interrupts to uKernel. A doorbell is a single cache line (QW)
+ * mapped into process space.
+ *
+ * Work Items:
+ * There are several types of work items that the host may place into a
+ * workqueue, each with its own requirements and limitations. Currently only
+ * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which
+ * represents in-order queue. The kernel driver packs ring tail pointer and an
+ * ELSP context descriptor dword into Work Item.
+ * See guc_add_workqueue_item()
+ *
+ */
+
+/*
+ * Read GuC command/status register (SOFT_SCRATCH_0)
+ * Return true if it contains a response rather than a command
+ */
+static inline bool host2guc_action_response(struct drm_i915_private *dev_priv,
+ u32 *status)
+{
+ u32 val = I915_READ(SOFT_SCRATCH(0));
+ *status = val;
+ return GUC2HOST_IS_RESPONSE(val);
+}
+
+static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
+{
+ struct drm_i915_private *dev_priv = guc_to_i915(guc);
+ u32 status;
+ int i;
+ int ret;
+
+ if (WARN_ON(len < 1 || len > 15))
+ return -EINVAL;
+
+ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+ spin_lock(&dev_priv->guc.host2guc_lock);
+
+ dev_priv->guc.action_count += 1;
+ dev_priv->guc.action_cmd = data[0];
+
+ for (i = 0; i < len; i++)
+ I915_WRITE(SOFT_SCRATCH(i), data[i]);
+
+ POSTING_READ(SOFT_SCRATCH(i - 1));
+
+ I915_WRITE(HOST2GUC_INTERRUPT, HOST2GUC_TRIGGER);
+
+ /* No HOST2GUC command should take longer than 10ms */
+ ret = wait_for_atomic(host2guc_action_response(dev_priv, &status), 10);
+ if (status != GUC2HOST_STATUS_SUCCESS) {
+ /*
+ * Either the GuC explicitly returned an error (which
+ * we convert to -EIO here) or no response at all was
+ * received within the timeout limit (-ETIMEDOUT)
+ */
+ if (ret != -ETIMEDOUT)
+ ret = -EIO;
+
+ DRM_ERROR("GUC: host2guc action 0x%X failed. ret=%d "
+ "status=0x%08X response=0x%08X\n",
+ data[0], ret, status,
+ I915_READ(SOFT_SCRATCH(15)));
+
+ dev_priv->guc.action_fail += 1;
+ dev_priv->guc.action_err = ret;
+ }
+ dev_priv->guc.action_status = status;
+
+ spin_unlock(&dev_priv->guc.host2guc_lock);
+ intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+
+ return ret;
+}
+
+/*
+ * Tell the GuC to allocate or deallocate a specific doorbell
+ */
+
+static int host2guc_allocate_doorbell(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ u32 data[2];
+
+ data[0] = HOST2GUC_ACTION_ALLOCATE_DOORBELL;
+ data[1] = client->ctx_index;
+
+ return host2guc_action(guc, data, 2);
+}
+
+static int host2guc_release_doorbell(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ u32 data[2];
+
+ data[0] = HOST2GUC_ACTION_DEALLOCATE_DOORBELL;
+ data[1] = client->ctx_index;
+
+ return host2guc_action(guc, data, 2);
+}
+
+static int host2guc_sample_forcewake(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ struct drm_i915_private *dev_priv = guc_to_i915(guc);
+ struct drm_device *dev = dev_priv->dev;
+ u32 data[2];
+
+ data[0] = HOST2GUC_ACTION_SAMPLE_FORCEWAKE;
+ /* WaRsDisableCoarsePowerGating:skl,bxt */
+ if (!intel_enable_rc6(dev_priv->dev) ||
+ (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) ||
+ (IS_SKL_GT3(dev) && (INTEL_REVID(dev) <= SKL_REVID_E0)) ||
+ (IS_SKL_GT4(dev) && (INTEL_REVID(dev) <= SKL_REVID_E0)))
+ data[1] = 0;
+ else
+ /* bit 0 and 1 are for Render and Media domain separately */
+ data[1] = GUC_FORCEWAKE_RENDER | GUC_FORCEWAKE_MEDIA;
+
+ return host2guc_action(guc, data, ARRAY_SIZE(data));
+}
+
+/*
+ * Initialise, update, or clear doorbell data shared with the GuC
+ *
+ * These functions modify shared data and so need access to the mapped
+ * client object which contains the page being used for the doorbell
+ */
+
+static void guc_init_doorbell(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ struct guc_doorbell_info *doorbell;
+ void *base;
+
+ base = kmap_atomic(i915_gem_object_get_page(client->client_obj, 0));
+ doorbell = base + client->doorbell_offset;
+
+ doorbell->db_status = 1;
+ doorbell->cookie = 0;
+
+ kunmap_atomic(base);
+}
+
+static int guc_ring_doorbell(struct i915_guc_client *gc)
+{
+ struct guc_process_desc *desc;
+ union guc_doorbell_qw db_cmp, db_exc, db_ret;
+ union guc_doorbell_qw *db;
+ void *base;
+ int attempt = 2, ret = -EAGAIN;
+
+ base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, 0));
+ desc = base + gc->proc_desc_offset;
+
+ /* Update the tail so it is visible to GuC */
+ desc->tail = gc->wq_tail;
+
+ /* current cookie */
+ db_cmp.db_status = GUC_DOORBELL_ENABLED;
+ db_cmp.cookie = gc->cookie;
+
+ /* cookie to be updated */
+ db_exc.db_status = GUC_DOORBELL_ENABLED;
+ db_exc.cookie = gc->cookie + 1;
+ if (db_exc.cookie == 0)
+ db_exc.cookie = 1;
+
+ /* pointer of current doorbell cacheline */
+ db = base + gc->doorbell_offset;
+
+ while (attempt--) {
+ /* lets ring the doorbell */
+ db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db,
+ db_cmp.value_qw, db_exc.value_qw);
+
+ /* if the exchange was successfully executed */
+ if (db_ret.value_qw == db_cmp.value_qw) {
+ /* db was successfully rung */
+ gc->cookie = db_exc.cookie;
+ ret = 0;
+ break;
+ }
+
+ /* XXX: doorbell was lost and need to acquire it again */
+ if (db_ret.db_status == GUC_DOORBELL_DISABLED)
+ break;
+
+ DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n",
+ db_cmp.cookie, db_ret.cookie);
+
+ /* update the cookie to newly read cookie from GuC */
+ db_cmp.cookie = db_ret.cookie;
+ db_exc.cookie = db_ret.cookie + 1;
+ if (db_exc.cookie == 0)
+ db_exc.cookie = 1;
+ }
+
+ kunmap_atomic(base);
+ return ret;
+}
+
+static void guc_disable_doorbell(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ struct drm_i915_private *dev_priv = guc_to_i915(guc);
+ struct guc_doorbell_info *doorbell;
+ void *base;
+ int drbreg = GEN8_DRBREGL(client->doorbell_id);
+ int value;
+
+ base = kmap_atomic(i915_gem_object_get_page(client->client_obj, 0));
+ doorbell = base + client->doorbell_offset;
+
+ doorbell->db_status = 0;
+
+ kunmap_atomic(base);
+
+ I915_WRITE(drbreg, I915_READ(drbreg) & ~GEN8_DRB_VALID);
+
+ value = I915_READ(drbreg);
+ WARN_ON((value & GEN8_DRB_VALID) != 0);
+
+ I915_WRITE(GEN8_DRBREGU(client->doorbell_id), 0);
+ I915_WRITE(drbreg, 0);
+
+ /* XXX: wait for any interrupts */
+ /* XXX: wait for workqueue to drain */
+}
+
+/*
+ * Select, assign and relase doorbell cachelines
+ *
+ * These functions track which doorbell cachelines are in use.
+ * The data they manipulate is protected by the host2guc lock.
+ */
+
+static uint32_t select_doorbell_cacheline(struct intel_guc *guc)
+{
+ const uint32_t cacheline_size = cache_line_size();
+ uint32_t offset;
+
+ spin_lock(&guc->host2guc_lock);
+
+ /* Doorbell uses a single cache line within a page */
+ offset = offset_in_page(guc->db_cacheline);
+
+ /* Moving to next cache line to reduce contention */
+ guc->db_cacheline += cacheline_size;
+
+ spin_unlock(&guc->host2guc_lock);
+
+ DRM_DEBUG_DRIVER("selected doorbell cacheline 0x%x, next 0x%x, linesize %u\n",
+ offset, guc->db_cacheline, cacheline_size);
+
+ return offset;
+}
+
+static uint16_t assign_doorbell(struct intel_guc *guc, uint32_t priority)
+{
+ /*
+ * The bitmap is split into two halves; the first half is used for
+ * normal priority contexts, the second half for high-priority ones.
+ * Note that logically higher priorities are numerically less than
+ * normal ones, so the test below means "is it high-priority?"
+ */
+ const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH);
+ const uint16_t half = GUC_MAX_DOORBELLS / 2;
+ const uint16_t start = hi_pri ? half : 0;
+ const uint16_t end = start + half;
+ uint16_t id;
+
+ spin_lock(&guc->host2guc_lock);
+ id = find_next_zero_bit(guc->doorbell_bitmap, end, start);
+ if (id == end)
+ id = GUC_INVALID_DOORBELL_ID;
+ else
+ bitmap_set(guc->doorbell_bitmap, id, 1);
+ spin_unlock(&guc->host2guc_lock);
+
+ DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n",
+ hi_pri ? "high" : "normal", id);
+
+ return id;
+}
+
+static void release_doorbell(struct intel_guc *guc, uint16_t id)
+{
+ spin_lock(&guc->host2guc_lock);
+ bitmap_clear(guc->doorbell_bitmap, id, 1);
+ spin_unlock(&guc->host2guc_lock);
+}
+
+/*
+ * Initialise the process descriptor shared with the GuC firmware.
+ */
+static void guc_init_proc_desc(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ struct guc_process_desc *desc;
+ void *base;
+
+ base = kmap_atomic(i915_gem_object_get_page(client->client_obj, 0));
+ desc = base + client->proc_desc_offset;
+
+ memset(desc, 0, sizeof(*desc));
+
+ /*
+ * XXX: pDoorbell and WQVBaseAddress are pointers in process address
+ * space for ring3 clients (set them as in mmap_ioctl) or kernel
+ * space for kernel clients (map on demand instead? May make debug
+ * easier to have it mapped).
+ */
+ desc->wq_base_addr = 0;
+ desc->db_base_addr = 0;
+
+ desc->context_id = client->ctx_index;
+ desc->wq_size_bytes = client->wq_size;
+ desc->wq_status = WQ_STATUS_ACTIVE;
+ desc->priority = client->priority;
+
+ kunmap_atomic(base);
+}
+
+/*
+ * Initialise/clear the context descriptor shared with the GuC firmware.
+ *
+ * This descriptor tells the GuC where (in GGTT space) to find the important
+ * data structures relating to this client (doorbell, process descriptor,
+ * write queue, etc).
+ */
+
+static void guc_init_ctx_desc(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ struct intel_context *ctx = client->owner;
+ struct guc_context_desc desc;
+ struct sg_table *sg;
+ int i;
+
+ memset(&desc, 0, sizeof(desc));
+
+ desc.attribute = GUC_CTX_DESC_ATTR_ACTIVE | GUC_CTX_DESC_ATTR_KERNEL;
+ desc.context_id = client->ctx_index;
+ desc.priority = client->priority;
+ desc.db_id = client->doorbell_id;
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ struct guc_execlist_context *lrc = &desc.lrc[i];
+ struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
+ struct intel_engine_cs *ring;
+ struct drm_i915_gem_object *obj;
+ uint64_t ctx_desc;
+
+ /* TODO: We have a design issue to be solved here. Only when we
+ * receive the first batch, we know which engine is used by the
+ * user. But here GuC expects the lrc and ring to be pinned. It
+ * is not an issue for default context, which is the only one
+ * for now who owns a GuC client. But for future owner of GuC
+ * client, need to make sure lrc is pinned prior to enter here.
+ */
+ obj = ctx->engine[i].state;
+ if (!obj)
+ break; /* XXX: continue? */
+
+ ring = ringbuf->ring;
+ ctx_desc = intel_lr_context_descriptor(ctx, ring);
+ lrc->context_desc = (u32)ctx_desc;
+
+ /* The state page is after PPHWSP */
+ lrc->ring_lcra = i915_gem_obj_ggtt_offset(obj) +
+ LRC_STATE_PN * PAGE_SIZE;
+ lrc->context_id = (client->ctx_index << GUC_ELC_CTXID_OFFSET) |
+ (ring->id << GUC_ELC_ENGINE_OFFSET);
+
+ obj = ringbuf->obj;
+
+ lrc->ring_begin = i915_gem_obj_ggtt_offset(obj);
+ lrc->ring_end = lrc->ring_begin + obj->base.size - 1;
+ lrc->ring_next_free_location = lrc->ring_begin;
+ lrc->ring_current_tail_pointer_value = 0;
+
+ desc.engines_used |= (1 << ring->id);
+ }
+
+ WARN_ON(desc.engines_used == 0);
+
+ /*
+ * The CPU address is only needed at certain points, so kmap_atomic on
+ * demand instead of storing it in the ctx descriptor.
+ * XXX: May make debug easier to have it mapped
+ */
+ desc.db_trigger_cpu = 0;
+ desc.db_trigger_uk = client->doorbell_offset +
+ i915_gem_obj_ggtt_offset(client->client_obj);
+ desc.db_trigger_phy = client->doorbell_offset +
+ sg_dma_address(client->client_obj->pages->sgl);
+
+ desc.process_desc = client->proc_desc_offset +
+ i915_gem_obj_ggtt_offset(client->client_obj);
+
+ desc.wq_addr = client->wq_offset +
+ i915_gem_obj_ggtt_offset(client->client_obj);
+
+ desc.wq_size = client->wq_size;
+
+ /*
+ * XXX: Take LRCs from an existing intel_context if this is not an
+ * IsKMDCreatedContext client
+ */
+ desc.desc_private = (uintptr_t)client;
+
+ /* Pool context is pinned already */
+ sg = guc->ctx_pool_obj->pages;
+ sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
+ sizeof(desc) * client->ctx_index);
+}
+
+static void guc_fini_ctx_desc(struct intel_guc *guc,
+ struct i915_guc_client *client)
+{
+ struct guc_context_desc desc;
+ struct sg_table *sg;
+
+ memset(&desc, 0, sizeof(desc));
+
+ sg = guc->ctx_pool_obj->pages;
+ sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
+ sizeof(desc) * client->ctx_index);
+}
+
+/* Get valid workqueue item and return it back to offset */
+static int guc_get_workqueue_space(struct i915_guc_client *gc, u32 *offset)
+{
+ struct guc_process_desc *desc;
+ void *base;
+ u32 size = sizeof(struct guc_wq_item);
+ int ret = 0, timeout_counter = 200;
+
+ base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, 0));
+ desc = base + gc->proc_desc_offset;
+
+ while (timeout_counter-- > 0) {
+ ret = wait_for_atomic(CIRC_SPACE(gc->wq_tail, desc->head,
+ gc->wq_size) >= size, 1);
+
+ if (!ret) {
+ *offset = gc->wq_tail;
+
+ /* advance the tail for next workqueue item */
+ gc->wq_tail += size;
+ gc->wq_tail &= gc->wq_size - 1;
+
+ /* this will break the loop */
+ timeout_counter = 0;
+ }
+ };
+
+ kunmap_atomic(base);
+
+ return ret;
+}
+
+static int guc_add_workqueue_item(struct i915_guc_client *gc,
+ struct drm_i915_gem_request *rq)
+{
+ enum intel_ring_id ring_id = rq->ring->id;
+ struct guc_wq_item *wqi;
+ void *base;
+ u32 tail, wq_len, wq_off = 0;
+ int ret;
+
+ ret = guc_get_workqueue_space(gc, &wq_off);
+ if (ret)
+ return ret;
+
+ /* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
+ * should not have the case where structure wqi is across page, neither
+ * wrapped to the beginning. This simplifies the implementation below.
+ *
+ * XXX: if not the case, we need save data to a temp wqi and copy it to
+ * workqueue buffer dw by dw.
+ */
+ WARN_ON(sizeof(struct guc_wq_item) != 16);
+ WARN_ON(wq_off & 3);
+
+ /* wq starts from the page after doorbell / process_desc */
+ base = kmap_atomic(i915_gem_object_get_page(gc->client_obj,
+ (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT));
+ wq_off &= PAGE_SIZE - 1;
+ wqi = (struct guc_wq_item *)((char *)base + wq_off);
+
+ /* len does not include the header */
+ wq_len = sizeof(struct guc_wq_item) / sizeof(u32) - 1;
+ wqi->header = WQ_TYPE_INORDER |
+ (wq_len << WQ_LEN_SHIFT) |
+ (ring_id << WQ_TARGET_SHIFT) |
+ WQ_NO_WCFLUSH_WAIT;
+
+ /* The GuC wants only the low-order word of the context descriptor */
+ wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, rq->ring);
+
+ /* The GuC firmware wants the tail index in QWords, not bytes */
+ tail = rq->ringbuf->tail >> 3;
+ wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT;
+ wqi->fence_id = 0; /*XXX: what fence to be here */
+
+ kunmap_atomic(base);
+
+ return 0;
+}
+
+#define CTX_RING_BUFFER_START 0x08
+
+/* Update the ringbuffer pointer in a saved context image */
+static void lr_context_update(struct drm_i915_gem_request *rq)
+{
+ enum intel_ring_id ring_id = rq->ring->id;
+ struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring_id].state;
+ struct drm_i915_gem_object *rb_obj = rq->ringbuf->obj;
+ struct page *page;
+ uint32_t *reg_state;
+
+ BUG_ON(!ctx_obj);
+ WARN_ON(!i915_gem_obj_is_pinned(ctx_obj));
+ WARN_ON(!i915_gem_obj_is_pinned(rb_obj));
+
+ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+ reg_state = kmap_atomic(page);
+
+ reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(rb_obj);
+
+ kunmap_atomic(reg_state);
+}
+
+/**
+ * i915_guc_submit() - Submit commands through GuC
+ * @client: the guc client where commands will go through
+ * @ctx: LRC where commands come from
+ * @ring: HW engine that will excute the commands
+ *
+ * Return: 0 if succeed
+ */
+int i915_guc_submit(struct i915_guc_client *client,
+ struct drm_i915_gem_request *rq)
+{
+ struct intel_guc *guc = client->guc;
+ enum intel_ring_id ring_id = rq->ring->id;
+ unsigned long flags;
+ int q_ret, b_ret;
+
+ /* Need this because of the deferred pin ctx and ring */
+ /* Shall we move this right after ring is pinned? */
+ lr_context_update(rq);
+
+ spin_lock_irqsave(&client->wq_lock, flags);
+
+ q_ret = guc_add_workqueue_item(client, rq);
+ if (q_ret == 0)
+ b_ret = guc_ring_doorbell(client);
+
+ client->submissions[ring_id] += 1;
+ if (q_ret) {
+ client->q_fail += 1;
+ client->retcode = q_ret;
+ } else if (b_ret) {
+ client->b_fail += 1;
+ client->retcode = q_ret = b_ret;
+ } else {
+ client->retcode = 0;
+ }
+ spin_unlock_irqrestore(&client->wq_lock, flags);
+
+ spin_lock(&guc->host2guc_lock);
+ guc->submissions[ring_id] += 1;
+ guc->last_seqno[ring_id] = rq->seqno;
+ spin_unlock(&guc->host2guc_lock);
+
+ return q_ret;
+}
+
+/*
+ * Everything below here is concerned with setup & teardown, and is
+ * therefore not part of the somewhat time-critical batch-submission
+ * path of i915_guc_submit() above.
+ */
+
+/**
+ * gem_allocate_guc_obj() - Allocate gem object for GuC usage
+ * @dev: drm device
+ * @size: size of object
+ *
+ * This is a wrapper to create a gem obj. In order to use it inside GuC, the
+ * object needs to be pinned lifetime. Also we must pin it to gtt space other
+ * than [0, GUC_WOPCM_TOP) because this range is reserved inside GuC.
+ *
+ * Return: A drm_i915_gem_object if successful, otherwise NULL.
+ */
+static struct drm_i915_gem_object *gem_allocate_guc_obj(struct drm_device *dev,
+ u32 size)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_gem_object *obj;
+
+ obj = i915_gem_alloc_object(dev, size);
+ if (!obj)
+ return NULL;
+
+ if (i915_gem_object_get_pages(obj)) {
+ drm_gem_object_unreference(&obj->base);
+ return NULL;
+ }
+
+ if (i915_gem_obj_ggtt_pin(obj, PAGE_SIZE,
+ PIN_OFFSET_BIAS | GUC_WOPCM_TOP)) {
+ drm_gem_object_unreference(&obj->base);
+ return NULL;
+ }
+
+ /* Invalidate GuC TLB to let GuC take the latest updates to GTT. */
+ I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+
+ return obj;
+}
+
+/**
+ * gem_release_guc_obj() - Release gem object allocated for GuC usage
+ * @obj: gem obj to be released
+ */
+static void gem_release_guc_obj(struct drm_i915_gem_object *obj)
+{
+ if (!obj)
+ return;
+
+ if (i915_gem_obj_is_pinned(obj))
+ i915_gem_object_ggtt_unpin(obj);
+
+ drm_gem_object_unreference(&obj->base);
+}
+
+static void guc_client_free(struct drm_device *dev,
+ struct i915_guc_client *client)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc *guc = &dev_priv->guc;
+
+ if (!client)
+ return;
+
+ if (client->doorbell_id != GUC_INVALID_DOORBELL_ID) {
+ /*
+ * First disable the doorbell, then tell the GuC we've
+ * finished with it, finally deallocate it in our bitmap
+ */
+ guc_disable_doorbell(guc, client);
+ host2guc_release_doorbell(guc, client);
+ release_doorbell(guc, client->doorbell_id);
+ }
+
+ /*
+ * XXX: wait for any outstanding submissions before freeing memory.
+ * Be sure to drop any locks
+ */
+
+ gem_release_guc_obj(client->client_obj);
+
+ if (client->ctx_index != GUC_INVALID_CTX_ID) {
+ guc_fini_ctx_desc(guc, client);
+ ida_simple_remove(&guc->ctx_ids, client->ctx_index);
+ }
+
+ kfree(client);
+}
+
+/**
+ * guc_client_alloc() - Allocate an i915_guc_client
+ * @dev: drm device
+ * @priority: four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW
+ * The kernel client to replace ExecList submission is created with
+ * NORMAL priority. Priority of a client for scheduler can be HIGH,
+ * while a preemption context can use CRITICAL.
+ * @ctx the context to own the client (we use the default render context)
+ *
+ * Return: An i915_guc_client object if success.
+ */
+static struct i915_guc_client *guc_client_alloc(struct drm_device *dev,
+ uint32_t priority,
+ struct intel_context *ctx)
+{
+ struct i915_guc_client *client;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc *guc = &dev_priv->guc;
+ struct drm_i915_gem_object *obj;
+
+ client = kzalloc(sizeof(*client), GFP_KERNEL);
+ if (!client)
+ return NULL;
+
+ client->doorbell_id = GUC_INVALID_DOORBELL_ID;
+ client->priority = priority;
+ client->owner = ctx;
+ client->guc = guc;
+
+ client->ctx_index = (uint32_t)ida_simple_get(&guc->ctx_ids, 0,
+ GUC_MAX_GPU_CONTEXTS, GFP_KERNEL);
+ if (client->ctx_index >= GUC_MAX_GPU_CONTEXTS) {
+ client->ctx_index = GUC_INVALID_CTX_ID;
+ goto err;
+ }
+
+ /* The first page is doorbell/proc_desc. Two followed pages are wq. */
+ obj = gem_allocate_guc_obj(dev, GUC_DB_SIZE + GUC_WQ_SIZE);
+ if (!obj)
+ goto err;
+
+ client->client_obj = obj;
+ client->wq_offset = GUC_DB_SIZE;
+ client->wq_size = GUC_WQ_SIZE;
+ spin_lock_init(&client->wq_lock);
+
+ client->doorbell_offset = select_doorbell_cacheline(guc);
+
+ /*
+ * Since the doorbell only requires a single cacheline, we can save
+ * space by putting the application process descriptor in the same
+ * page. Use the half of the page that doesn't include the doorbell.
+ */
+ if (client->doorbell_offset >= (GUC_DB_SIZE / 2))
+ client->proc_desc_offset = 0;
+ else
+ client->proc_desc_offset = (GUC_DB_SIZE / 2);
+
+ client->doorbell_id = assign_doorbell(guc, client->priority);
+ if (client->doorbell_id == GUC_INVALID_DOORBELL_ID)
+ /* XXX: evict a doorbell instead */
+ goto err;
+
+ guc_init_proc_desc(guc, client);
+ guc_init_ctx_desc(guc, client);
+ guc_init_doorbell(guc, client);
+
+ /* XXX: Any cache flushes needed? General domain mgmt calls? */
+
+ if (host2guc_allocate_doorbell(guc, client))
+ goto err;
+
+ DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u db_id %u\n",
+ priority, client, client->ctx_index, client->doorbell_id);
+
+ return client;
+
+err:
+ DRM_ERROR("FAILED to create priority %u GuC client!\n", priority);
+
+ guc_client_free(dev, client);
+ return NULL;
+}
+
+static void guc_create_log(struct intel_guc *guc)
+{
+ struct drm_i915_private *dev_priv = guc_to_i915(guc);
+ struct drm_i915_gem_object *obj;
+ unsigned long offset;
+ uint32_t size, flags;
+
+ if (i915.guc_log_level < GUC_LOG_VERBOSITY_MIN)
+ return;
+
+ if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX)
+ i915.guc_log_level = GUC_LOG_VERBOSITY_MAX;
+
+ /* The first page is to save log buffer state. Allocate one
+ * extra page for others in case for overlap */
+ size = (1 + GUC_LOG_DPC_PAGES + 1 +
+ GUC_LOG_ISR_PAGES + 1 +
+ GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT;
+
+ obj = guc->log_obj;
+ if (!obj) {
+ obj = gem_allocate_guc_obj(dev_priv->dev, size);
+ if (!obj) {
+ /* logging will be off */
+ i915.guc_log_level = -1;
+ return;
+ }
+
+ guc->log_obj = obj;
+ }
+
+ /* each allocated unit is a page */
+ flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL |
+ (GUC_LOG_DPC_PAGES << GUC_LOG_DPC_SHIFT) |
+ (GUC_LOG_ISR_PAGES << GUC_LOG_ISR_SHIFT) |
+ (GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT);
+
+ offset = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; /* in pages */
+ guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags;
+}
+
+/*
+ * Set up the memory resources to be shared with the GuC. At this point,
+ * we require just one object that can be mapped through the GGTT.
+ */
+int i915_guc_submission_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ const size_t ctxsize = sizeof(struct guc_context_desc);
+ const size_t poolsize = GUC_MAX_GPU_CONTEXTS * ctxsize;
+ const size_t gemsize = round_up(poolsize, PAGE_SIZE);
+ struct intel_guc *guc = &dev_priv->guc;
+
+ if (!i915.enable_guc_submission)
+ return 0; /* not enabled */
+
+ if (guc->ctx_pool_obj)
+ return 0; /* already allocated */
+
+ guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv->dev, gemsize);
+ if (!guc->ctx_pool_obj)
+ return -ENOMEM;
+
+ spin_lock_init(&dev_priv->guc.host2guc_lock);
+
+ ida_init(&guc->ctx_ids);
+
+ guc_create_log(guc);
+
+ return 0;
+}
+
+int i915_guc_submission_enable(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc *guc = &dev_priv->guc;
+ struct intel_context *ctx = dev_priv->ring[RCS].default_context;
+ struct i915_guc_client *client;
+
+ /* client for execbuf submission */
+ client = guc_client_alloc(dev, GUC_CTX_PRIORITY_KMD_NORMAL, ctx);
+ if (!client) {
+ DRM_ERROR("Failed to create execbuf guc_client\n");
+ return -ENOMEM;
+ }
+
+ guc->execbuf_client = client;
+
+ host2guc_sample_forcewake(guc, client);
+
+ return 0;
+}
+
+void i915_guc_submission_disable(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc *guc = &dev_priv->guc;
+
+ guc_client_free(dev, guc->execbuf_client);
+ guc->execbuf_client = NULL;
+}
+
+void i915_guc_submission_fini(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc *guc = &dev_priv->guc;
+
+ gem_release_guc_obj(dev_priv->guc.log_obj);
+ guc->log_obj = NULL;
+
+ if (guc->ctx_pool_obj)
+ ida_destroy(&guc->ctx_ids);
+ gem_release_guc_obj(guc->ctx_pool_obj);
+ guc->ctx_pool_obj = NULL;
+}
+
+/**
+ * intel_guc_suspend() - notify GuC entering suspend state
+ * @dev: drm device
+ */
+int intel_guc_suspend(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc *guc = &dev_priv->guc;
+ struct intel_context *ctx;
+ u32 data[3];
+
+ if (!i915.enable_guc_submission)
+ return 0;
+
+ ctx = dev_priv->ring[RCS].default_context;
+
+ data[0] = HOST2GUC_ACTION_ENTER_S_STATE;
+ /* any value greater than GUC_POWER_D0 */
+ data[1] = GUC_POWER_D1;
+ /* first page is shared data with GuC */
+ data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state);
+
+ return host2guc_action(guc, data, ARRAY_SIZE(data));
+}
+
+
+/**
+ * intel_guc_resume() - notify GuC resuming from suspend state
+ * @dev: drm device
+ */
+int intel_guc_resume(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc *guc = &dev_priv->guc;
+ struct intel_context *ctx;
+ u32 data[3];
+
+ if (!i915.enable_guc_submission)
+ return 0;
+
+ ctx = dev_priv->ring[RCS].default_context;
+
+ data[0] = HOST2GUC_ACTION_EXIT_S_STATE;
+ data[1] = GUC_POWER_D0;
+ /* first page is shared data with GuC */
+ data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state);
+
+ return host2guc_action(guc, data, ARRAY_SIZE(data));
+}
* and related files, but that will be described in separate chapters.
*/
+static const u32 hpd_ilk[HPD_NUM_PINS] = {
+ [HPD_PORT_A] = DE_DP_A_HOTPLUG,
+};
+
+static const u32 hpd_ivb[HPD_NUM_PINS] = {
+ [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
+};
+
+static const u32 hpd_bdw[HPD_NUM_PINS] = {
+ [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
+};
+
static const u32 hpd_ibx[HPD_NUM_PINS] = {
[HPD_CRT] = SDE_CRT_HOTPLUG,
[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
};
static const u32 hpd_spt[HPD_NUM_PINS] = {
+ [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
+ [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};
/*
* We should clear IMR at preinstall/uninstall, and just check at postinstall.
*/
-#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
- u32 val = I915_READ(reg); \
- if (val) { \
- WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
- (reg), val); \
- I915_WRITE((reg), 0xffffffff); \
- POSTING_READ(reg); \
- I915_WRITE((reg), 0xffffffff); \
- POSTING_READ(reg); \
- } \
-} while (0)
+static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
+{
+ u32 val = I915_READ(reg);
+
+ if (val == 0)
+ return;
+
+ WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
+ reg, val);
+ I915_WRITE(reg, 0xffffffff);
+ POSTING_READ(reg);
+ I915_WRITE(reg, 0xffffffff);
+ POSTING_READ(reg);
+}
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
- GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
+ gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
POSTING_READ(GEN8_##type##_IMR(which)); \
} while (0)
#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
- GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
+ gen5_assert_iir_is_zero(dev_priv, type##IIR); \
I915_WRITE(type##IER, (ier_val)); \
I915_WRITE(type##IMR, (imr_val)); \
POSTING_READ(type##IMR); \
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
/* For display hotplug interrupt */
-void
-ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
+static inline void
+i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
+ uint32_t mask,
+ uint32_t bits)
{
+ uint32_t val;
+
assert_spin_locked(&dev_priv->irq_lock);
+ WARN_ON(bits & ~mask);
- if (WARN_ON(!intel_irqs_enabled(dev_priv)))
- return;
+ val = I915_READ(PORT_HOTPLUG_EN);
+ val &= ~mask;
+ val |= bits;
+ I915_WRITE(PORT_HOTPLUG_EN, val);
+}
- if ((dev_priv->irq_mask & mask) != 0) {
- dev_priv->irq_mask &= ~mask;
- I915_WRITE(DEIMR, dev_priv->irq_mask);
- POSTING_READ(DEIMR);
- }
+/**
+ * i915_hotplug_interrupt_update - update hotplug interrupt enable
+ * @dev_priv: driver private
+ * @mask: bits to update
+ * @bits: bits to enable
+ * NOTE: the HPD enable bits are modified both inside and outside
+ * of an interrupt context. To avoid that read-modify-write cycles
+ * interfer, these bits are protected by a spinlock. Since this
+ * function is usually not called from a context where the lock is
+ * held already, this function acquires the lock itself. A non-locking
+ * version is also available.
+ */
+void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
+ uint32_t mask,
+ uint32_t bits)
+{
+ spin_lock_irq(&dev_priv->irq_lock);
+ i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
+ spin_unlock_irq(&dev_priv->irq_lock);
}
-void
-ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
+/**
+ * ilk_update_display_irq - update DEIMR
+ * @dev_priv: driver private
+ * @interrupt_mask: mask of interrupt bits to update
+ * @enabled_irq_mask: mask of interrupt bits to enable
+ */
+static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
+ uint32_t interrupt_mask,
+ uint32_t enabled_irq_mask)
{
+ uint32_t new_val;
+
assert_spin_locked(&dev_priv->irq_lock);
+ WARN_ON(enabled_irq_mask & ~interrupt_mask);
+
if (WARN_ON(!intel_irqs_enabled(dev_priv)))
return;
- if ((dev_priv->irq_mask & mask) != mask) {
- dev_priv->irq_mask |= mask;
+ new_val = dev_priv->irq_mask;
+ new_val &= ~interrupt_mask;
+ new_val |= (~enabled_irq_mask & interrupt_mask);
+
+ if (new_val != dev_priv->irq_mask) {
+ dev_priv->irq_mask = new_val;
I915_WRITE(DEIMR, dev_priv->irq_mask);
POSTING_READ(DEIMR);
}
}
+void
+ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
+{
+ ilk_update_display_irq(dev_priv, mask, mask);
+}
+
+void
+ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
+{
+ ilk_update_display_irq(dev_priv, mask, 0);
+}
+
/**
* ilk_update_gt_irq - update GTIMR
* @dev_priv: driver private
synchronize_irq(dev->irq);
}
+/**
+ * bdw_update_port_irq - update DE port interrupt
+ * @dev_priv: driver private
+ * @interrupt_mask: mask of interrupt bits to update
+ * @enabled_irq_mask: mask of interrupt bits to enable
+ */
+static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
+ uint32_t interrupt_mask,
+ uint32_t enabled_irq_mask)
+{
+ uint32_t new_val;
+ uint32_t old_val;
+
+ assert_spin_locked(&dev_priv->irq_lock);
+
+ WARN_ON(enabled_irq_mask & ~interrupt_mask);
+
+ if (WARN_ON(!intel_irqs_enabled(dev_priv)))
+ return;
+
+ old_val = I915_READ(GEN8_DE_PORT_IMR);
+
+ new_val = old_val;
+ new_val &= ~interrupt_mask;
+ new_val |= (~enabled_irq_mask & interrupt_mask);
+
+ if (new_val != old_val) {
+ I915_WRITE(GEN8_DE_PORT_IMR, new_val);
+ POSTING_READ(GEN8_DE_PORT_IMR);
+ }
+}
+
/**
* ibx_display_interrupt_update - update SDEIMR
* @dev_priv: driver private
/**
* i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
+ * @dev: drm device
*/
static void i915_enable_asle_pipestat(struct drm_device *dev)
{
* of horizontal active on the first line of vertical active
*/
-static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
+static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
/* Gen2 doesn't have a hardware frame counter */
return 0;
/* Called from drm generic code, passed a 'crtc', which
* we use as a pipe index
*/
-static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
+static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long high_frame;
return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
}
-static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
+static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- int reg = PIPE_FRMCOUNT_GM45(pipe);
- return I915_READ(reg);
+ return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
}
/* raw reads, only for fast reads of display block, no need for forcewake etc. */
else
position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
+ /*
+ * On HSW, the DSL reg (0x70000) appears to return 0 if we
+ * read it just before the start of vblank. So try it again
+ * so we don't accidentally end up spanning a vblank frame
+ * increment, causing the pipe_update_end() code to squak at us.
+ *
+ * The nature of this problem means we can't simply check the ISR
+ * bit and return the vblank start value; nor can we use the scanline
+ * debug register in the transcoder as it appears to have the same
+ * problem. We may need to extend this to include other platforms,
+ * but so far testing only shows the problem on HSW.
+ */
+ if (IS_HASWELL(dev) && !position) {
+ int i, temp;
+
+ for (i = 0; i < 100; i++) {
+ udelay(1);
+ temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
+ DSL_LINEMASK_GEN3;
+ if (temp != position) {
+ position = temp;
+ break;
+ }
+ }
+ }
+
/*
* See update_scanline_offset() for the details on the
* scanline_offset adjustment.
return (position + crtc->scanline_offset) % vtotal;
}
-static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
+static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
unsigned int flags, int *vpos, int *hpos,
- ktime_t *stime, ktime_t *etime)
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
int position;
int vbl_start, vbl_end, hsync_start, htotal, vtotal;
bool in_vbl = true;
return position;
}
-static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
+static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
int *max_error,
struct timeval *vblank_time,
unsigned flags)
{
struct drm_crtc *crtc;
- if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
- DRM_ERROR("Invalid crtc %d\n", pipe);
+ if (pipe >= INTEL_INFO(dev)->num_pipes) {
+ DRM_ERROR("Invalid crtc %u\n", pipe);
return -EINVAL;
}
/* Get drm_crtc to timestamp: */
crtc = intel_get_crtc_for_pipe(dev, pipe);
if (crtc == NULL) {
- DRM_ERROR("Invalid crtc %d\n", pipe);
+ DRM_ERROR("Invalid crtc %u\n", pipe);
return -EINVAL;
}
if (!crtc->hwmode.crtc_clock) {
- DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
+ DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
return -EBUSY;
}
/* Helper routine in DRM core does all the work: */
return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
vblank_time, flags,
- crtc,
&crtc->hwmode);
}
int threshold)
{
u64 time, c0;
+ unsigned int mul = 100;
if (old->cz_clock == 0)
return false;
+ if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
+ mul <<= 8;
+
time = now->cz_clock - old->cz_clock;
- time *= threshold * dev_priv->mem_freq;
+ time *= threshold * dev_priv->czclk_freq;
/* Workload can be split between render + media, e.g. SwapBuffers
* being blitted in X after being rendered in mesa. To account for
*/
c0 = now->render_c0 - old->render_c0;
c0 += now->media_c0 - old->media_c0;
- c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000;
+ c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
return c0 >= time;
}
{
switch (port) {
case PORT_A:
- return val & BXT_PORTA_HOTPLUG_LONG_DETECT;
+ return val & PORTA_HOTPLUG_LONG_DETECT;
+ case PORT_B:
+ return val & PORTB_HOTPLUG_LONG_DETECT;
+ case PORT_C:
+ return val & PORTC_HOTPLUG_LONG_DETECT;
+ default:
+ return false;
+ }
+}
+
+static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
+{
+ switch (port) {
+ case PORT_E:
+ return val & PORTE_HOTPLUG_LONG_DETECT;
+ default:
+ return false;
+ }
+}
+
+static bool spt_port_hotplug_long_detect(enum port port, u32 val)
+{
+ switch (port) {
+ case PORT_A:
+ return val & PORTA_HOTPLUG_LONG_DETECT;
case PORT_B:
return val & PORTB_HOTPLUG_LONG_DETECT;
case PORT_C:
}
}
+static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
+{
+ switch (port) {
+ case PORT_A:
+ return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
+ default:
+ return false;
+ }
+}
+
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
{
switch (port) {
return val & PORTC_HOTPLUG_LONG_DETECT;
case PORT_D:
return val & PORTD_HOTPLUG_LONG_DETECT;
- case PORT_E:
- return val & PORTE_HOTPLUG_LONG_DETECT;
default:
return false;
}
}
}
-/* Get a bit mask of pins that have triggered, and which ones may be long. */
+/*
+ * Get a bit mask of pins that have triggered, and which ones may be long.
+ * This can be called multiple times with the same masks to accumulate
+ * hotplug detection results from several registers.
+ *
+ * Note that the caller is expected to zero out the masks initially.
+ */
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
u32 hotplug_trigger, u32 dig_hotplug_reg,
const u32 hpd[HPD_NUM_PINS],
enum port port;
int i;
- *pin_mask = 0;
- *long_mask = 0;
-
for_each_hpd_pin(i) {
if ((hpd[i] & hotplug_trigger) == 0)
continue;
{
struct drm_i915_private *dev_priv = dev->dev_private;
u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
- u32 pin_mask, long_mask;
+ u32 pin_mask = 0, long_mask = 0;
if (!hotplug_status)
return;
if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
- intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
- hotplug_trigger, hpd_status_g4x,
- i9xx_port_hotplug_long_detect);
- intel_hpd_irq_handler(dev, pin_mask, long_mask);
+ if (hotplug_trigger) {
+ intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
+ hotplug_trigger, hpd_status_g4x,
+ i9xx_port_hotplug_long_detect);
+
+ intel_hpd_irq_handler(dev, pin_mask, long_mask);
+ }
if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
dp_aux_irq_handler(dev);
} else {
u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
- intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
- hotplug_trigger, hpd_status_i915,
- i9xx_port_hotplug_long_detect);
- intel_hpd_irq_handler(dev, pin_mask, long_mask);
+ if (hotplug_trigger) {
+ intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
+ hotplug_trigger, hpd_status_i915,
+ i9xx_port_hotplug_long_detect);
+ intel_hpd_irq_handler(dev, pin_mask, long_mask);
+ }
}
}
return ret;
}
+static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
+ const u32 hpd[HPD_NUM_PINS])
+{
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
+
+ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
+ I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
+
+ intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
+ dig_hotplug_reg, hpd,
+ pch_port_hotplug_long_detect);
+
+ intel_hpd_irq_handler(dev, pin_mask, long_mask);
+}
+
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
{
struct drm_i915_private *dev_priv = dev->dev_private;
int pipe;
u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
- if (hotplug_trigger) {
- u32 dig_hotplug_reg, pin_mask, long_mask;
-
- dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
- I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
-
- intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
- dig_hotplug_reg, hpd_ibx,
- pch_port_hotplug_long_detect);
- intel_hpd_irq_handler(dev, pin_mask, long_mask);
- }
+ if (hotplug_trigger)
+ ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
if (pch_iir & SDE_AUDIO_POWER_MASK) {
int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
{
struct drm_i915_private *dev_priv = dev->dev_private;
int pipe;
- u32 hotplug_trigger;
-
- if (HAS_PCH_SPT(dev))
- hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT;
- else
- hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
-
- if (hotplug_trigger) {
- u32 dig_hotplug_reg, pin_mask, long_mask;
-
- dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
- I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
-
- if (HAS_PCH_SPT(dev)) {
- intel_get_hpd_pins(&pin_mask, &long_mask,
- hotplug_trigger,
- dig_hotplug_reg, hpd_spt,
- pch_port_hotplug_long_detect);
-
- /* detect PORTE HP event */
- dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
- if (pch_port_hotplug_long_detect(PORT_E,
- dig_hotplug_reg))
- long_mask |= 1 << HPD_PORT_E;
- } else
- intel_get_hpd_pins(&pin_mask, &long_mask,
- hotplug_trigger,
- dig_hotplug_reg, hpd_cpt,
- pch_port_hotplug_long_detect);
+ u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
- intel_hpd_irq_handler(dev, pin_mask, long_mask);
- }
+ if (hotplug_trigger)
+ ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
cpt_serr_int_handler(dev);
}
+static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
+ ~SDE_PORTE_HOTPLUG_SPT;
+ u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
+ u32 pin_mask = 0, long_mask = 0;
+
+ if (hotplug_trigger) {
+ u32 dig_hotplug_reg;
+
+ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
+ I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
+
+ intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
+ dig_hotplug_reg, hpd_spt,
+ spt_port_hotplug_long_detect);
+ }
+
+ if (hotplug2_trigger) {
+ u32 dig_hotplug_reg;
+
+ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
+ I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
+
+ intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
+ dig_hotplug_reg, hpd_spt,
+ spt_port_hotplug2_long_detect);
+ }
+
+ if (pin_mask)
+ intel_hpd_irq_handler(dev, pin_mask, long_mask);
+
+ if (pch_iir & SDE_GMBUS_CPT)
+ gmbus_irq_handler(dev);
+}
+
+static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
+ const u32 hpd[HPD_NUM_PINS])
+{
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
+
+ dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
+ I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
+
+ intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
+ dig_hotplug_reg, hpd,
+ ilk_port_hotplug_long_detect);
+
+ intel_hpd_irq_handler(dev, pin_mask, long_mask);
+}
+
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
struct drm_i915_private *dev_priv = dev->dev_private;
enum pipe pipe;
+ u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
+
+ if (hotplug_trigger)
+ ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
if (de_iir & DE_AUX_CHANNEL_A)
dp_aux_irq_handler(dev);
{
struct drm_i915_private *dev_priv = dev->dev_private;
enum pipe pipe;
+ u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
+
+ if (hotplug_trigger)
+ ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
if (de_iir & DE_ERR_INT_IVB)
ivb_err_int_handler(dev);
return ret;
}
-static void bxt_hpd_handler(struct drm_device *dev, uint32_t iir_status)
+static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
+ const u32 hpd[HPD_NUM_PINS])
{
- struct drm_i915_private *dev_priv = dev->dev_private;
- u32 hp_control, hp_trigger;
- u32 pin_mask, long_mask;
-
- /* Get the status */
- hp_trigger = iir_status & BXT_DE_PORT_HOTPLUG_MASK;
- hp_control = I915_READ(BXT_HOTPLUG_CTL);
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
- /* Hotplug not enabled ? */
- if (!(hp_control & BXT_HOTPLUG_CTL_MASK)) {
- DRM_ERROR("Interrupt when HPD disabled\n");
- return;
- }
+ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
+ I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
- /* Clear sticky bits in hpd status */
- I915_WRITE(BXT_HOTPLUG_CTL, hp_control);
+ intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
+ dig_hotplug_reg, hpd,
+ bxt_port_hotplug_long_detect);
- intel_get_hpd_pins(&pin_mask, &long_mask, hp_trigger, hp_control,
- hpd_bxt, bxt_port_hotplug_long_detect);
intel_hpd_irq_handler(dev, pin_mask, long_mask);
}
if (!intel_irqs_enabled(dev_priv))
return IRQ_NONE;
- if (IS_GEN9(dev))
+ if (INTEL_INFO(dev_priv)->gen >= 9)
aux_mask |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
GEN9_AUX_CHANNEL_D;
tmp = I915_READ(GEN8_DE_PORT_IIR);
if (tmp) {
bool found = false;
+ u32 hotplug_trigger = 0;
+
+ if (IS_BROXTON(dev_priv))
+ hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
+ else if (IS_BROADWELL(dev_priv))
+ hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
I915_WRITE(GEN8_DE_PORT_IIR, tmp);
ret = IRQ_HANDLED;
found = true;
}
- if (IS_BROXTON(dev) && tmp & BXT_DE_PORT_HOTPLUG_MASK) {
- bxt_hpd_handler(dev, tmp);
+ if (hotplug_trigger) {
+ if (IS_BROXTON(dev))
+ bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
+ else
+ ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
found = true;
}
intel_pipe_handle_vblank(dev, pipe))
intel_check_page_flip(dev, pipe);
- if (IS_GEN9(dev))
+ if (INTEL_INFO(dev_priv)->gen >= 9)
flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
else
flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
pipe);
- if (IS_GEN9(dev))
+ if (INTEL_INFO(dev_priv)->gen >= 9)
fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
else
fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
if (pch_iir) {
I915_WRITE(SDEIIR, pch_iir);
ret = IRQ_HANDLED;
- cpt_irq_handler(dev, pch_iir);
+
+ if (HAS_PCH_SPT(dev_priv))
+ spt_irq_handler(dev, pch_iir);
+ else
+ cpt_irq_handler(dev, pch_iir);
} else
DRM_ERROR("The master control interrupt lied (SDE)!\n");
/**
* i915_reset_and_wakeup - do process context error handling work
+ * @dev: drm device
*
* Fire an error uevent so userspace can see that a hang or error
* was detected.
* i915_handle_error - handle a gpu error
* @dev: drm device
*
- * Do some basic checking of regsiter state at error time and
+ * Do some basic checking of register state at error time and
* dump it to the syslog. Also call i915_capture_error_state() to make
* sure we get a record and make it available in debugfs. Fire a uevent
* so userspace knows something bad happened (should trigger collection
/* Called from drm generic code, passed 'crtc' which
* we use as a pipe index
*/
-static int i915_enable_vblank(struct drm_device *dev, int pipe)
+static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
return 0;
}
-static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
+static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
return 0;
}
-static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
+static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
return 0;
}
-static int gen8_enable_vblank(struct drm_device *dev, int pipe)
+static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
/* Called from drm generic code, passed 'crtc' which
* we use as a pipe index
*/
-static void i915_disable_vblank(struct drm_device *dev, int pipe)
+static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}
-static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
+static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}
-static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
+static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}
-static void gen8_disable_vblank(struct drm_device *dev, int pipe)
+static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags;
u64 offset = 0;
int i, backwards;
+ /*
+ * This function does not support execlist mode - any attempt to
+ * proceed further into this function will result in a kernel panic
+ * when dereferencing ring->buffer, which is not set up in execlist
+ * mode.
+ *
+ * The correct way of doing it would be to derive the currently
+ * executing ring buffer from the current context, which is derived
+ * from the currently running request. Unfortunately, to get the
+ * current request we would have to grab the struct_mutex before doing
+ * anything else, which would be ill-advised since some other thread
+ * might have grabbed it already and managed to hang itself, causing
+ * the hang checker to deadlock.
+ *
+ * Therefore, this function does not support execlist mode in its
+ * current form. Just return NULL and move on.
+ */
+ if (ring->buffer == NULL)
+ return NULL;
+
ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
return NULL;
{
enum pipe pipe;
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
for_each_pipe(dev_priv, pipe)
vlv_display_irq_reset(dev_priv);
}
+static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
+ const u32 hpd[HPD_NUM_PINS])
+{
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct intel_encoder *encoder;
+ u32 enabled_irqs = 0;
+
+ for_each_intel_encoder(dev, encoder)
+ if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
+ enabled_irqs |= hpd[encoder->hpd_pin];
+
+ return enabled_irqs;
+}
+
static void ibx_hpd_irq_setup(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_encoder *intel_encoder;
- u32 hotplug_irqs, hotplug, enabled_irqs = 0;
+ u32 hotplug_irqs, hotplug, enabled_irqs;
if (HAS_PCH_IBX(dev)) {
hotplug_irqs = SDE_HOTPLUG_MASK;
- for_each_intel_encoder(dev, intel_encoder)
- if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
- enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
- } else if (HAS_PCH_SPT(dev)) {
- hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
- for_each_intel_encoder(dev, intel_encoder)
- if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
- enabled_irqs |= hpd_spt[intel_encoder->hpd_pin];
+ enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
} else {
hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
- for_each_intel_encoder(dev, intel_encoder)
- if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
- enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
+ enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
}
ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
/*
* Enable digital hotplug on the PCH, and configure the DP short pulse
- * duration to 2ms (which is the minimum in the Display Port spec)
- *
- * This register is the same on all known PCH chips.
+ * duration to 2ms (which is the minimum in the Display Port spec).
+ * The pulse duration bits are reserved on LPT+.
*/
hotplug = I915_READ(PCH_PORT_HOTPLUG);
hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
+ /*
+ * When CPU and PCH are on the same package, port A
+ * HPD must be enabled in both north and south.
+ */
+ if (HAS_PCH_LPT_LP(dev))
+ hotplug |= PORTA_HOTPLUG_ENABLE;
I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
+}
- /* enable SPT PORTE hot plug */
- if (HAS_PCH_SPT(dev)) {
- hotplug = I915_READ(PCH_PORT_HOTPLUG2);
- hotplug |= PORTE_HOTPLUG_ENABLE;
- I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
- }
+static void spt_hpd_irq_setup(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u32 hotplug_irqs, hotplug, enabled_irqs;
+
+ hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
+ enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
+
+ ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
+
+ /* Enable digital hotplug on the PCH */
+ hotplug = I915_READ(PCH_PORT_HOTPLUG);
+ hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
+ PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
+ I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
+
+ hotplug = I915_READ(PCH_PORT_HOTPLUG2);
+ hotplug |= PORTE_HOTPLUG_ENABLE;
+ I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
}
-static void bxt_hpd_irq_setup(struct drm_device *dev)
+static void ilk_hpd_irq_setup(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_encoder *intel_encoder;
- u32 hotplug_port = 0;
- u32 hotplug_ctrl;
-
- /* Now, enable HPD */
- for_each_intel_encoder(dev, intel_encoder) {
- if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state
- == HPD_ENABLED)
- hotplug_port |= hpd_bxt[intel_encoder->hpd_pin];
+ u32 hotplug_irqs, hotplug, enabled_irqs;
+
+ if (INTEL_INFO(dev)->gen >= 8) {
+ hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
+ enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
+
+ bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
+ } else if (INTEL_INFO(dev)->gen >= 7) {
+ hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
+ enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
+
+ ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
+ } else {
+ hotplug_irqs = DE_DP_A_HOTPLUG;
+ enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
+
+ ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
}
- /* Mask all HPD control bits */
- hotplug_ctrl = I915_READ(BXT_HOTPLUG_CTL) & ~BXT_HOTPLUG_CTL_MASK;
+ /*
+ * Enable digital hotplug on the CPU, and configure the DP short pulse
+ * duration to 2ms (which is the minimum in the Display Port spec)
+ * The pulse duration bits are reserved on HSW+.
+ */
+ hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
+ hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
+ hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
+ I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
- /* Enable requested port in hotplug control */
- /* TODO: implement (short) HPD support on port A */
- WARN_ON_ONCE(hotplug_port & BXT_DE_PORT_HP_DDIA);
- if (hotplug_port & BXT_DE_PORT_HP_DDIB)
- hotplug_ctrl |= BXT_DDIB_HPD_ENABLE;
- if (hotplug_port & BXT_DE_PORT_HP_DDIC)
- hotplug_ctrl |= BXT_DDIC_HPD_ENABLE;
- I915_WRITE(BXT_HOTPLUG_CTL, hotplug_ctrl);
+ ibx_hpd_irq_setup(dev);
+}
- /* Unmask DDI hotplug in IMR */
- hotplug_ctrl = I915_READ(GEN8_DE_PORT_IMR) & ~hotplug_port;
- I915_WRITE(GEN8_DE_PORT_IMR, hotplug_ctrl);
+static void bxt_hpd_irq_setup(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u32 hotplug_irqs, hotplug, enabled_irqs;
+
+ enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
+ hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
+
+ bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
- /* Enable DDI hotplug in IER */
- hotplug_ctrl = I915_READ(GEN8_DE_PORT_IER) | hotplug_port;
- I915_WRITE(GEN8_DE_PORT_IER, hotplug_ctrl);
- POSTING_READ(GEN8_DE_PORT_IER);
+ hotplug = I915_READ(PCH_PORT_HOTPLUG);
+ hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
+ PORTA_HOTPLUG_ENABLE;
+ I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
}
static void ibx_irq_postinstall(struct drm_device *dev)
else
mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
- GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
+ gen5_assert_iir_is_zero(dev_priv, SDEIIR);
I915_WRITE(SDEIMR, ~mask);
}
DE_PLANEB_FLIP_DONE_IVB |
DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
- DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
+ DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
+ DE_DP_A_HOTPLUG_IVB);
} else {
display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
DE_AUX_CHANNEL_A |
DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
DE_POISON);
- extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
- DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
+ extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
+ DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
+ DE_DP_A_HOTPLUG);
}
dev_priv->irq_mask = ~display_mask;
{
dev_priv->irq_mask = ~0;
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
POSTING_READ(PORT_HOTPLUG_EN);
I915_WRITE(VLV_IIR, 0xffffffff);
{
uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
uint32_t de_pipe_enables;
- int pipe;
- u32 de_port_en = GEN8_AUX_CHANNEL_A;
+ u32 de_port_masked = GEN8_AUX_CHANNEL_A;
+ u32 de_port_enables;
+ enum pipe pipe;
- if (IS_GEN9(dev_priv)) {
+ if (INTEL_INFO(dev_priv)->gen >= 9) {
de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
- de_port_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
- GEN9_AUX_CHANNEL_D;
-
+ de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
+ GEN9_AUX_CHANNEL_D;
if (IS_BROXTON(dev_priv))
- de_port_en |= BXT_DE_PORT_GMBUS;
- } else
+ de_port_masked |= BXT_DE_PORT_GMBUS;
+ } else {
de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
+ }
de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
GEN8_PIPE_FIFO_UNDERRUN;
+ de_port_enables = de_port_masked;
+ if (IS_BROXTON(dev_priv))
+ de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
+ else if (IS_BROADWELL(dev_priv))
+ de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
+
dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
dev_priv->de_irq_mask[pipe],
de_pipe_enables);
- GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_en, de_port_en);
+ GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
}
static int gen8_irq_postinstall(struct drm_device *dev)
int pipe;
if (I915_HAS_HOTPLUG(dev)) {
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
}
I915_USER_INTERRUPT;
if (I915_HAS_HOTPLUG(dev)) {
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
POSTING_READ(PORT_HOTPLUG_EN);
/* Enable in IER... */
int pipe;
if (I915_HAS_HOTPLUG(dev)) {
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
}
struct drm_i915_private *dev_priv = dev->dev_private;
int pipe;
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
I915_WRITE(HWSTAM, 0xeffe);
I915_WRITE(IER, enable_mask);
POSTING_READ(IER);
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
POSTING_READ(PORT_HOTPLUG_EN);
i915_enable_asle_pipestat(dev);
static void i915_hpd_irq_setup(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_encoder *intel_encoder;
u32 hotplug_en;
assert_spin_locked(&dev_priv->irq_lock);
- hotplug_en = I915_READ(PORT_HOTPLUG_EN);
- hotplug_en &= ~HOTPLUG_INT_EN_MASK;
/* Note HDMI and DP share hotplug bits */
/* enable bits are the same for all generations */
- for_each_intel_encoder(dev, intel_encoder)
- if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
- hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
+ hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
/* Programming the CRT detection parameters tends
to generate a spurious hotplug event about three
seconds later. So just do it once.
*/
if (IS_G4X(dev))
hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
- hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
/* Ignore TV since it's buggy */
- I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
+ i915_hotplug_interrupt_update_locked(dev_priv,
+ (HOTPLUG_INT_EN_MASK
+ | CRT_HOTPLUG_VOLTAGE_COMPARE_MASK),
+ hotplug_en);
}
static irqreturn_t i965_irq_handler(int irq, void *arg)
if (!dev_priv)
return;
- I915_WRITE(PORT_HOTPLUG_EN, 0);
+ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
I915_WRITE(HWSTAM, 0xffffffff);
dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
- dev->driver->get_vblank_counter = gm45_get_vblank_counter;
+ dev->driver->get_vblank_counter = g4x_get_vblank_counter;
} else {
dev->driver->get_vblank_counter = i915_get_vblank_counter;
dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
dev->driver->irq_uninstall = gen8_irq_uninstall;
dev->driver->enable_vblank = gen8_enable_vblank;
dev->driver->disable_vblank = gen8_disable_vblank;
- if (HAS_PCH_SPLIT(dev))
- dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
- else
+ if (IS_BROXTON(dev))
dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
+ else if (HAS_PCH_SPT(dev))
+ dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
+ else
+ dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
} else if (HAS_PCH_SPLIT(dev)) {
dev->driver->irq_handler = ironlake_irq_handler;
dev->driver->irq_preinstall = ironlake_irq_reset;
dev->driver->irq_uninstall = ironlake_irq_uninstall;
dev->driver->enable_vblank = ironlake_enable_vblank;
dev->driver->disable_vblank = ironlake_disable_vblank;
- dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
+ dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
} else {
if (INTEL_INFO(dev_priv)->gen == 2) {
dev->driver->irq_preinstall = i8xx_irq_preinstall;
.preliminary_hw_support = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT),
.disable_power_well = 1,
.enable_ips = 1,
- .fastboot = 0,
.prefault_disable = 0,
.load_detect_test = 0,
.reset = true,
.use_mmio_flip = 0,
.mmio_debug = 0,
.verbose_state_checks = 1,
+ .nuclear_pageflip = 0,
.edp_vswing = 0,
.enable_guc_submission = false,
.guc_log_level = -1,
"Use kernel modesetting [KMS] (0=disable, "
"1=on, -1=force vga console preference [default])");
-module_param_named(panel_ignore_lid, i915.panel_ignore_lid, int, 0600);
+module_param_named_unsafe(panel_ignore_lid, i915.panel_ignore_lid, int, 0600);
MODULE_PARM_DESC(panel_ignore_lid,
"Override lid status (0=autodetect, 1=autodetect disabled [default], "
"-1=force lid closed, -2=force lid open)");
"Enable frame buffer compression for power savings "
"(default: -1 (use per-chip default))");
-module_param_named(lvds_channel_mode, i915.lvds_channel_mode, int, 0600);
+module_param_named_unsafe(lvds_channel_mode, i915.lvds_channel_mode, int, 0600);
MODULE_PARM_DESC(lvds_channel_mode,
"Specify LVDS channel mode "
"(0=probe BIOS [default], 1=single-channel, 2=dual-channel)");
-module_param_named(lvds_use_ssc, i915.panel_use_ssc, int, 0600);
+module_param_named_unsafe(lvds_use_ssc, i915.panel_use_ssc, int, 0600);
MODULE_PARM_DESC(lvds_use_ssc,
"Use Spread Spectrum Clock with panels [LVDS/eDP] "
"(default: auto from VBT)");
-module_param_named(vbt_sdvo_panel_type, i915.vbt_sdvo_panel_type, int, 0600);
+module_param_named_unsafe(vbt_sdvo_panel_type, i915.vbt_sdvo_panel_type, int, 0600);
MODULE_PARM_DESC(vbt_sdvo_panel_type,
"Override/Ignore selection of SDVO panel mode in the VBT "
"(-2=ignore, -1=auto [default], index in VBT BIOS table)");
module_param_named_unsafe(reset, i915.reset, bool, 0600);
MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
-module_param_named(enable_hangcheck, i915.enable_hangcheck, bool, 0644);
+module_param_named_unsafe(enable_hangcheck, i915.enable_hangcheck, bool, 0644);
MODULE_PARM_DESC(enable_hangcheck,
"Periodically check GPU activity for detecting hangs. "
"WARNING: Disabling this can cause system wide hangs. "
"Override PPGTT usage. "
"(-1=auto [default], 0=disabled, 1=aliasing, 2=full)");
-module_param_named(enable_execlists, i915.enable_execlists, int, 0400);
+module_param_named_unsafe(enable_execlists, i915.enable_execlists, int, 0400);
MODULE_PARM_DESC(enable_execlists,
"Override execlists usage. "
"(-1=auto [default], 0=disabled, 1=enabled)");
-module_param_named(enable_psr, i915.enable_psr, int, 0600);
+module_param_named_unsafe(enable_psr, i915.enable_psr, int, 0600);
MODULE_PARM_DESC(enable_psr, "Enable PSR (default: false)");
-module_param_named(preliminary_hw_support, i915.preliminary_hw_support, int, 0600);
+module_param_named_unsafe(preliminary_hw_support, i915.preliminary_hw_support, int, 0600);
MODULE_PARM_DESC(preliminary_hw_support,
"Enable preliminary hardware support.");
-module_param_named(disable_power_well, i915.disable_power_well, int, 0600);
+module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0600);
MODULE_PARM_DESC(disable_power_well,
"Disable the power well when possible (default: true)");
-module_param_named(enable_ips, i915.enable_ips, int, 0600);
+module_param_named_unsafe(enable_ips, i915.enable_ips, int, 0600);
MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)");
-module_param_named(fastboot, i915.fastboot, bool, 0600);
-MODULE_PARM_DESC(fastboot,
- "Try to skip unnecessary mode sets at boot time (default: false)");
-
module_param_named_unsafe(prefault_disable, i915.prefault_disable, bool, 0600);
MODULE_PARM_DESC(prefault_disable,
"Disable page prefaulting for pread/pwrite/reloc (default:false). "
"Force-enable the VGA load detect code for testing (default:false). "
"For developers only.");
-module_param_named(invert_brightness, i915.invert_brightness, int, 0600);
+module_param_named_unsafe(invert_brightness, i915.invert_brightness, int, 0600);
MODULE_PARM_DESC(invert_brightness,
"Invert backlight brightness "
"(-1 force normal, 0 machine defaults, 1 force inversion), please "
module_param_named(disable_display, i915.disable_display, bool, 0600);
MODULE_PARM_DESC(disable_display, "Disable display (default: false)");
-module_param_named(disable_vtd_wa, i915.disable_vtd_wa, bool, 0600);
+module_param_named_unsafe(disable_vtd_wa, i915.disable_vtd_wa, bool, 0600);
MODULE_PARM_DESC(disable_vtd_wa, "Disable all VT-d workarounds (default: false)");
-module_param_named(enable_cmd_parser, i915.enable_cmd_parser, int, 0600);
+module_param_named_unsafe(enable_cmd_parser, i915.enable_cmd_parser, int, 0600);
MODULE_PARM_DESC(enable_cmd_parser,
"Enable command parsing (1=enabled [default], 0=disabled)");
-module_param_named(use_mmio_flip, i915.use_mmio_flip, int, 0600);
+module_param_named_unsafe(use_mmio_flip, i915.use_mmio_flip, int, 0600);
MODULE_PARM_DESC(use_mmio_flip,
"use MMIO flips (-1=never, 0=driver discretion [default], 1=always)");
MODULE_PARM_DESC(verbose_state_checks,
"Enable verbose logs (ie. WARN_ON()) in case of unexpected hw state conditions.");
+module_param_named_unsafe(nuclear_pageflip, i915.nuclear_pageflip, bool, 0600);
+MODULE_PARM_DESC(nuclear_pageflip,
+ "Force atomic modeset functionality; asynchronous mode is not yet supported. (default: false).");
+
/* WA to get away with the default setting in VBT for early platforms.Will be removed */
module_param_named_unsafe(edp_vswing, i915.edp_vswing, int, 0400);
MODULE_PARM_DESC(edp_vswing,
#define GRDOM_RESET_STATUS (1<<1)
#define GRDOM_RESET_ENABLE (1<<0)
-#define ILK_GDSR 0x2ca4 /* MCHBAR offset */
+#define ILK_GDSR (MCHBAR_MIRROR_BASE + 0x2ca4)
#define ILK_GRDOM_FULL (0<<1)
#define ILK_GRDOM_RENDER (1<<1)
#define ILK_GRDOM_MEDIA (3<<1)
*/
#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*(x)-1)
#define MI_LRI_FORCE_POSTED (1<<12)
-#define MI_STORE_REGISTER_MEM(x) MI_INSTR(0x24, 2*(x)-1)
-#define MI_STORE_REGISTER_MEM_GEN8(x) MI_INSTR(0x24, 3*(x)-1)
+#define MI_STORE_REGISTER_MEM MI_INSTR(0x24, 1)
+#define MI_STORE_REGISTER_MEM_GEN8 MI_INSTR(0x24, 2)
#define MI_SRM_LRM_GLOBAL_GTT (1<<22)
#define MI_FLUSH_DW MI_INSTR(0x26, 1) /* for GEN6 */
#define MI_FLUSH_DW_STORE_INDEX (1<<21)
#define MI_INVALIDATE_BSD (1<<7)
#define MI_FLUSH_DW_USE_GTT (1<<2)
#define MI_FLUSH_DW_USE_PPGTT (0<<2)
-#define MI_LOAD_REGISTER_MEM(x) MI_INSTR(0x29, 2*(x)-1)
-#define MI_LOAD_REGISTER_MEM_GEN8(x) MI_INSTR(0x29, 3*(x)-1)
+#define MI_LOAD_REGISTER_MEM MI_INSTR(0x29, 1)
+#define MI_LOAD_REGISTER_MEM_GEN8 MI_INSTR(0x29, 2)
#define MI_BATCH_BUFFER MI_INSTR(0x30, 1)
#define MI_BATCH_NON_SECURE (1)
/* for snb/ivb/vlv this also means "batch in ppgtt" when ppgtt is enabled. */
#define ASYNC_FLIP (1<<22)
#define DISPLAY_PLANE_A (0<<20)
#define DISPLAY_PLANE_B (1<<20)
-#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|(len-2))
+#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2))
#define PIPE_CONTROL_FLUSH_L3 (1<<27)
#define PIPE_CONTROL_GLOBAL_GTT_IVB (1<<24) /* gen7+ */
#define PIPE_CONTROL_MMIO_WRITE (1<<23)
#define GEN7_3DPRIM_START_INSTANCE 0x243C
#define GEN7_3DPRIM_BASE_VERTEX 0x2440
+#define GEN7_GPGPU_DISPATCHDIMX 0x2500
+#define GEN7_GPGPU_DISPATCHDIMY 0x2504
+#define GEN7_GPGPU_DISPATCHDIMZ 0x2508
+
#define OACONTROL 0x2360
#define _GEN7_PIPEA_DE_LOAD_SL 0x70068
#define DSI_PLL_N1_DIV_MASK (3 << 16)
#define DSI_PLL_M1_DIV_SHIFT 0
#define DSI_PLL_M1_DIV_MASK (0x1ff << 0)
+#define CCK_CZ_CLOCK_CONTROL 0x62
#define CCK_DISPLAY_CLOCK_CONTROL 0x6b
-#define DISPLAY_TRUNK_FORCE_ON (1 << 17)
-#define DISPLAY_TRUNK_FORCE_OFF (1 << 16)
-#define DISPLAY_FREQUENCY_STATUS (0x1f << 8)
-#define DISPLAY_FREQUENCY_STATUS_SHIFT 8
-#define DISPLAY_FREQUENCY_VALUES (0x1f << 0)
+#define CCK_TRUNK_FORCE_ON (1 << 17)
+#define CCK_TRUNK_FORCE_OFF (1 << 16)
+#define CCK_FREQUENCY_STATUS (0x1f << 8)
+#define CCK_FREQUENCY_STATUS_SHIFT 8
+#define CCK_FREQUENCY_VALUES (0x1f << 0)
/**
* DOC: DPIO
#define DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE 1 /* 1: coarse & 0 : fine */
#define CHV_PLL_DW9(ch) _PIPE(ch, _CHV_PLL_DW9_CH0, _CHV_PLL_DW9_CH1)
+#define _CHV_CMN_DW0_CH0 0x8100
+#define DPIO_ALLDL_POWERDOWN_SHIFT_CH0 19
+#define DPIO_ANYDL_POWERDOWN_SHIFT_CH0 18
+#define DPIO_ALLDL_POWERDOWN (1 << 1)
+#define DPIO_ANYDL_POWERDOWN (1 << 0)
+
#define _CHV_CMN_DW5_CH0 0x8114
#define CHV_BUFRIGHTENA1_DISABLE (0 << 20)
#define CHV_BUFRIGHTENA1_NORMAL (1 << 20)
#define _CHV_CMN_DW19_CH0 0x814c
#define _CHV_CMN_DW6_CH1 0x8098
+#define DPIO_ALLDL_POWERDOWN_SHIFT_CH1 30 /* CL2 DW6 only */
+#define DPIO_ANYDL_POWERDOWN_SHIFT_CH1 29 /* CL2 DW6 only */
+#define DPIO_DYNPWRDOWNEN_CH1 (1 << 28) /* CL2 DW6 only */
#define CHV_CMN_USEDCLKCHANNEL (1 << 13)
+
#define CHV_CMN_DW19(ch) _PIPE(ch, _CHV_CMN_DW19_CH0, _CHV_CMN_DW6_CH1)
+#define CHV_CMN_DW28 0x8170
+#define DPIO_CL1POWERDOWNEN (1 << 23)
+#define DPIO_DYNPWRDOWNEN_CH0 (1 << 22)
+#define DPIO_SUS_CLK_CONFIG_ON (0 << 0)
+#define DPIO_SUS_CLK_CONFIG_CLKREQ (1 << 0)
+#define DPIO_SUS_CLK_CONFIG_GATE (2 << 0)
+#define DPIO_SUS_CLK_CONFIG_GATE_CLKREQ (3 << 0)
+
#define CHV_CMN_DW30 0x8178
+#define DPIO_CL2_LDOFUSE_PWRENB (1 << 6)
#define DPIO_LRC_BYPASS (1 << 3)
#define _TXLANE(ch, lane, offset) ((ch ? 0x2400 : 0) + \
#define PORT_PLL_DCO_AMP_OVR_EN_H (1<<27)
#define PORT_PLL_DCO_AMP_DEFAULT 15
#define PORT_PLL_DCO_AMP_MASK 0x3c00
-#define PORT_PLL_DCO_AMP(x) (x<<10)
+#define PORT_PLL_DCO_AMP(x) ((x)<<10)
#define _PORT_PLL_BASE(port) _PORT3(port, _PORT_PLL_0_A, \
_PORT_PLL_0_B, \
_PORT_PLL_0_C)
#define BXT_PORT_TX_DW3_LN0(port) _PORT3(port, _PORT_TX_DW3_LN0_A, \
_PORT_TX_DW3_LN0_B, \
_PORT_TX_DW3_LN0_C)
-#define UNIQE_TRANGE_EN_METHOD (1 << 27)
+#define SCALE_DCOMP_METHOD (1 << 26)
+#define UNIQUE_TRANGE_EN_METHOD (1 << 27)
#define _PORT_TX_DW4_LN0_A 0x162510
#define _PORT_TX_DW4_LN0_B 0x6C510
/*
* Fence registers
+ * [0-7] @ 0x2000 gen2,gen3
+ * [8-15] @ 0x3000 945,g33,pnv
+ *
+ * [0-15] @ 0x3000 gen4,gen5
+ *
+ * [0-15] @ 0x100000 gen6,vlv,chv
+ * [0-31] @ 0x100000 gen7+
*/
-#define FENCE_REG_830_0 0x2000
-#define FENCE_REG_945_8 0x3000
+#define FENCE_REG(i) (0x2000 + (((i) & 8) << 9) + ((i) & 7) * 4)
#define I830_FENCE_START_MASK 0x07f80000
#define I830_FENCE_TILING_Y_SHIFT 12
#define I830_FENCE_SIZE_BITS(size) ((ffs((size) >> 19) - 1) << 8)
#define I915_FENCE_START_MASK 0x0ff00000
#define I915_FENCE_SIZE_BITS(size) ((ffs((size) >> 20) - 1) << 8)
-#define FENCE_REG_965_0 0x03000
+#define FENCE_REG_965_LO(i) (0x03000 + (i) * 8)
+#define FENCE_REG_965_HI(i) (0x03000 + (i) * 8 + 4)
#define I965_FENCE_PITCH_SHIFT 2
#define I965_FENCE_TILING_Y_SHIFT 1
#define I965_FENCE_REG_VALID (1<<0)
#define I965_FENCE_MAX_PITCH_VAL 0x0400
-#define FENCE_REG_SANDYBRIDGE_0 0x100000
-#define SANDYBRIDGE_FENCE_PITCH_SHIFT 32
+#define FENCE_REG_GEN6_LO(i) (0x100000 + (i) * 8)
+#define FENCE_REG_GEN6_HI(i) (0x100000 + (i) * 8 + 4)
+#define GEN6_FENCE_PITCH_SHIFT 32
#define GEN7_FENCE_MAX_PITCH_VAL 0x0800
#define GEN7_GFX_PEND_TLB0 0x4034
#define GEN7_GFX_PEND_TLB1 0x4038
/* L3, CVS, ZTLB, RCC, CASC LRA min, max values */
-#define GEN7_LRA_LIMITS_BASE 0x403C
+#define GEN7_LRA_LIMITS(i) (0x403C + (i) * 4)
#define GEN7_LRA_LIMITS_REG_NUM 13
#define GEN7_MEDIA_MAX_REQ_COUNT 0x4070
#define GEN7_GFX_MAX_REQ_COUNT 0x4074
#define RENDER_HWS_PGA_GEN7 (0x04080)
#define RING_FAULT_REG(ring) (0x4094 + 0x100*(ring)->id)
#define RING_FAULT_GTTSEL_MASK (1<<11)
-#define RING_FAULT_SRCID(x) ((x >> 3) & 0xff)
-#define RING_FAULT_FAULT_TYPE(x) ((x >> 1) & 0x3)
+#define RING_FAULT_SRCID(x) (((x) >> 3) & 0xff)
+#define RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3)
#define RING_FAULT_VALID (1<<0)
#define DONE_REG 0x40b0
-#define GEN8_PRIVATE_PAT 0x40e0
+#define GEN8_PRIVATE_PAT_LO 0x40e0
+#define GEN8_PRIVATE_PAT_HI (0x40e0 + 4)
#define BSD_HWS_PGA_GEN7 (0x04180)
#define BLT_HWS_PGA_GEN7 (0x04280)
#define VEBOX_HWS_PGA_GEN7 (0x04380)
#endif
#define IPEIR_I965 0x02064
#define IPEHR_I965 0x02068
-#define INSTDONE_I965 0x0206c
-#define GEN7_INSTDONE_1 0x0206c
#define GEN7_SC_INSTDONE 0x07100
#define GEN7_SAMPLER_INSTDONE 0x0e160
#define GEN7_ROW_INSTDONE 0x0e164
#define I915_NUM_INSTDONE_REG 4
#define RING_IPEIR(base) ((base)+0x64)
#define RING_IPEHR(base) ((base)+0x68)
+/*
+ * On GEN4, only the render ring INSTDONE exists and has a different
+ * layout than the GEN7+ version.
+ * The GEN2 counterpart of this register is GEN2_INSTDONE.
+ */
#define RING_INSTDONE(base) ((base)+0x6c)
#define RING_INSTPS(base) ((base)+0x70)
#define RING_DMA_FADD(base) ((base)+0x78)
#define RING_INSTPM(base) ((base)+0xc0)
#define RING_MI_MODE(base) ((base)+0x9c)
#define INSTPS 0x02070 /* 965+ only */
-#define INSTDONE1 0x0207c /* 965+ only */
+#define GEN4_INSTDONE1 0x0207c /* 965+ only, aka INSTDONE_2 on SNB */
#define ACTHD_I965 0x02074
#define HWS_PGA 0x02080
#define HWS_ADDRESS_MASK 0xfffff000
#define PWRCTX_EN (1<<0)
#define IPEIR 0x02088
#define IPEHR 0x0208c
-#define INSTDONE 0x02090
+#define GEN2_INSTDONE 0x02090
#define NOPID 0x02094
#define HWSTAM 0x02098
#define DMA_FADD_I8XX 0x020d0
#define ERR_INT_PIPE_CRC_DONE_B (1<<5)
#define ERR_INT_FIFO_UNDERRUN_B (1<<3)
#define ERR_INT_PIPE_CRC_DONE_A (1<<2)
-#define ERR_INT_PIPE_CRC_DONE(pipe) (1<<(2 + pipe*3))
+#define ERR_INT_PIPE_CRC_DONE(pipe) (1<<(2 + (pipe)*3))
#define ERR_INT_FIFO_UNDERRUN_A (1<<0)
-#define ERR_INT_FIFO_UNDERRUN(pipe) (1<<(pipe*3))
+#define ERR_INT_FIFO_UNDERRUN(pipe) (1<<((pipe)*3))
#define GEN8_FAULT_TLB_DATA0 0x04b10
#define GEN8_FAULT_TLB_DATA1 0x04b14
#define GEN6_WIZ_HASHING_16x4 GEN6_WIZ_HASHING(1, 0)
#define GEN6_WIZ_HASHING_MASK GEN6_WIZ_HASHING(1, 1)
#define GEN6_TD_FOUR_ROW_DISPATCH_DISABLE (1 << 5)
-#define GEN9_IZ_HASHING_MASK(slice) (0x3 << (slice * 2))
-#define GEN9_IZ_HASHING(slice, val) ((val) << (slice * 2))
+#define GEN9_IZ_HASHING_MASK(slice) (0x3 << ((slice) * 2))
+#define GEN9_IZ_HASHING(slice, val) ((val) << ((slice) * 2))
#define GFX_MODE 0x02520
#define GFX_MODE_GEN7 0x0229c
#define RING_MODE_GEN7(ring) ((ring)->mmio_base+0x29c)
#define GFX_RUN_LIST_ENABLE (1<<15)
+#define GFX_INTERRUPT_STEERING (1<<14)
#define GFX_TLB_INVALIDATE_EXPLICIT (1<<13)
#define GFX_SURFACE_FAULT_ENABLE (1<<12)
#define GFX_REPLAY_MODE (1<<11)
#define GFX_PSMI_GRANULARITY (1<<10)
#define GFX_PPGTT_ENABLE (1<<9)
+#define GEN8_GFX_PPGTT_48B (1<<7)
+
+#define GFX_FORWARD_VBLANK_MASK (3<<5)
+#define GFX_FORWARD_VBLANK_NEVER (0<<5)
+#define GFX_FORWARD_VBLANK_ALWAYS (1<<5)
+#define GFX_FORWARD_VBLANK_COND (2<<5)
#define VLV_DISPLAY_BASE 0x180000
#define VLV_MIPI_BASE VLV_DISPLAY_BASE
#define CHV_FGT_EU_DIS_SS1_R1_MASK (0xf << CHV_FGT_EU_DIS_SS1_R1_SHIFT)
#define GEN8_FUSE2 0x9120
+#define GEN8_F2_SS_DIS_SHIFT 21
+#define GEN8_F2_SS_DIS_MASK (0x7 << GEN8_F2_SS_DIS_SHIFT)
#define GEN8_F2_S_ENA_SHIFT 25
#define GEN8_F2_S_ENA_MASK (0x7 << GEN8_F2_S_ENA_SHIFT)
#define GEN9_F2_SS_DIS_SHIFT 20
#define GEN9_F2_SS_DIS_MASK (0xf << GEN9_F2_SS_DIS_SHIFT)
+#define GEN8_EU_DISABLE0 0x9134
+#define GEN8_EU_DIS0_S0_MASK 0xffffff
+#define GEN8_EU_DIS0_S1_SHIFT 24
+#define GEN8_EU_DIS0_S1_MASK (0xff << GEN8_EU_DIS0_S1_SHIFT)
+
+#define GEN8_EU_DISABLE1 0x9138
+#define GEN8_EU_DIS1_S1_MASK 0xffff
+#define GEN8_EU_DIS1_S2_SHIFT 16
+#define GEN8_EU_DIS1_S2_MASK (0xffff << GEN8_EU_DIS1_S2_SHIFT)
+
+#define GEN8_EU_DISABLE2 0x913c
+#define GEN8_EU_DIS2_S2_MASK 0xff
+
#define GEN9_EU_DISABLE(slice) (0x9134 + (slice)*0x4)
#define GEN6_BSD_SLEEP_PSMI_CONTROL 0x12050
#define FBC_CTL_CPU_FENCE (1<<1)
#define FBC_CTL_PLANE(plane) ((plane)<<0)
#define FBC_FENCE_OFF 0x03218 /* BSpec typo has 321Bh */
-#define FBC_TAG 0x03300
+#define FBC_TAG(i) (0x03300 + (i) * 4)
#define FBC_STATUS2 0x43214
#define FBC_COMPRESSION_MASK 0x7ff
# define GPIO_DATA_VAL_IN (1 << 12)
# define GPIO_DATA_PULLUP_DISABLE (1 << 13)
-#define GMBUS0 0x5100 /* clock/port select */
+#define GMBUS0 (dev_priv->gpio_mmio_base + 0x5100) /* clock/port select */
#define GMBUS_RATE_100KHZ (0<<8)
#define GMBUS_RATE_50KHZ (1<<8)
#define GMBUS_RATE_400KHZ (2<<8) /* reserved on Pineview */
#define GMBUS_PIN_2_BXT 2
#define GMBUS_PIN_3_BXT 3
#define GMBUS_NUM_PINS 7 /* including 0 */
-#define GMBUS1 0x5104 /* command/status */
+#define GMBUS1 (dev_priv->gpio_mmio_base + 0x5104) /* command/status */
#define GMBUS_SW_CLR_INT (1<<31)
#define GMBUS_SW_RDY (1<<30)
#define GMBUS_ENT (1<<29) /* enable timeout */
#define GMBUS_SLAVE_ADDR_SHIFT 1
#define GMBUS_SLAVE_READ (1<<0)
#define GMBUS_SLAVE_WRITE (0<<0)
-#define GMBUS2 0x5108 /* status */
+#define GMBUS2 (dev_priv->gpio_mmio_base + 0x5108) /* status */
#define GMBUS_INUSE (1<<15)
#define GMBUS_HW_WAIT_PHASE (1<<14)
#define GMBUS_STALL_TIMEOUT (1<<13)
#define GMBUS_HW_RDY (1<<11)
#define GMBUS_SATOER (1<<10)
#define GMBUS_ACTIVE (1<<9)
-#define GMBUS3 0x510c /* data buffer bytes 3-0 */
-#define GMBUS4 0x5110 /* interrupt mask (Pineview+) */
+#define GMBUS3 (dev_priv->gpio_mmio_base + 0x510c) /* data buffer bytes 3-0 */
+#define GMBUS4 (dev_priv->gpio_mmio_base + 0x5110) /* interrupt mask (Pineview+) */
#define GMBUS_SLAVE_TIMEOUT_EN (1<<4)
#define GMBUS_NAK_EN (1<<3)
#define GMBUS_IDLE_EN (1<<2)
#define GMBUS_HW_WAIT_EN (1<<1)
#define GMBUS_HW_RDY_EN (1<<0)
-#define GMBUS5 0x5120 /* byte index */
+#define GMBUS5 (dev_priv->gpio_mmio_base + 0x5120) /* byte index */
#define GMBUS_2BYTE_INDEX_EN (1<<31)
/*
#define DPIO_PHY_STATUS (VLV_DISPLAY_BASE + 0x6240)
#define DPLL_PORTD_READY_MASK (0xf)
#define DISPLAY_PHY_CONTROL (VLV_DISPLAY_BASE + 0x60100)
+#define PHY_CH_POWER_DOWN_OVRD_EN(phy, ch) (1 << (2*(phy)+(ch)+27))
#define PHY_LDO_DELAY_0NS 0x0
#define PHY_LDO_DELAY_200NS 0x1
#define PHY_LDO_DELAY_600NS 0x2
#define PHY_LDO_SEQ_DELAY(delay, phy) ((delay) << (2*(phy)+23))
+#define PHY_CH_POWER_DOWN_OVRD(mask, phy, ch) ((mask) << (8*(phy)+4*(ch)+11))
#define PHY_CH_SU_PSR 0x1
#define PHY_CH_DEEP_PSR 0x7
#define PHY_CH_POWER_MODE(mode, phy, ch) ((mode) << (6*(phy)+3*(ch)+2))
#define PHY_COM_LANE_RESET_DEASSERT(phy) (1 << (phy))
#define DISPLAY_PHY_STATUS (VLV_DISPLAY_BASE + 0x60104)
#define PHY_POWERGOOD(phy) (((phy) == DPIO_PHY0) ? (1<<31) : (1<<30))
+#define PHY_STATUS_CMN_LDO(phy, ch) (1 << (6-(6*(phy)+3*(ch))))
+#define PHY_STATUS_SPLINE_LDO(phy, ch, spline) (1 << (8-(6*(phy)+3*(ch)+(spline))))
/*
* The i830 generation, in LVDS mode, defines P1 as the bit number set within
#define PALETTE_A_OFFSET 0xa000
#define PALETTE_B_OFFSET 0xa800
#define CHV_PALETTE_C_OFFSET 0xc000
-#define PALETTE(pipe) (dev_priv->info.palette_offsets[pipe] + \
- dev_priv->info.display_mmio_offset)
+#define PALETTE(pipe, i) (dev_priv->info.palette_offsets[pipe] + \
+ dev_priv->info.display_mmio_offset + (i) * 4)
/* MCH MMIO space */
#define MCHBAR_MIRROR_BASE_SNB 0x140000
+#define CTG_STOLEN_RESERVED (MCHBAR_MIRROR_BASE + 0x34)
+#define ELK_STOLEN_RESERVED (MCHBAR_MIRROR_BASE + 0x48)
+#define G4X_STOLEN_RESERVED_ADDR1_MASK (0xFFFF << 16)
+#define G4X_STOLEN_RESERVED_ADDR2_MASK (0xFFF << 4)
+
/* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */
#define DCLK (MCHBAR_MIRROR_BASE_SNB + 0x5e04)
#define TSFS_INTR_MASK 0x000000ff
#define CRSTANDVID 0x11100
-#define PXVFREQ_BASE 0x11110 /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */
+#define PXVFREQ(i) (0x11110 + (i) * 4) /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */
#define PXVFREQ_PX_MASK 0x7f000000
#define PXVFREQ_PX_SHIFT 24
#define VIDFREQ_BASE 0x11110
#define CSIEW0 0x11250
#define CSIEW1 0x11254
#define CSIEW2 0x11258
-#define PEW 0x1125c
-#define DEW 0x11270
+#define PEW(i) (0x1125c + (i) * 4) /* 5 registers */
+#define DEW(i) (0x11270 + (i) * 4) /* 3 registers */
#define MCHAFE 0x112c0
#define CSIEC 0x112e0
#define DMIEC 0x112e4
#define EG5 0x11624
#define EG6 0x11628
#define EG7 0x1162c
-#define PXW 0x11664
-#define PXWL 0x11680
+#define PXW(i) (0x11664 + (i) * 4) /* 4 registers */
+#define PXWL(i) (0x11680 + (i) * 4) /* 8 registers */
#define LCFUSE02 0x116c0
#define LCFUSE_HIV_MASK 0x000000ff
#define CSIPLL0 0x12c10
#define INTERVAL_1_28_US(us) (((us) * 100) >> 7)
#define INTERVAL_1_33_US(us) (((us) * 3) >> 2)
+#define INTERVAL_0_833_US(us) (((us) * 6) / 5)
#define GT_INTERVAL_FROM_US(dev_priv, us) (IS_GEN9(dev_priv) ? \
- INTERVAL_1_33_US(us) : \
+ (IS_BROXTON(dev_priv) ? \
+ INTERVAL_0_833_US(us) : \
+ INTERVAL_1_33_US(us)) : \
INTERVAL_1_28_US(us))
/*
* doesn't need saving on GT1
*/
#define CXT_SIZE 0x21a0
-#define GEN6_CXT_POWER_SIZE(cxt_reg) ((cxt_reg >> 24) & 0x3f)
-#define GEN6_CXT_RING_SIZE(cxt_reg) ((cxt_reg >> 18) & 0x3f)
-#define GEN6_CXT_RENDER_SIZE(cxt_reg) ((cxt_reg >> 12) & 0x3f)
-#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) ((cxt_reg >> 6) & 0x3f)
-#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) ((cxt_reg >> 0) & 0x3f)
+#define GEN6_CXT_POWER_SIZE(cxt_reg) (((cxt_reg) >> 24) & 0x3f)
+#define GEN6_CXT_RING_SIZE(cxt_reg) (((cxt_reg) >> 18) & 0x3f)
+#define GEN6_CXT_RENDER_SIZE(cxt_reg) (((cxt_reg) >> 12) & 0x3f)
+#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) (((cxt_reg) >> 6) & 0x3f)
+#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) (((cxt_reg) >> 0) & 0x3f)
#define GEN6_CXT_TOTAL_SIZE(cxt_reg) (GEN6_CXT_RING_SIZE(cxt_reg) + \
GEN6_CXT_EXTENDED_SIZE(cxt_reg) + \
GEN6_CXT_PIPELINE_SIZE(cxt_reg))
#define GEN7_CXT_SIZE 0x21a8
-#define GEN7_CXT_POWER_SIZE(ctx_reg) ((ctx_reg >> 25) & 0x7f)
-#define GEN7_CXT_RING_SIZE(ctx_reg) ((ctx_reg >> 22) & 0x7)
-#define GEN7_CXT_RENDER_SIZE(ctx_reg) ((ctx_reg >> 16) & 0x3f)
-#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) ((ctx_reg >> 9) & 0x7f)
-#define GEN7_CXT_GT1_SIZE(ctx_reg) ((ctx_reg >> 6) & 0x7)
-#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) ((ctx_reg >> 0) & 0x3f)
+#define GEN7_CXT_POWER_SIZE(ctx_reg) (((ctx_reg) >> 25) & 0x7f)
+#define GEN7_CXT_RING_SIZE(ctx_reg) (((ctx_reg) >> 22) & 0x7)
+#define GEN7_CXT_RENDER_SIZE(ctx_reg) (((ctx_reg) >> 16) & 0x3f)
+#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) (((ctx_reg) >> 9) & 0x7f)
+#define GEN7_CXT_GT1_SIZE(ctx_reg) (((ctx_reg) >> 6) & 0x7)
+#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) (((ctx_reg) >> 0) & 0x3f)
#define GEN7_CXT_TOTAL_SIZE(ctx_reg) (GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \
GEN7_CXT_VFSTATE_SIZE(ctx_reg))
/* Haswell does have the CXT_SIZE register however it does not appear to be
#define GEN3_SDVOC 0x61160
#define GEN4_HDMIB GEN3_SDVOB
#define GEN4_HDMIC GEN3_SDVOC
-#define CHV_HDMID 0x6116C
+#define VLV_HDMIB (VLV_DISPLAY_BASE + GEN4_HDMIB)
+#define VLV_HDMIC (VLV_DISPLAY_BASE + GEN4_HDMIC)
+#define CHV_HDMID (VLV_DISPLAY_BASE + 0x6116C)
#define PCH_SDVOB 0xe1140
#define PCH_HDMIB PCH_SDVOB
#define PCH_HDMIC 0xe1150
#define UTIL_PIN_CTL 0x48400
#define UTIL_PIN_ENABLE (1 << 31)
+#define UTIL_PIN_PIPE(x) ((x) << 29)
+#define UTIL_PIN_PIPE_MASK (3 << 29)
+#define UTIL_PIN_MODE_PWM (1 << 24)
+#define UTIL_PIN_MODE_MASK (0xf << 24)
+#define UTIL_PIN_POLARITY (1 << 22)
+
/* BXT backlight register definition. */
-#define BXT_BLC_PWM_CTL1 0xC8250
+#define _BXT_BLC_PWM_CTL1 0xC8250
#define BXT_BLC_PWM_ENABLE (1 << 31)
#define BXT_BLC_PWM_POLARITY (1 << 29)
-#define BXT_BLC_PWM_FREQ1 0xC8254
-#define BXT_BLC_PWM_DUTY1 0xC8258
+#define _BXT_BLC_PWM_FREQ1 0xC8254
+#define _BXT_BLC_PWM_DUTY1 0xC8258
-#define BXT_BLC_PWM_CTL2 0xC8350
-#define BXT_BLC_PWM_FREQ2 0xC8354
-#define BXT_BLC_PWM_DUTY2 0xC8358
+#define _BXT_BLC_PWM_CTL2 0xC8350
+#define _BXT_BLC_PWM_FREQ2 0xC8354
+#define _BXT_BLC_PWM_DUTY2 0xC8358
+#define BXT_BLC_PWM_CTL(controller) _PIPE(controller, \
+ _BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2)
+#define BXT_BLC_PWM_FREQ(controller) _PIPE(controller, \
+ _BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2)
+#define BXT_BLC_PWM_DUTY(controller) _PIPE(controller, \
+ _BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2)
#define PCH_GTC_CTL 0xe7000
#define PCH_GTC_ENABLE (1 << 31)
# define TV_CC_DATA_1_MASK 0x0000007f
# define TV_CC_DATA_1_SHIFT 0
-#define TV_H_LUMA_0 0x68100
-#define TV_H_LUMA_59 0x681ec
-#define TV_H_CHROMA_0 0x68200
-#define TV_H_CHROMA_59 0x682ec
-#define TV_V_LUMA_0 0x68300
-#define TV_V_LUMA_42 0x683a8
-#define TV_V_CHROMA_0 0x68400
-#define TV_V_CHROMA_42 0x684a8
+#define TV_H_LUMA(i) (0x68100 + (i) * 4) /* 60 registers */
+#define TV_H_CHROMA(i) (0x68200 + (i) * 4) /* 60 registers */
+#define TV_V_LUMA(i) (0x68300 + (i) * 4) /* 43 registers */
+#define TV_V_CHROMA(i) (0x68400 + (i) * 4) /* 43 registers */
/* Display Port */
#define DP_A 0x64000 /* eDP */
#define DP_C 0x64200
#define DP_D 0x64300
+#define VLV_DP_B (VLV_DISPLAY_BASE + DP_B)
+#define VLV_DP_C (VLV_DISPLAY_BASE + DP_C)
+#define CHV_DP_D (VLV_DISPLAY_BASE + DP_D)
+
#define DP_PORT_EN (1 << 31)
#define DP_PIPEB_SELECT (1 << 30)
#define DP_PIPE_MASK (1 << 30)
/* How many wires to use. I guess 3 was too hard */
#define DP_PORT_WIDTH(width) (((width) - 1) << 19)
#define DP_PORT_WIDTH_MASK (7 << 19)
+#define DP_PORT_WIDTH_SHIFT 19
/* Mystic DPCD version 1.1 special mode */
#define DP_ENHANCED_FRAMING (1 << 18)
#define DP_AUX_CH_CTL_PSR_DATA_AUX_REG_SKL (1 << 14)
#define DP_AUX_CH_CTL_FS_DATA_AUX_REG_SKL (1 << 13)
#define DP_AUX_CH_CTL_GTC_DATA_AUX_REG_SKL (1 << 12)
-#define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL_MASK (1f << 5)
+#define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL_MASK (0x1f << 5)
#define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(c) (((c) - 1) << 5)
#define DP_AUX_CH_CTL_SYNC_PULSE_SKL(c) ((c) - 1)
#define CBR1_VLV (VLV_DISPLAY_BASE + 0x70400)
#define CBR_PND_DEADLINE_DISABLE (1<<31)
+#define CBR_PWM_CLOCK_MUX_SELECT (1<<30)
/* FIFO watermark sizes etc */
#define G4X_FIFO_LINE_SIZE 64
#define PIPE_PIXEL_MASK 0x00ffffff
#define PIPE_PIXEL_SHIFT 0
/* GM45+ just has to be different */
-#define _PIPEA_FRMCOUNT_GM45 0x70040
-#define _PIPEA_FLIPCOUNT_GM45 0x70044
-#define PIPE_FRMCOUNT_GM45(pipe) _PIPE2(pipe, _PIPEA_FRMCOUNT_GM45)
-#define PIPE_FLIPCOUNT_GM45(pipe) _PIPE2(pipe, _PIPEA_FLIPCOUNT_GM45)
+#define _PIPEA_FRMCOUNT_G4X 0x70040
+#define _PIPEA_FLIPCOUNT_G4X 0x70044
+#define PIPE_FRMCOUNT_G4X(pipe) _PIPE2(pipe, _PIPEA_FRMCOUNT_G4X)
+#define PIPE_FLIPCOUNT_G4X(pipe) _PIPE2(pipe, _PIPEA_FLIPCOUNT_G4X)
/* Cursor A & B regs */
#define _CURACNTR 0x70080
#define I915_LO_DISPBASE(val) (val & ~DISP_BASEADDR_MASK)
#define I915_HI_DISPBASE(val) (val & DISP_BASEADDR_MASK)
-/* VBIOS flags */
-#define SWF00 (dev_priv->info.display_mmio_offset + 0x71410)
-#define SWF01 (dev_priv->info.display_mmio_offset + 0x71414)
-#define SWF02 (dev_priv->info.display_mmio_offset + 0x71418)
-#define SWF03 (dev_priv->info.display_mmio_offset + 0x7141c)
-#define SWF04 (dev_priv->info.display_mmio_offset + 0x71420)
-#define SWF05 (dev_priv->info.display_mmio_offset + 0x71424)
-#define SWF06 (dev_priv->info.display_mmio_offset + 0x71428)
-#define SWF10 (dev_priv->info.display_mmio_offset + 0x70410)
-#define SWF11 (dev_priv->info.display_mmio_offset + 0x70414)
-#define SWF14 (dev_priv->info.display_mmio_offset + 0x71420)
-#define SWF30 (dev_priv->info.display_mmio_offset + 0x72414)
-#define SWF31 (dev_priv->info.display_mmio_offset + 0x72418)
-#define SWF32 (dev_priv->info.display_mmio_offset + 0x7241c)
+/*
+ * VBIOS flags
+ * gen2:
+ * [00:06] alm,mgm
+ * [10:16] all
+ * [30:32] alm,mgm
+ * gen3+:
+ * [00:0f] all
+ * [10:1f] all
+ * [30:32] all
+ */
+#define SWF0(i) (dev_priv->info.display_mmio_offset + 0x70410 + (i) * 4)
+#define SWF1(i) (dev_priv->info.display_mmio_offset + 0x71410 + (i) * 4)
+#define SWF3(i) (dev_priv->info.display_mmio_offset + 0x72414 + (i) * 4)
/* Pipe B */
#define _PIPEBDSL (dev_priv->info.display_mmio_offset + 0x71000)
#define _PIPEBSTAT (dev_priv->info.display_mmio_offset + 0x71024)
#define _PIPEBFRAMEHIGH 0x71040
#define _PIPEBFRAMEPIXEL 0x71044
-#define _PIPEB_FRMCOUNT_GM45 (dev_priv->info.display_mmio_offset + 0x71040)
-#define _PIPEB_FLIPCOUNT_GM45 (dev_priv->info.display_mmio_offset + 0x71044)
+#define _PIPEB_FRMCOUNT_G4X (dev_priv->info.display_mmio_offset + 0x71040)
+#define _PIPEB_FLIPCOUNT_G4X (dev_priv->info.display_mmio_offset + 0x71044)
/* Display B control */
#define _SPBCONSTALPHA (VLV_DISPLAY_BASE + 0x722a8)
#define _SPBGAMC (VLV_DISPLAY_BASE + 0x722f4)
-#define SPCNTR(pipe, plane) _PIPE(pipe * 2 + plane, _SPACNTR, _SPBCNTR)
-#define SPLINOFF(pipe, plane) _PIPE(pipe * 2 + plane, _SPALINOFF, _SPBLINOFF)
-#define SPSTRIDE(pipe, plane) _PIPE(pipe * 2 + plane, _SPASTRIDE, _SPBSTRIDE)
-#define SPPOS(pipe, plane) _PIPE(pipe * 2 + plane, _SPAPOS, _SPBPOS)
-#define SPSIZE(pipe, plane) _PIPE(pipe * 2 + plane, _SPASIZE, _SPBSIZE)
-#define SPKEYMINVAL(pipe, plane) _PIPE(pipe * 2 + plane, _SPAKEYMINVAL, _SPBKEYMINVAL)
-#define SPKEYMSK(pipe, plane) _PIPE(pipe * 2 + plane, _SPAKEYMSK, _SPBKEYMSK)
-#define SPSURF(pipe, plane) _PIPE(pipe * 2 + plane, _SPASURF, _SPBSURF)
-#define SPKEYMAXVAL(pipe, plane) _PIPE(pipe * 2 + plane, _SPAKEYMAXVAL, _SPBKEYMAXVAL)
-#define SPTILEOFF(pipe, plane) _PIPE(pipe * 2 + plane, _SPATILEOFF, _SPBTILEOFF)
-#define SPCONSTALPHA(pipe, plane) _PIPE(pipe * 2 + plane, _SPACONSTALPHA, _SPBCONSTALPHA)
-#define SPGAMC(pipe, plane) _PIPE(pipe * 2 + plane, _SPAGAMC, _SPBGAMC)
+#define SPCNTR(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPACNTR, _SPBCNTR)
+#define SPLINOFF(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPALINOFF, _SPBLINOFF)
+#define SPSTRIDE(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPASTRIDE, _SPBSTRIDE)
+#define SPPOS(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAPOS, _SPBPOS)
+#define SPSIZE(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPASIZE, _SPBSIZE)
+#define SPKEYMINVAL(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAKEYMINVAL, _SPBKEYMINVAL)
+#define SPKEYMSK(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAKEYMSK, _SPBKEYMSK)
+#define SPSURF(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPASURF, _SPBSURF)
+#define SPKEYMAXVAL(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAKEYMAXVAL, _SPBKEYMAXVAL)
+#define SPTILEOFF(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPATILEOFF, _SPBTILEOFF)
+#define SPCONSTALPHA(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPACONSTALPHA, _SPBCONSTALPHA)
+#define SPGAMC(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAGAMC, _SPBGAMC)
/*
* CHV pipe B sprite CSC
#define CPU_VGACNTRL 0x41000
-#define DIGITAL_PORT_HOTPLUG_CNTRL 0x44030
-#define DIGITAL_PORTA_HOTPLUG_ENABLE (1 << 4)
-#define DIGITAL_PORTA_SHORT_PULSE_2MS (0 << 2)
-#define DIGITAL_PORTA_SHORT_PULSE_4_5MS (1 << 2)
-#define DIGITAL_PORTA_SHORT_PULSE_6MS (2 << 2)
-#define DIGITAL_PORTA_SHORT_PULSE_100MS (3 << 2)
-#define DIGITAL_PORTA_NO_DETECT (0 << 0)
-#define DIGITAL_PORTA_LONG_PULSE_DETECT_MASK (1 << 1)
-#define DIGITAL_PORTA_SHORT_PULSE_DETECT_MASK (1 << 0)
+#define DIGITAL_PORT_HOTPLUG_CNTRL 0x44030
+#define DIGITAL_PORTA_HOTPLUG_ENABLE (1 << 4)
+#define DIGITAL_PORTA_PULSE_DURATION_2ms (0 << 2) /* pre-HSW */
+#define DIGITAL_PORTA_PULSE_DURATION_4_5ms (1 << 2) /* pre-HSW */
+#define DIGITAL_PORTA_PULSE_DURATION_6ms (2 << 2) /* pre-HSW */
+#define DIGITAL_PORTA_PULSE_DURATION_100ms (3 << 2) /* pre-HSW */
+#define DIGITAL_PORTA_PULSE_DURATION_MASK (3 << 2) /* pre-HSW */
+#define DIGITAL_PORTA_HOTPLUG_STATUS_MASK (3 << 0)
+#define DIGITAL_PORTA_HOTPLUG_NO_DETECT (0 << 0)
+#define DIGITAL_PORTA_HOTPLUG_SHORT_DETECT (1 << 0)
+#define DIGITAL_PORTA_HOTPLUG_LONG_DETECT (2 << 0)
/* refresh rate hardware control */
#define RR_HW_CTL 0x45300
#define PS_SCALER_MODE_DYN (0 << 28)
#define PS_SCALER_MODE_HQ (1 << 28)
#define PS_PLANE_SEL_MASK (7 << 25)
-#define PS_PLANE_SEL(plane) ((plane + 1) << 25)
+#define PS_PLANE_SEL(plane) (((plane) + 1) << 25)
#define PS_FILTER_MASK (3 << 23)
#define PS_FILTER_MEDIUM (0 << 23)
#define PS_FILTER_EDGE_ENHANCE (2 << 23)
/* legacy palette */
#define _LGC_PALETTE_A 0x4a000
#define _LGC_PALETTE_B 0x4a800
-#define LGC_PALETTE(pipe) _PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B)
+#define LGC_PALETTE(pipe, i) (_PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B) + (i) * 4)
#define _GAMMA_MODE_A 0x4a480
#define _GAMMA_MODE_B 0x4ac80
#define DE_PLANEA_FLIP_DONE_IVB (1<<3)
#define DE_PLANE_FLIP_DONE_IVB(plane) (1<< (3 + 5*(plane)))
#define DE_PIPEA_VBLANK_IVB (1<<0)
-#define DE_PIPE_VBLANK_IVB(pipe) (1 << (pipe * 5))
+#define DE_PIPE_VBLANK_IVB(pipe) (1 << ((pipe) * 5))
#define VLV_MASTER_IER 0x4400c /* Gunit master IER */
#define MASTER_INTERRUPT_ENABLE (1<<31)
#define GEN8_DE_PIPE_C_IRQ (1<<18)
#define GEN8_DE_PIPE_B_IRQ (1<<17)
#define GEN8_DE_PIPE_A_IRQ (1<<16)
-#define GEN8_DE_PIPE_IRQ(pipe) (1<<(16+pipe))
+#define GEN8_DE_PIPE_IRQ(pipe) (1<<(16+(pipe)))
#define GEN8_GT_VECS_IRQ (1<<6)
#define GEN8_GT_PM_IRQ (1<<4)
#define GEN8_GT_VCS2_IRQ (1<<3)
#define GEN8_GT_IIR(which) (0x44308 + (0x10 * (which)))
#define GEN8_GT_IER(which) (0x4430c + (0x10 * (which)))
-#define GEN8_BCS_IRQ_SHIFT 16
#define GEN8_RCS_IRQ_SHIFT 0
-#define GEN8_VCS2_IRQ_SHIFT 16
+#define GEN8_BCS_IRQ_SHIFT 16
#define GEN8_VCS1_IRQ_SHIFT 0
+#define GEN8_VCS2_IRQ_SHIFT 16
#define GEN8_VECS_IRQ_SHIFT 0
+#define GEN8_WD_IRQ_SHIFT 16
#define GEN8_DE_PIPE_ISR(pipe) (0x44400 + (0x10 * (pipe)))
#define GEN8_DE_PIPE_IMR(pipe) (0x44404 + (0x10 * (pipe)))
#define GEN9_PIPE_PLANE3_FLIP_DONE (1 << 5)
#define GEN9_PIPE_PLANE2_FLIP_DONE (1 << 4)
#define GEN9_PIPE_PLANE1_FLIP_DONE (1 << 3)
-#define GEN9_PIPE_PLANE_FLIP_DONE(p) (1 << (3 + p))
+#define GEN9_PIPE_PLANE_FLIP_DONE(p) (1 << (3 + (p)))
#define GEN8_DE_PIPE_IRQ_FAULT_ERRORS \
(GEN8_PIPE_CURSOR_FAULT | \
GEN8_PIPE_SPRITE_FAULT | \
#define GEN8_PCU_IIR 0x444e8
#define GEN8_PCU_IER 0x444ec
-/* BXT hotplug control */
-#define BXT_HOTPLUG_CTL 0xC4030
-#define BXT_DDIA_HPD_ENABLE (1 << 28)
-#define BXT_DDIA_HPD_STATUS (3 << 24)
-#define BXT_DDIC_HPD_ENABLE (1 << 12)
-#define BXT_DDIC_HPD_STATUS (3 << 8)
-#define BXT_DDIB_HPD_ENABLE (1 << 4)
-#define BXT_DDIB_HPD_STATUS (3 << 0)
-#define BXT_HOTPLUG_CTL_MASK (BXT_DDIA_HPD_ENABLE | \
- BXT_DDIB_HPD_ENABLE | \
- BXT_DDIC_HPD_ENABLE)
-#define BXT_HPD_STATUS_MASK (BXT_DDIA_HPD_STATUS | \
- BXT_DDIB_HPD_STATUS | \
- BXT_DDIC_HPD_STATUS)
-
#define ILK_DISPLAY_CHICKEN2 0x42004
/* Required on all Ironlake and Sandybridge according to the B-Spec. */
#define ILK_ELPIN_409_SELECT (1 << 25)
#define SDE_AUXB_CPT (1 << 25)
#define SDE_AUX_MASK_CPT (7 << 25)
#define SDE_PORTE_HOTPLUG_SPT (1 << 25)
+#define SDE_PORTA_HOTPLUG_SPT (1 << 24)
#define SDE_PORTD_HOTPLUG_CPT (1 << 23)
#define SDE_PORTC_HOTPLUG_CPT (1 << 22)
#define SDE_PORTB_HOTPLUG_CPT (1 << 21)
#define SDE_HOTPLUG_MASK_SPT (SDE_PORTE_HOTPLUG_SPT | \
SDE_PORTD_HOTPLUG_CPT | \
SDE_PORTC_HOTPLUG_CPT | \
- SDE_PORTB_HOTPLUG_CPT)
+ SDE_PORTB_HOTPLUG_CPT | \
+ SDE_PORTA_HOTPLUG_SPT)
#define SDE_GMBUS_CPT (1 << 17)
#define SDE_ERROR_CPT (1 << 16)
#define SDE_AUDIO_CP_REQ_C_CPT (1 << 10)
#define SERR_INT_TRANS_C_FIFO_UNDERRUN (1<<6)
#define SERR_INT_TRANS_B_FIFO_UNDERRUN (1<<3)
#define SERR_INT_TRANS_A_FIFO_UNDERRUN (1<<0)
-#define SERR_INT_TRANS_FIFO_UNDERRUN(pipe) (1<<(pipe*3))
+#define SERR_INT_TRANS_FIFO_UNDERRUN(pipe) (1<<((pipe)*3))
/* digital port hotplug */
-#define PCH_PORT_HOTPLUG 0xc4030 /* SHOTPLUG_CTL */
-#define BXT_PORTA_HOTPLUG_ENABLE (1 << 28)
-#define BXT_PORTA_HOTPLUG_STATUS_MASK (0x3 << 24)
-#define BXT_PORTA_HOTPLUG_NO_DETECT (0 << 24)
-#define BXT_PORTA_HOTPLUG_SHORT_DETECT (1 << 24)
-#define BXT_PORTA_HOTPLUG_LONG_DETECT (2 << 24)
-#define PORTD_HOTPLUG_ENABLE (1 << 20)
-#define PORTD_PULSE_DURATION_2ms (0)
-#define PORTD_PULSE_DURATION_4_5ms (1 << 18)
-#define PORTD_PULSE_DURATION_6ms (2 << 18)
-#define PORTD_PULSE_DURATION_100ms (3 << 18)
-#define PORTD_PULSE_DURATION_MASK (3 << 18)
-#define PORTD_HOTPLUG_STATUS_MASK (0x3 << 16)
+#define PCH_PORT_HOTPLUG 0xc4030 /* SHOTPLUG_CTL */
+#define PORTA_HOTPLUG_ENABLE (1 << 28) /* LPT:LP+ & BXT */
+#define PORTA_HOTPLUG_STATUS_MASK (3 << 24) /* SPT+ & BXT */
+#define PORTA_HOTPLUG_NO_DETECT (0 << 24) /* SPT+ & BXT */
+#define PORTA_HOTPLUG_SHORT_DETECT (1 << 24) /* SPT+ & BXT */
+#define PORTA_HOTPLUG_LONG_DETECT (2 << 24) /* SPT+ & BXT */
+#define PORTD_HOTPLUG_ENABLE (1 << 20)
+#define PORTD_PULSE_DURATION_2ms (0 << 18) /* pre-LPT */
+#define PORTD_PULSE_DURATION_4_5ms (1 << 18) /* pre-LPT */
+#define PORTD_PULSE_DURATION_6ms (2 << 18) /* pre-LPT */
+#define PORTD_PULSE_DURATION_100ms (3 << 18) /* pre-LPT */
+#define PORTD_PULSE_DURATION_MASK (3 << 18) /* pre-LPT */
+#define PORTD_HOTPLUG_STATUS_MASK (3 << 16)
#define PORTD_HOTPLUG_NO_DETECT (0 << 16)
#define PORTD_HOTPLUG_SHORT_DETECT (1 << 16)
#define PORTD_HOTPLUG_LONG_DETECT (2 << 16)
-#define PORTC_HOTPLUG_ENABLE (1 << 12)
-#define PORTC_PULSE_DURATION_2ms (0)
-#define PORTC_PULSE_DURATION_4_5ms (1 << 10)
-#define PORTC_PULSE_DURATION_6ms (2 << 10)
-#define PORTC_PULSE_DURATION_100ms (3 << 10)
-#define PORTC_PULSE_DURATION_MASK (3 << 10)
-#define PORTC_HOTPLUG_STATUS_MASK (0x3 << 8)
+#define PORTC_HOTPLUG_ENABLE (1 << 12)
+#define PORTC_PULSE_DURATION_2ms (0 << 10) /* pre-LPT */
+#define PORTC_PULSE_DURATION_4_5ms (1 << 10) /* pre-LPT */
+#define PORTC_PULSE_DURATION_6ms (2 << 10) /* pre-LPT */
+#define PORTC_PULSE_DURATION_100ms (3 << 10) /* pre-LPT */
+#define PORTC_PULSE_DURATION_MASK (3 << 10) /* pre-LPT */
+#define PORTC_HOTPLUG_STATUS_MASK (3 << 8)
#define PORTC_HOTPLUG_NO_DETECT (0 << 8)
#define PORTC_HOTPLUG_SHORT_DETECT (1 << 8)
#define PORTC_HOTPLUG_LONG_DETECT (2 << 8)
-#define PORTB_HOTPLUG_ENABLE (1 << 4)
-#define PORTB_PULSE_DURATION_2ms (0)
-#define PORTB_PULSE_DURATION_4_5ms (1 << 2)
-#define PORTB_PULSE_DURATION_6ms (2 << 2)
-#define PORTB_PULSE_DURATION_100ms (3 << 2)
-#define PORTB_PULSE_DURATION_MASK (3 << 2)
-#define PORTB_HOTPLUG_STATUS_MASK (0x3 << 0)
+#define PORTB_HOTPLUG_ENABLE (1 << 4)
+#define PORTB_PULSE_DURATION_2ms (0 << 2) /* pre-LPT */
+#define PORTB_PULSE_DURATION_4_5ms (1 << 2) /* pre-LPT */
+#define PORTB_PULSE_DURATION_6ms (2 << 2) /* pre-LPT */
+#define PORTB_PULSE_DURATION_100ms (3 << 2) /* pre-LPT */
+#define PORTB_PULSE_DURATION_MASK (3 << 2) /* pre-LPT */
+#define PORTB_HOTPLUG_STATUS_MASK (3 << 0)
#define PORTB_HOTPLUG_NO_DETECT (0 << 0)
#define PORTB_HOTPLUG_SHORT_DETECT (1 << 0)
#define PORTB_HOTPLUG_LONG_DETECT (2 << 0)
-#define PCH_PORT_HOTPLUG2 0xc403C /* SHOTPLUG_CTL2 */
-#define PORTE_HOTPLUG_ENABLE (1 << 4)
-#define PORTE_HOTPLUG_STATUS_MASK (0x3 << 0)
+#define PCH_PORT_HOTPLUG2 0xc403C /* SHOTPLUG_CTL2 SPT+ */
+#define PORTE_HOTPLUG_ENABLE (1 << 4)
+#define PORTE_HOTPLUG_STATUS_MASK (3 << 0)
#define PORTE_HOTPLUG_NO_DETECT (0 << 0)
#define PORTE_HOTPLUG_SHORT_DETECT (1 << 0)
#define PORTE_HOTPLUG_LONG_DETECT (2 << 0)
#define PCH_SSC4_AUX_PARMS 0xc6214
#define PCH_DPLL_SEL 0xc7000
-#define TRANS_DPLLB_SEL(pipe) (1 << (pipe * 4))
+#define TRANS_DPLLB_SEL(pipe) (1 << ((pipe) * 4))
#define TRANS_DPLLA_SEL(pipe) 0
-#define TRANS_DPLL_ENABLE(pipe) (1 << (pipe * 4 + 3))
+#define TRANS_DPLL_ENABLE(pipe) (1 << ((pipe) * 4 + 3))
/* transcoder */
#define HSW_TVIDEO_DIP_CTL(trans) \
_TRANSCODER2(trans, HSW_VIDEO_DIP_CTL_A)
-#define HSW_TVIDEO_DIP_AVI_DATA(trans) \
- _TRANSCODER2(trans, HSW_VIDEO_DIP_AVI_DATA_A)
-#define HSW_TVIDEO_DIP_VS_DATA(trans) \
- _TRANSCODER2(trans, HSW_VIDEO_DIP_VS_DATA_A)
-#define HSW_TVIDEO_DIP_SPD_DATA(trans) \
- _TRANSCODER2(trans, HSW_VIDEO_DIP_SPD_DATA_A)
+#define HSW_TVIDEO_DIP_AVI_DATA(trans, i) \
+ (_TRANSCODER2(trans, HSW_VIDEO_DIP_AVI_DATA_A) + (i) * 4)
+#define HSW_TVIDEO_DIP_VS_DATA(trans, i) \
+ (_TRANSCODER2(trans, HSW_VIDEO_DIP_VS_DATA_A) + (i) * 4)
+#define HSW_TVIDEO_DIP_SPD_DATA(trans, i) \
+ (_TRANSCODER2(trans, HSW_VIDEO_DIP_SPD_DATA_A) + (i) * 4)
#define HSW_TVIDEO_DIP_GCP(trans) \
_TRANSCODER2(trans, HSW_VIDEO_DIP_GCP_A)
-#define HSW_TVIDEO_DIP_VSC_DATA(trans) \
- _TRANSCODER2(trans, HSW_VIDEO_DIP_VSC_DATA_A)
+#define HSW_TVIDEO_DIP_VSC_DATA(trans, i) \
+ (_TRANSCODER2(trans, HSW_VIDEO_DIP_VSC_DATA_A) + (i) * 4)
#define HSW_STEREO_3D_CTL_A 0x70020
#define S3D_ENABLE (1<<31)
#define FDI_PHASE_SYNC_OVR(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2)))
#define FDI_PHASE_SYNC_EN(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2)))
#define FDI_BC_BIFURCATION_SELECT (1 << 12)
+#define SPT_PWM_GRANULARITY (1<<0)
#define SOUTH_CHICKEN2 0xc2004
#define FDI_MPHY_IOSFSB_RESET_STATUS (1<<13)
#define FDI_MPHY_IOSFSB_RESET_CTL (1<<12)
+#define LPT_PWM_GRANULARITY (1<<5)
#define DPLS_EDP_PPS_FIX_DIS (1<<0)
#define _FDI_RXA_CHICKEN 0xc200c
#define _BXT_PP_ON_DELAYS2 0xc7308
#define _BXT_PP_OFF_DELAYS2 0xc730c
-#define BXT_PP_STATUS(n) ((!n) ? PCH_PP_STATUS : _BXT_PP_STATUS2)
-#define BXT_PP_CONTROL(n) ((!n) ? PCH_PP_CONTROL : _BXT_PP_CONTROL2)
-#define BXT_PP_ON_DELAYS(n) ((!n) ? PCH_PP_ON_DELAYS : _BXT_PP_ON_DELAYS2)
-#define BXT_PP_OFF_DELAYS(n) ((!n) ? PCH_PP_OFF_DELAYS : _BXT_PP_OFF_DELAYS2)
+#define BXT_PP_STATUS(n) _PIPE(n, PCH_PP_STATUS, _BXT_PP_STATUS2)
+#define BXT_PP_CONTROL(n) _PIPE(n, PCH_PP_CONTROL, _BXT_PP_CONTROL2)
+#define BXT_PP_ON_DELAYS(n) _PIPE(n, PCH_PP_ON_DELAYS, _BXT_PP_ON_DELAYS2)
+#define BXT_PP_OFF_DELAYS(n) _PIPE(n, PCH_PP_OFF_DELAYS, _BXT_PP_OFF_DELAYS2)
#define PCH_DP_B 0xe4100
#define PCH_DPB_AUX_CH_CTL 0xe4110
GEN6_PM_RP_DOWN_THRESHOLD | \
GEN6_PM_RP_DOWN_TIMEOUT)
-#define GEN7_GT_SCRATCH_BASE 0x4F100
+#define GEN7_GT_SCRATCH(i) (0x4F100 + (i) * 4)
#define GEN7_GT_SCRATCH_REG_NUM 8
#define VLV_GTLC_SURVIVABILITY_REG 0x130098
#define GEN6_RC6 3
#define GEN6_RC7 4
+#define GEN8_GT_SLICE_INFO 0x138064
+#define GEN8_LSLICESTAT_MASK 0x7
+
#define CHV_POWER_SS0_SIG1 0xa720
#define CHV_POWER_SS1_SIG1 0xa728
#define CHV_SS_PG_ENABLE (1<<1)
#define GEN9_PGCTL_SSB_EU311_ACK (1 << 14)
#define GEN7_MISCCPCTL (0x9424)
-#define GEN7_DOP_CLOCK_GATE_ENABLE (1<<0)
+#define GEN7_DOP_CLOCK_GATE_ENABLE (1<<0)
+#define GEN8_DOP_CLOCK_GATE_CFCLK_ENABLE (1<<2)
+#define GEN8_DOP_CLOCK_GATE_GUC_ENABLE (1<<4)
+#define GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE (1<<6)
#define GEN8_GARBCNTL 0xB004
#define GEN9_GAPS_TSV_CREDIT_DISABLE (1<<7)
#define HSW_ROW_CHICKEN3 0xe49c
#define HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE (1 << 6)
+#define HALF_SLICE_CHICKEN2 0xe180
+#define GEN8_ST_PO_DISABLE (1<<13)
+
#define HALF_SLICE_CHICKEN3 0xe184
#define HSW_SAMPLE_C_PERFORMANCE (1<<9)
#define GEN8_CENTROID_PIXEL_OPT_DIS (1<<8)
#define DDI_BUF_IS_IDLE (1<<7)
#define DDI_A_4_LANES (1<<4)
#define DDI_PORT_WIDTH(width) (((width) - 1) << 1)
+#define DDI_PORT_WIDTH_MASK (7 << 1)
+#define DDI_PORT_WIDTH_SHIFT 1
#define DDI_INIT_DISPLAY_DETECTED (1<<0)
/* DDI Buffer Translations */
#define DDI_BUF_TRANS_A 0x64E00
#define DDI_BUF_TRANS_B 0x64E60
-#define DDI_BUF_TRANS(port) _PORT(port, DDI_BUF_TRANS_A, DDI_BUF_TRANS_B)
+#define DDI_BUF_TRANS_LO(port, i) (_PORT(port, DDI_BUF_TRANS_A, DDI_BUF_TRANS_B) + (i) * 8)
+#define DDI_BUF_TRANS_HI(port, i) (_PORT(port, DDI_BUF_TRANS_A, DDI_BUF_TRANS_B) + (i) * 8 + 4)
/* Sideband Interface (SBI) is programmed indirectly, via
* SBI_ADDR, which contains the register offset; and SBI_DATA,
#define TRANS_CLK_SEL(tran) _TRANSCODER(tran, TRANS_CLK_SEL_A, TRANS_CLK_SEL_B)
/* For each transcoder, we need to select the corresponding port clock */
#define TRANS_CLK_SEL_DISABLED (0x0<<29)
-#define TRANS_CLK_SEL_PORT(x) ((x+1)<<29)
+#define TRANS_CLK_SEL_PORT(x) (((x)+1)<<29)
#define TRANSA_MSA_MISC 0x60410
#define TRANSB_MSA_MISC 0x61410
/* DPLL control2 */
#define DPLL_CTRL2 0x6C05C
-#define DPLL_CTRL2_DDI_CLK_OFF(port) (1<<(port+15))
+#define DPLL_CTRL2_DDI_CLK_OFF(port) (1<<((port)+15))
#define DPLL_CTRL2_DDI_CLK_SEL_MASK(port) (3<<((port)*3+1))
#define DPLL_CTRL2_DDI_CLK_SEL_SHIFT(port) ((port)*3+1)
-#define DPLL_CTRL2_DDI_CLK_SEL(clk, port) (clk<<((port)*3+1))
+#define DPLL_CTRL2_DDI_CLK_SEL(clk, port) ((clk)<<((port)*3+1))
#define DPLL_CTRL2_DDI_SEL_OVERRIDE(port) (1<<((port)*3))
/* DPLL Status */
#define DPLL3_CFGCR1 0x6C050
#define DPLL_CFGCR1_FREQ_ENABLE (1<<31)
#define DPLL_CFGCR1_DCO_FRACTION_MASK (0x7fff<<9)
-#define DPLL_CFGCR1_DCO_FRACTION(x) (x<<9)
+#define DPLL_CFGCR1_DCO_FRACTION(x) ((x)<<9)
#define DPLL_CFGCR1_DCO_INTEGER_MASK (0x1ff)
#define DPLL1_CFGCR2 0x6C044
#define DPLL2_CFGCR2 0x6C04C
#define DPLL3_CFGCR2 0x6C054
#define DPLL_CFGCR2_QDIV_RATIO_MASK (0xff<<8)
-#define DPLL_CFGCR2_QDIV_RATIO(x) (x<<8)
-#define DPLL_CFGCR2_QDIV_MODE(x) (x<<7)
+#define DPLL_CFGCR2_QDIV_RATIO(x) ((x)<<8)
+#define DPLL_CFGCR2_QDIV_MODE(x) ((x)<<7)
#define DPLL_CFGCR2_KDIV_MASK (3<<5)
-#define DPLL_CFGCR2_KDIV(x) (x<<5)
+#define DPLL_CFGCR2_KDIV(x) ((x)<<5)
#define DPLL_CFGCR2_KDIV_5 (0<<5)
#define DPLL_CFGCR2_KDIV_2 (1<<5)
#define DPLL_CFGCR2_KDIV_3 (2<<5)
#define DPLL_CFGCR2_KDIV_1 (3<<5)
#define DPLL_CFGCR2_PDIV_MASK (7<<2)
-#define DPLL_CFGCR2_PDIV(x) (x<<2)
+#define DPLL_CFGCR2_PDIV(x) ((x)<<2)
#define DPLL_CFGCR2_PDIV_1 (0<<2)
#define DPLL_CFGCR2_PDIV_2 (1<<2)
#define DPLL_CFGCR2_PDIV_3 (2<<2)
#define DPLL_CFGCR2_PDIV_7 (4<<2)
#define DPLL_CFGCR2_CENTRAL_FREQ_MASK (3)
-#define GET_CFG_CR1_REG(id) (DPLL1_CFGCR1 + (id - SKL_DPLL1) * 8)
-#define GET_CFG_CR2_REG(id) (DPLL1_CFGCR2 + (id - SKL_DPLL1) * 8)
+#define DPLL_CFGCR1(id) (DPLL1_CFGCR1 + ((id) - SKL_DPLL1) * 8)
+#define DPLL_CFGCR2(id) (DPLL1_CFGCR2 + ((id) - SKL_DPLL1) * 8)
/* BXT display engine PLL */
#define BXT_DE_PLL_CTL 0x6d000
#define _MIPI_PORT(port, a, c) _PORT3(port, a, 0, c) /* ports A and C only */
+/* BXT MIPI clock controls */
+#define BXT_MAX_VAR_OUTPUT_KHZ 39500
+
+#define BXT_MIPI_CLOCK_CTL 0x46090
+#define BXT_MIPI1_DIV_SHIFT 26
+#define BXT_MIPI2_DIV_SHIFT 10
+#define BXT_MIPI_DIV_SHIFT(port) \
+ _MIPI_PORT(port, BXT_MIPI1_DIV_SHIFT, \
+ BXT_MIPI2_DIV_SHIFT)
+/* Var clock divider to generate TX source. Result must be < 39.5 M */
+#define BXT_MIPI1_ESCLK_VAR_DIV_MASK (0x3F << 26)
+#define BXT_MIPI2_ESCLK_VAR_DIV_MASK (0x3F << 10)
+#define BXT_MIPI_ESCLK_VAR_DIV_MASK(port) \
+ _MIPI_PORT(port, BXT_MIPI1_ESCLK_VAR_DIV_MASK, \
+ BXT_MIPI2_ESCLK_VAR_DIV_MASK)
+
+#define BXT_MIPI_ESCLK_VAR_DIV(port, val) \
+ (val << BXT_MIPI_DIV_SHIFT(port))
+/* TX control divider to select actual TX clock output from (8x/var) */
+#define BXT_MIPI1_TX_ESCLK_SHIFT 21
+#define BXT_MIPI2_TX_ESCLK_SHIFT 5
+#define BXT_MIPI_TX_ESCLK_SHIFT(port) \
+ _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_SHIFT, \
+ BXT_MIPI2_TX_ESCLK_SHIFT)
+#define BXT_MIPI1_TX_ESCLK_FIXDIV_MASK (3 << 21)
+#define BXT_MIPI2_TX_ESCLK_FIXDIV_MASK (3 << 5)
+#define BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port) \
+ _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_FIXDIV_MASK, \
+ BXT_MIPI2_TX_ESCLK_FIXDIV_MASK)
+#define BXT_MIPI_TX_ESCLK_8XDIV_BY2(port) \
+ (0x0 << BXT_MIPI_TX_ESCLK_SHIFT(port))
+#define BXT_MIPI_TX_ESCLK_8XDIV_BY4(port) \
+ (0x1 << BXT_MIPI_TX_ESCLK_SHIFT(port))
+#define BXT_MIPI_TX_ESCLK_8XDIV_BY8(port) \
+ (0x2 << BXT_MIPI_TX_ESCLK_SHIFT(port))
+/* RX control divider to select actual RX clock output from 8x*/
+#define BXT_MIPI1_RX_ESCLK_SHIFT 19
+#define BXT_MIPI2_RX_ESCLK_SHIFT 3
+#define BXT_MIPI_RX_ESCLK_SHIFT(port) \
+ _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_SHIFT, \
+ BXT_MIPI2_RX_ESCLK_SHIFT)
+#define BXT_MIPI1_RX_ESCLK_FIXDIV_MASK (3 << 19)
+#define BXT_MIPI2_RX_ESCLK_FIXDIV_MASK (3 << 3)
+#define BXT_MIPI_RX_ESCLK_FIXDIV_MASK(port) \
+ (3 << BXT_MIPI_RX_ESCLK_SHIFT(port))
+#define BXT_MIPI_RX_ESCLK_8X_BY2(port) \
+ (1 << BXT_MIPI_RX_ESCLK_SHIFT(port))
+#define BXT_MIPI_RX_ESCLK_8X_BY3(port) \
+ (2 << BXT_MIPI_RX_ESCLK_SHIFT(port))
+#define BXT_MIPI_RX_ESCLK_8X_BY4(port) \
+ (3 << BXT_MIPI_RX_ESCLK_SHIFT(port))
+/* BXT-A WA: Always prog DPHY dividers to 00 */
+#define BXT_MIPI1_DPHY_DIV_SHIFT 16
+#define BXT_MIPI2_DPHY_DIV_SHIFT 0
+#define BXT_MIPI_DPHY_DIV_SHIFT(port) \
+ _MIPI_PORT(port, BXT_MIPI1_DPHY_DIV_SHIFT, \
+ BXT_MIPI2_DPHY_DIV_SHIFT)
+#define BXT_MIPI_1_DPHY_DIVIDER_MASK (3 << 16)
+#define BXT_MIPI_2_DPHY_DIVIDER_MASK (3 << 0)
+#define BXT_MIPI_DPHY_DIVIDER_MASK(port) \
+ (3 << BXT_MIPI_DPHY_DIV_SHIFT(port))
+
+/* BXT MIPI mode configure */
+#define _BXT_MIPIA_TRANS_HACTIVE 0x6B0F8
+#define _BXT_MIPIC_TRANS_HACTIVE 0x6B8F8
+#define BXT_MIPI_TRANS_HACTIVE(tc) _MIPI_PORT(tc, \
+ _BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE)
+
+#define _BXT_MIPIA_TRANS_VACTIVE 0x6B0FC
+#define _BXT_MIPIC_TRANS_VACTIVE 0x6B8FC
+#define BXT_MIPI_TRANS_VACTIVE(tc) _MIPI_PORT(tc, \
+ _BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE)
+
+#define _BXT_MIPIA_TRANS_VTOTAL 0x6B100
+#define _BXT_MIPIC_TRANS_VTOTAL 0x6B900
+#define BXT_MIPI_TRANS_VTOTAL(tc) _MIPI_PORT(tc, \
+ _BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL)
+
+#define BXT_DSI_PLL_CTL 0x161000
+#define BXT_DSI_PLL_PVD_RATIO_SHIFT 16
+#define BXT_DSI_PLL_PVD_RATIO_MASK (3 << BXT_DSI_PLL_PVD_RATIO_SHIFT)
+#define BXT_DSI_PLL_PVD_RATIO_1 (1 << BXT_DSI_PLL_PVD_RATIO_SHIFT)
+#define BXT_DSIC_16X_BY2 (1 << 10)
+#define BXT_DSIC_16X_BY3 (2 << 10)
+#define BXT_DSIC_16X_BY4 (3 << 10)
+#define BXT_DSIA_16X_BY2 (1 << 8)
+#define BXT_DSIA_16X_BY3 (2 << 8)
+#define BXT_DSIA_16X_BY4 (3 << 8)
+#define BXT_DSI_FREQ_SEL_SHIFT 8
+#define BXT_DSI_FREQ_SEL_MASK (0xF << BXT_DSI_FREQ_SEL_SHIFT)
+
+#define BXT_DSI_PLL_RATIO_MAX 0x7D
+#define BXT_DSI_PLL_RATIO_MIN 0x22
+#define BXT_DSI_PLL_RATIO_MASK 0xFF
+#define BXT_REF_CLOCK_KHZ 19500
+
+#define BXT_DSI_PLL_ENABLE 0x46080
+#define BXT_DSI_PLL_DO_ENABLE (1 << 31)
+#define BXT_DSI_PLL_LOCKED (1 << 30)
+
#define _MIPIA_PORT_CTRL (VLV_DISPLAY_BASE + 0x61190)
#define _MIPIC_PORT_CTRL (VLV_DISPLAY_BASE + 0x61700)
#define MIPI_PORT_CTRL(port) _MIPI_PORT(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL)
+
+ /* BXT port control */
+#define _BXT_MIPIA_PORT_CTRL 0x6B0C0
+#define _BXT_MIPIC_PORT_CTRL 0x6B8C0
+#define BXT_MIPI_PORT_CTRL(tc) _MIPI_PORT(tc, _BXT_MIPIA_PORT_CTRL, \
+ _BXT_MIPIC_PORT_CTRL)
+
#define DPI_ENABLE (1 << 31) /* A + C */
#define MIPIA_MIPI4DPHY_DELAY_COUNT_SHIFT 27
#define MIPIA_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 27)
#define VIRTUAL_CHANNEL_SHIFT 6
#define VIRTUAL_CHANNEL_MASK (3 << 6)
#define DATA_TYPE_SHIFT 0
-#define DATA_TYPE_MASK (3f << 0)
+#define DATA_TYPE_MASK (0x3f << 0)
/* data type values, see include/video/mipi_display.h */
#define _MIPIA_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb074)
#define READ_REQUEST_PRIORITY_HIGH (3 << 3)
#define RGB_FLIP_TO_BGR (1 << 2)
+#define BXT_PIPE_SELECT_MASK (7 << 7)
+#define BXT_PIPE_SELECT_C (2 << 7)
+#define BXT_PIPE_SELECT_B (1 << 7)
+#define BXT_PIPE_SELECT_A (0 << 7)
+
#define _MIPIA_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb108)
#define _MIPIC_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb908)
#define MIPI_DATA_ADDRESS(port) _MIPI_PORT(port, _MIPIA_DATA_ADDRESS, \
dev_priv->regfile.saveMI_ARB_STATE = I915_READ(MI_ARB_STATE);
/* Scratch space */
- for (i = 0; i < 16; i++) {
- dev_priv->regfile.saveSWF0[i] = I915_READ(SWF00 + (i << 2));
- dev_priv->regfile.saveSWF1[i] = I915_READ(SWF10 + (i << 2));
+ if (IS_GEN2(dev_priv) && IS_MOBILE(dev_priv)) {
+ for (i = 0; i < 7; i++) {
+ dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i));
+ dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i));
+ }
+ for (i = 0; i < 3; i++)
+ dev_priv->regfile.saveSWF3[i] = I915_READ(SWF3(i));
+ } else if (IS_GEN2(dev_priv)) {
+ for (i = 0; i < 7; i++)
+ dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i));
+ } else if (HAS_GMCH_DISPLAY(dev_priv)) {
+ for (i = 0; i < 16; i++) {
+ dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i));
+ dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i));
+ }
+ for (i = 0; i < 3; i++)
+ dev_priv->regfile.saveSWF3[i] = I915_READ(SWF3(i));
}
- for (i = 0; i < 3; i++)
- dev_priv->regfile.saveSWF2[i] = I915_READ(SWF30 + (i << 2));
mutex_unlock(&dev->struct_mutex);
/* Memory arbitration state */
I915_WRITE(MI_ARB_STATE, dev_priv->regfile.saveMI_ARB_STATE | 0xffff0000);
- for (i = 0; i < 16; i++) {
- I915_WRITE(SWF00 + (i << 2), dev_priv->regfile.saveSWF0[i]);
- I915_WRITE(SWF10 + (i << 2), dev_priv->regfile.saveSWF1[i]);
+ /* Scratch space */
+ if (IS_GEN2(dev_priv) && IS_MOBILE(dev_priv)) {
+ for (i = 0; i < 7; i++) {
+ I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]);
+ I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]);
+ }
+ for (i = 0; i < 3; i++)
+ I915_WRITE(SWF3(i), dev_priv->regfile.saveSWF3[i]);
+ } else if (IS_GEN2(dev_priv)) {
+ for (i = 0; i < 7; i++)
+ I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]);
+ } else if (HAS_GMCH_DISPLAY(dev_priv)) {
+ for (i = 0; i < 16; i++) {
+ I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]);
+ I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]);
+ }
+ for (i = 0; i < 3; i++)
+ I915_WRITE(SWF3(i), dev_priv->regfile.saveSWF3[i]);
}
- for (i = 0; i < 3; i++)
- I915_WRITE(SWF30 + (i << 2), dev_priv->regfile.saveSWF2[i]);
mutex_unlock(&dev->struct_mutex);
{
struct drm_i915_private *dev_priv = dev->dev_private;
u64 raw_time; /* 32b value may overflow during fixed point math */
- u64 units = 128ULL, div = 100000ULL, bias = 100ULL;
+ u64 units = 128ULL, div = 100000ULL;
u32 ret;
if (!intel_enable_rc6(dev))
/* On VLV and CHV, residency time is in CZ units rather than 1.28us */
if (IS_VALLEYVIEW(dev)) {
- u32 clk_reg, czcount_30ns;
-
- if (IS_CHERRYVIEW(dev))
- clk_reg = CHV_CLK_CTL1;
- else
- clk_reg = VLV_CLK_CTL2;
-
- czcount_30ns = I915_READ(clk_reg) >> CLK_CTL2_CZCOUNT_30NS_SHIFT;
-
- if (!czcount_30ns) {
- WARN(!czcount_30ns, "bogus CZ count value");
- ret = 0;
- goto out;
- }
-
- if (IS_CHERRYVIEW(dev) && czcount_30ns == 1) {
- /* Special case for 320Mhz */
- div = 10000000ULL;
- units = 3125ULL;
- } else {
- czcount_30ns += 1;
- div = 1000000ULL;
- units = DIV_ROUND_UP_ULL(30ULL * bias, czcount_30ns);
- }
+ units = 1;
+ div = dev_priv->czclk_freq;
if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
units <<= 8;
-
- div = div * bias;
+ } else if (IS_BROXTON(dev)) {
+ units = 1;
+ div = 1200; /* 833.33ns */
}
raw_time = I915_READ(reg) * units;
ret = DIV_ROUND_UP_ULL(raw_time, div);
-out:
intel_runtime_pm_put(dev_priv);
return ret;
}
/* pipe updates */
TRACE_EVENT(i915_pipe_update_start,
- TP_PROTO(struct intel_crtc *crtc, u32 min, u32 max),
- TP_ARGS(crtc, min, max),
+ TP_PROTO(struct intel_crtc *crtc),
+ TP_ARGS(crtc),
TP_STRUCT__entry(
__field(enum pipe, pipe)
__entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev,
crtc->pipe);
__entry->scanline = intel_get_crtc_scanline(crtc);
- __entry->min = min;
- __entry->max = max;
+ __entry->min = crtc->debug.min_vbl;
+ __entry->max = crtc->debug.max_vbl;
),
TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u",
);
TRACE_EVENT(i915_pipe_update_vblank_evaded,
- TP_PROTO(struct intel_crtc *crtc, u32 min, u32 max, u32 frame),
- TP_ARGS(crtc, min, max, frame),
+ TP_PROTO(struct intel_crtc *crtc),
+ TP_ARGS(crtc),
TP_STRUCT__entry(
__field(enum pipe, pipe)
TP_fast_assign(
__entry->pipe = crtc->pipe;
- __entry->frame = frame;
- __entry->scanline = intel_get_crtc_scanline(crtc);
- __entry->min = min;
- __entry->max = max;
+ __entry->frame = crtc->debug.start_vbl_count;
+ __entry->scanline = crtc->debug.scanline_start;
+ __entry->min = crtc->debug.min_vbl;
+ __entry->max = crtc->debug.max_vbl;
),
TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u",
);
TRACE_EVENT(i915_pipe_update_end,
- TP_PROTO(struct intel_crtc *crtc, u32 frame),
- TP_ARGS(crtc, frame),
+ TP_PROTO(struct intel_crtc *crtc, u32 frame, int scanline_end),
+ TP_ARGS(crtc, frame, scanline_end),
TP_STRUCT__entry(
__field(enum pipe, pipe)
TP_fast_assign(
__entry->pipe = crtc->pipe;
__entry->frame = frame;
- __entry->scanline = intel_get_crtc_scanline(crtc);
+ __entry->scanline = scanline_end;
),
TP_printk("pipe %c, frame=%u, scanline=%u",
TP_printk("obj=%p, size=%u", __entry->obj, __entry->size)
);
+TRACE_EVENT(i915_gem_shrink,
+ TP_PROTO(struct drm_i915_private *i915, unsigned long target, unsigned flags),
+ TP_ARGS(i915, target, flags),
+
+ TP_STRUCT__entry(
+ __field(int, dev)
+ __field(unsigned long, target)
+ __field(unsigned, flags)
+ ),
+
+ TP_fast_assign(
+ __entry->dev = i915->dev->primary->index;
+ __entry->target = target;
+ __entry->flags = flags;
+ ),
+
+ TP_printk("dev=%d, target=%lu, flags=%x",
+ __entry->dev, __entry->target, __entry->flags)
+);
+
TRACE_EVENT(i915_vma_bind,
TP_PROTO(struct i915_vma *vma, unsigned flags),
TP_ARGS(vma, flags),
TP_ARGS(vm, start, length, name)
);
-DECLARE_EVENT_CLASS(i915_page_table_entry,
- TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift),
- TP_ARGS(vm, pde, start, pde_shift),
+DECLARE_EVENT_CLASS(i915_px_entry,
+ TP_PROTO(struct i915_address_space *vm, u32 px, u64 start, u64 px_shift),
+ TP_ARGS(vm, px, start, px_shift),
TP_STRUCT__entry(
__field(struct i915_address_space *, vm)
- __field(u32, pde)
+ __field(u32, px)
__field(u64, start)
__field(u64, end)
),
TP_fast_assign(
__entry->vm = vm;
- __entry->pde = pde;
+ __entry->px = px;
__entry->start = start;
- __entry->end = ((start + (1ULL << pde_shift)) & ~((1ULL << pde_shift)-1)) - 1;
+ __entry->end = ((start + (1ULL << px_shift)) & ~((1ULL << px_shift)-1)) - 1;
),
TP_printk("vm=%p, pde=%d (0x%llx-0x%llx)",
- __entry->vm, __entry->pde, __entry->start, __entry->end)
+ __entry->vm, __entry->px, __entry->start, __entry->end)
);
-DEFINE_EVENT(i915_page_table_entry, i915_page_table_entry_alloc,
+DEFINE_EVENT(i915_px_entry, i915_page_table_entry_alloc,
TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift),
TP_ARGS(vm, pde, start, pde_shift)
);
+DEFINE_EVENT_PRINT(i915_px_entry, i915_page_directory_entry_alloc,
+ TP_PROTO(struct i915_address_space *vm, u32 pdpe, u64 start, u64 pdpe_shift),
+ TP_ARGS(vm, pdpe, start, pdpe_shift),
+
+ TP_printk("vm=%p, pdpe=%d (0x%llx-0x%llx)",
+ __entry->vm, __entry->px, __entry->start, __entry->end)
+);
+
+DEFINE_EVENT_PRINT(i915_px_entry, i915_page_directory_pointer_entry_alloc,
+ TP_PROTO(struct i915_address_space *vm, u32 pml4e, u64 start, u64 pml4e_shift),
+ TP_ARGS(vm, pml4e, start, pml4e_shift),
+
+ TP_printk("vm=%p, pml4e=%d (0x%llx-0x%llx)",
+ __entry->vm, __entry->px, __entry->start, __entry->end)
+);
+
/* Avoid extra math because we only support two sizes. The format is defined by
* bitmap_scnprintf. Each 32 bits is 8 HEX digits followed by comma */
#define TRACE_PT_SIZE(bits) \
#define INTEL_VGT_IF_VERSION \
INTEL_VGT_IF_VERSION_ENCODE(VGT_VERSION_MAJOR, VGT_VERSION_MINOR)
+/*
+ * notifications from guest to vgpu device model
+ */
+enum vgt_g2v_type {
+ VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE = 2,
+ VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY,
+ VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE,
+ VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY,
+ VGT_G2V_EXECLIST_CONTEXT_CREATE,
+ VGT_G2V_EXECLIST_CONTEXT_DESTROY,
+ VGT_G2V_MAX,
+};
+
struct vgt_if {
uint64_t magic; /* VGT_MAGIC */
uint16_t version_major;
uint32_t rsv3[0x200 - 24]; /* pad to half page */
/*
* The bottom half page is for response from Gfx driver to hypervisor.
- * Set to reserved fields temporarily by now.
*/
uint32_t rsv4;
uint32_t display_ready; /* ready for display owner switch */
- uint32_t rsv5[0x200 - 2]; /* pad to one page */
+
+ uint32_t rsv5[4];
+
+ uint32_t g2v_notify;
+ uint32_t rsv6[7];
+
+ uint32_t pdp0_lo;
+ uint32_t pdp0_hi;
+ uint32_t pdp1_lo;
+ uint32_t pdp1_hi;
+ uint32_t pdp2_lo;
+ uint32_t pdp2_hi;
+ uint32_t pdp3_lo;
+ uint32_t pdp3_hi;
+
+ uint32_t execlist_context_descriptor_lo;
+ uint32_t execlist_context_descriptor_hi;
+
+ uint32_t rsv7[0x200 - 24]; /* pad to one page */
} __packed;
#define vgtif_reg(x) \
*/
#include <linux/pci.h>
#include <linux/acpi.h>
-#include <linux/vga_switcheroo.h>
#include <drm/drmP.h>
#include "i915_drv.h"
if (vga_count == 2 && has_dsm) {
acpi_get_name(intel_dsm_priv.dhandle, ACPI_FULL_PATHNAME, &buffer);
- DRM_DEBUG_DRIVER("VGA switcheroo: detected DSM switching method %s handle\n",
+ DRM_DEBUG_DRIVER("vga_switcheroo: detected DSM switching method %s handle\n",
acpi_method_name);
return true;
}
struct drm_crtc_state *
intel_crtc_duplicate_state(struct drm_crtc *crtc)
{
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_crtc_state *crtc_state;
- if (WARN_ON(!intel_crtc->config))
- crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
- else
- crtc_state = kmemdup(intel_crtc->config,
- sizeof(*intel_crtc->config), GFP_KERNEL);
-
+ crtc_state = kmemdup(crtc->state, sizeof(*crtc_state), GFP_KERNEL);
if (!crtc_state)
return NULL;
__drm_atomic_helper_crtc_duplicate_state(crtc, &crtc_state->base);
- crtc_state->base.crtc = crtc;
+ crtc_state->update_pipe = false;
return &crtc_state->base;
}
int i, j;
num_scalers_need = hweight32(scaler_state->scaler_users);
- DRM_DEBUG_KMS("crtc_state = %p need = %d avail = %d scaler_users = 0x%x\n",
- crtc_state, num_scalers_need, intel_crtc->num_scalers,
- scaler_state->scaler_users);
/*
* High level flow:
struct drm_plane_state *state;
struct intel_plane_state *intel_state;
- if (WARN_ON(!plane->state))
- intel_state = intel_create_plane_state(plane);
- else
- intel_state = kmemdup(plane->state, sizeof(*intel_state),
- GFP_KERNEL);
+ intel_state = kmemdup(plane->state, sizeof(*intel_state), GFP_KERNEL);
if (!intel_state)
return NULL;
* co-operation between the graphics and audio drivers is handled via audio
* related registers. (The notable exception is the power management, not
* covered here.)
+ *
+ * The struct i915_audio_component is used to interact between the graphics
+ * and audio drivers. The struct i915_audio_component_ops *ops in it is
+ * defined in graphics driver and called in audio driver. The
+ * struct i915_audio_component_audio_ops *audio_ops is called from i915 driver.
*/
static const struct {
int clock;
u32 config;
} hdmi_audio_clock[] = {
- { DIV_ROUND_UP(25200 * 1000, 1001), AUD_CONFIG_PIXEL_CLOCK_HDMI_25175 },
+ { 25175, AUD_CONFIG_PIXEL_CLOCK_HDMI_25175 },
{ 25200, AUD_CONFIG_PIXEL_CLOCK_HDMI_25200 }, /* default per bspec */
{ 27000, AUD_CONFIG_PIXEL_CLOCK_HDMI_27000 },
- { 27000 * 1001 / 1000, AUD_CONFIG_PIXEL_CLOCK_HDMI_27027 },
+ { 27027, AUD_CONFIG_PIXEL_CLOCK_HDMI_27027 },
{ 54000, AUD_CONFIG_PIXEL_CLOCK_HDMI_54000 },
- { 54000 * 1001 / 1000, AUD_CONFIG_PIXEL_CLOCK_HDMI_54054 },
- { DIV_ROUND_UP(74250 * 1000, 1001), AUD_CONFIG_PIXEL_CLOCK_HDMI_74176 },
+ { 54054, AUD_CONFIG_PIXEL_CLOCK_HDMI_54054 },
+ { 74176, AUD_CONFIG_PIXEL_CLOCK_HDMI_74176 },
{ 74250, AUD_CONFIG_PIXEL_CLOCK_HDMI_74250 },
- { DIV_ROUND_UP(148500 * 1000, 1001), AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 },
+ { 148352, AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 },
{ 148500, AUD_CONFIG_PIXEL_CLOCK_HDMI_148500 },
};
+/* HDMI N/CTS table */
+#define TMDS_297M 297000
+#define TMDS_296M 296703
+static const struct {
+ int sample_rate;
+ int clock;
+ int n;
+ int cts;
+} aud_ncts[] = {
+ { 44100, TMDS_296M, 4459, 234375 },
+ { 44100, TMDS_297M, 4704, 247500 },
+ { 48000, TMDS_296M, 5824, 281250 },
+ { 48000, TMDS_297M, 5120, 247500 },
+ { 32000, TMDS_296M, 5824, 421875 },
+ { 32000, TMDS_297M, 3072, 222750 },
+ { 88200, TMDS_296M, 8918, 234375 },
+ { 88200, TMDS_297M, 9408, 247500 },
+ { 96000, TMDS_296M, 11648, 281250 },
+ { 96000, TMDS_297M, 10240, 247500 },
+ { 176400, TMDS_296M, 17836, 234375 },
+ { 176400, TMDS_297M, 18816, 247500 },
+ { 192000, TMDS_296M, 23296, 281250 },
+ { 192000, TMDS_297M, 20480, 247500 },
+};
+
/* get AUD_CONFIG_PIXEL_CLOCK_HDMI_* value for mode */
-static u32 audio_config_hdmi_pixel_clock(struct drm_display_mode *mode)
+static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted_mode)
{
int i;
for (i = 0; i < ARRAY_SIZE(hdmi_audio_clock); i++) {
- if (mode->clock == hdmi_audio_clock[i].clock)
+ if (adjusted_mode->crtc_clock == hdmi_audio_clock[i].clock)
break;
}
if (i == ARRAY_SIZE(hdmi_audio_clock)) {
- DRM_DEBUG_KMS("HDMI audio pixel clock setting for %d not found, falling back to defaults\n", mode->clock);
+ DRM_DEBUG_KMS("HDMI audio pixel clock setting for %d not found, falling back to defaults\n",
+ adjusted_mode->crtc_clock);
i = 1;
}
return hdmi_audio_clock[i].config;
}
+static int audio_config_get_n(const struct drm_display_mode *mode, int rate)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(aud_ncts); i++) {
+ if ((rate == aud_ncts[i].sample_rate) &&
+ (mode->clock == aud_ncts[i].clock)) {
+ return aud_ncts[i].n;
+ }
+ }
+ return 0;
+}
+
+static uint32_t audio_config_setup_n_reg(int n, uint32_t val)
+{
+ int n_low, n_up;
+ uint32_t tmp = val;
+
+ n_low = n & 0xfff;
+ n_up = (n >> 12) & 0xff;
+ tmp &= ~(AUD_CONFIG_UPPER_N_MASK | AUD_CONFIG_LOWER_N_MASK);
+ tmp |= ((n_up << AUD_CONFIG_UPPER_N_SHIFT) |
+ (n_low << AUD_CONFIG_LOWER_N_SHIFT) |
+ AUD_CONFIG_N_PROG_ENABLE);
+ return tmp;
+}
+
+/* check whether N/CTS/M need be set manually */
+static bool audio_rate_need_prog(struct intel_crtc *crtc,
+ const struct drm_display_mode *mode)
+{
+ if (((mode->clock == TMDS_297M) ||
+ (mode->clock == TMDS_296M)) &&
+ intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
+ return true;
+ else
+ return false;
+}
+
static bool intel_eld_uptodate(struct drm_connector *connector,
int reg_eldv, uint32_t bits_eldv,
int reg_elda, uint32_t bits_elda,
static void g4x_audio_codec_enable(struct drm_connector *connector,
struct intel_encoder *encoder,
- struct drm_display_mode *mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = connector->dev->dev_private;
uint8_t *eld = connector->eld;
DRM_DEBUG_KMS("Disable audio codec on pipe %c\n", pipe_name(pipe));
+ mutex_lock(&dev_priv->av_mutex);
+
/* Disable timestamps */
tmp = I915_READ(HSW_AUD_CFG(pipe));
tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
tmp &= ~AUDIO_ELD_VALID(pipe);
tmp &= ~AUDIO_OUTPUT_ENABLE(pipe);
I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
+
+ mutex_unlock(&dev_priv->av_mutex);
}
static void hsw_audio_codec_enable(struct drm_connector *connector,
struct intel_encoder *encoder,
- struct drm_display_mode *mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = connector->dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
enum pipe pipe = intel_crtc->pipe;
+ struct i915_audio_component *acomp = dev_priv->audio_component;
const uint8_t *eld = connector->eld;
+ struct intel_digital_port *intel_dig_port =
+ enc_to_dig_port(&encoder->base);
+ enum port port = intel_dig_port->port;
uint32_t tmp;
int len, i;
+ int n, rate;
DRM_DEBUG_KMS("Enable audio codec on pipe %c, %u bytes ELD\n",
pipe_name(pipe), drm_eld_size(eld));
+ mutex_lock(&dev_priv->av_mutex);
+
/* Enable audio presence detect, invalidate ELD */
tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD);
tmp |= AUDIO_OUTPUT_ENABLE(pipe);
/* Enable timestamps */
tmp = I915_READ(HSW_AUD_CFG(pipe));
tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
- tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
tmp |= AUD_CONFIG_N_VALUE_INDEX;
else
- tmp |= audio_config_hdmi_pixel_clock(mode);
+ tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
+
+ tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
+ if (audio_rate_need_prog(intel_crtc, adjusted_mode)) {
+ if (!acomp)
+ rate = 0;
+ else if (port >= PORT_A && port <= PORT_E)
+ rate = acomp->aud_sample_rate[port];
+ else {
+ DRM_ERROR("invalid port: %d\n", port);
+ rate = 0;
+ }
+ n = audio_config_get_n(adjusted_mode, rate);
+ if (n != 0)
+ tmp = audio_config_setup_n_reg(n, tmp);
+ else
+ DRM_DEBUG_KMS("no suitable N value is found\n");
+ }
+
I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+
+ mutex_unlock(&dev_priv->av_mutex);
}
static void ilk_audio_codec_disable(struct intel_encoder *encoder)
static void ilk_audio_codec_enable(struct drm_connector *connector,
struct intel_encoder *encoder,
- struct drm_display_mode *mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = connector->dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
tmp |= AUD_CONFIG_N_VALUE_INDEX;
else
- tmp |= audio_config_hdmi_pixel_clock(mode);
+ tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
I915_WRITE(aud_config, tmp);
}
{
struct drm_encoder *encoder = &intel_encoder->base;
struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
- struct drm_display_mode *mode = &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
struct drm_connector *connector;
struct drm_device *dev = encoder->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
enum port port = intel_dig_port->port;
- connector = drm_select_eld(encoder, mode);
+ connector = drm_select_eld(encoder);
if (!connector)
return;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
connector->eld[5] |= (1 << 2);
- connector->eld[6] = drm_av_sync_delay(connector, mode) / 2;
+ connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
if (dev_priv->display.audio_codec_enable)
- dev_priv->display.audio_codec_enable(connector, intel_encoder, mode);
+ dev_priv->display.audio_codec_enable(connector, intel_encoder,
+ adjusted_mode);
if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port);
/**
* intel_audio_codec_disable - Disable the audio codec for HD audio
- * @encoder: encoder on which to disable audio
+ * @intel_encoder: encoder on which to disable audio
*
* The disable sequences must be performed before disabling the transcoder or
* port.
return ret;
}
+static int i915_audio_component_sync_audio_rate(struct device *dev,
+ int port, int rate)
+{
+ struct drm_i915_private *dev_priv = dev_to_i915(dev);
+ struct drm_device *drm_dev = dev_priv->dev;
+ struct intel_encoder *intel_encoder;
+ struct intel_digital_port *intel_dig_port;
+ struct intel_crtc *crtc;
+ struct drm_display_mode *mode;
+ struct i915_audio_component *acomp = dev_priv->audio_component;
+ enum pipe pipe = -1;
+ u32 tmp;
+ int n;
+
+ /* HSW, BDW SKL need this fix */
+ if (!IS_SKYLAKE(dev_priv) &&
+ !IS_BROADWELL(dev_priv) &&
+ !IS_HASWELL(dev_priv))
+ return 0;
+
+ mutex_lock(&dev_priv->av_mutex);
+ /* 1. get the pipe */
+ for_each_intel_encoder(drm_dev, intel_encoder) {
+ if (intel_encoder->type != INTEL_OUTPUT_HDMI)
+ continue;
+ intel_dig_port = enc_to_dig_port(&intel_encoder->base);
+ if (port == intel_dig_port->port) {
+ crtc = to_intel_crtc(intel_encoder->base.crtc);
+ if (!crtc) {
+ DRM_DEBUG_KMS("%s: crtc is NULL\n", __func__);
+ continue;
+ }
+ pipe = crtc->pipe;
+ break;
+ }
+ }
+
+ if (pipe == INVALID_PIPE) {
+ DRM_DEBUG_KMS("no pipe for the port %c\n", port_name(port));
+ mutex_unlock(&dev_priv->av_mutex);
+ return -ENODEV;
+ }
+ DRM_DEBUG_KMS("pipe %c connects port %c\n",
+ pipe_name(pipe), port_name(port));
+ mode = &crtc->config->base.adjusted_mode;
+
+ /* port must be valid now, otherwise the pipe will be invalid */
+ acomp->aud_sample_rate[port] = rate;
+
+ /* 2. check whether to set the N/CTS/M manually or not */
+ if (!audio_rate_need_prog(crtc, mode)) {
+ tmp = I915_READ(HSW_AUD_CFG(pipe));
+ tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
+ I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+ mutex_unlock(&dev_priv->av_mutex);
+ return 0;
+ }
+
+ n = audio_config_get_n(mode, rate);
+ if (n == 0) {
+ DRM_DEBUG_KMS("Using automatic mode for N value on port %c\n",
+ port_name(port));
+ tmp = I915_READ(HSW_AUD_CFG(pipe));
+ tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
+ I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+ mutex_unlock(&dev_priv->av_mutex);
+ return 0;
+ }
+
+ /* 3. set the N/CTS/M */
+ tmp = I915_READ(HSW_AUD_CFG(pipe));
+ tmp = audio_config_setup_n_reg(n, tmp);
+ I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+
+ mutex_unlock(&dev_priv->av_mutex);
+ return 0;
+}
+
static const struct i915_audio_component_ops i915_audio_component_ops = {
.owner = THIS_MODULE,
.get_power = i915_audio_component_get_power,
.put_power = i915_audio_component_put_power,
.codec_wake_override = i915_audio_component_codec_wake_override,
.get_cdclk_freq = i915_audio_component_get_cdclk_freq,
+ .sync_audio_rate = i915_audio_component_sync_audio_rate,
};
static int i915_audio_component_bind(struct device *i915_dev,
{
struct i915_audio_component *acomp = data;
struct drm_i915_private *dev_priv = dev_to_i915(i915_dev);
+ int i;
if (WARN_ON(acomp->ops || acomp->dev))
return -EEXIST;
drm_modeset_lock_all(dev_priv->dev);
acomp->ops = &i915_audio_component_ops;
acomp->dev = i915_dev;
+ BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS);
+ for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++)
+ acomp->aud_sample_rate[i] = 0;
dev_priv->audio_component = acomp;
drm_modeset_unlock_all(dev_priv->dev);
const struct bdb_header *bdb = _bdb;
const u8 *base = _bdb;
int index = 0;
- u16 total, current_size;
+ u32 total, current_size;
u8 current_id;
/* skip to first section */
current_size = *((const u16 *)(base + index));
index += 2;
+ /* The MIPI Sequence Block v3+ has a separate size field. */
+ if (current_id == BDB_MIPI_SEQUENCE && *(base + index) >= 3)
+ current_size = *((const u32 *)(base + index + 1));
+
if (index + current_size > total)
return NULL;
return;
}
+ /* Fail gracefully for forward incompatible sequence block. */
+ if (sequence->version >= 3) {
+ DRM_ERROR("Unable to parse MIPI Sequence Block v3+\n");
+ return;
+ }
+
DRM_DEBUG_DRIVER("Found MIPI sequence block\n");
block_size = get_blocksize(sequence);
{ }
};
-static const struct bdb_header *validate_vbt(const void __iomem *_base,
+static const struct bdb_header *validate_vbt(const void *base,
size_t size,
- const void __iomem *_vbt,
+ const void *_vbt,
const char *source)
{
- /*
- * This is the one place where we explicitly discard the address space
- * (__iomem) of the BIOS/VBT. (And this will cause a sparse complaint.)
- * From now on everything is based on 'base', and treated as regular
- * memory.
- */
- const void *base = (const void *) _base;
- size_t offset = _vbt - _base;
- const struct vbt_header *vbt = base + offset;
+ size_t offset = _vbt - base;
+ const struct vbt_header *vbt = _vbt;
const struct bdb_header *bdb;
if (offset + sizeof(struct vbt_header) > size) {
/* Scour memory looking for the VBT signature. */
for (i = 0; i + 4 < size; i++) {
if (ioread32(bios + i) == *((const u32 *) "$VBT")) {
- bdb = validate_vbt(bios, size, bios + i, "PCI ROM");
+ /*
+ * This is the one place where we explicitly discard the
+ * address space (__iomem) of the BIOS/VBT. From now on
+ * everything is based on 'base', and treated as regular
+ * memory.
+ */
+ void *_bios = (void __force *) bios;
+
+ bdb = validate_vbt(_bios, size, _bios + i, "PCI ROM");
break;
}
}
return 0;
}
-
-/* Ensure that vital registers have been initialised, even if the BIOS
- * is absent or just failing to do its job.
- */
-void intel_setup_bios(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- /* Set the Panel Power On/Off timings if uninitialized. */
- if (!HAS_PCH_SPLIT(dev) &&
- I915_READ(PP_ON_DELAYS) == 0 && I915_READ(PP_OFF_DELAYS) == 0) {
- /* Set T2 to 40ms and T5 to 200ms */
- I915_WRITE(PP_ON_DELAYS, 0x019007d0);
-
- /* Set T3 to 35ms and Tx to 200ms */
- I915_WRITE(PP_OFF_DELAYS, 0x015e07d0);
- }
-}
struct psr_table psr_table[16];
} __packed;
-void intel_setup_bios(struct drm_device *dev);
int intel_parse_bios(struct drm_device *dev);
/*
*/
#define DEVICE_TYPE_eDP_BITS \
(DEVICE_TYPE_INTERNAL_CONNECTOR | \
- DEVICE_TYPE_NOT_HDMI_OUTPUT | \
DEVICE_TYPE_MIPI_OUTPUT | \
DEVICE_TYPE_COMPOSITE_OUTPUT | \
DEVICE_TYPE_DUAL_CHANNEL | \
DEVICE_TYPE_TMDS_DVI_SIGNALING | \
DEVICE_TYPE_VIDEO_SIGNALING | \
DEVICE_TYPE_DISPLAYPORT_OUTPUT | \
- DEVICE_TYPE_DIGITAL_OUTPUT | \
DEVICE_TYPE_ANALOG_OUTPUT)
/* define the DVO port for HDMI output type */
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crt *crt = intel_encoder_to_crt(encoder);
struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
- struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
u32 adpa;
if (INTEL_INFO(dev)->gen >= 5)
{
struct drm_device *dev = connector->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- u32 hotplug_en, orig, stat;
+ u32 stat;
bool ret = false;
int i, tries = 0;
tries = 2;
else
tries = 1;
- hotplug_en = orig = I915_READ(PORT_HOTPLUG_EN);
- hotplug_en |= CRT_HOTPLUG_FORCE_DETECT;
for (i = 0; i < tries ; i++) {
/* turn on the FORCE_DETECT */
- I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
+ i915_hotplug_interrupt_update(dev_priv,
+ CRT_HOTPLUG_FORCE_DETECT,
+ CRT_HOTPLUG_FORCE_DETECT);
/* wait for FORCE_DETECT to go off */
if (wait_for((I915_READ(PORT_HOTPLUG_EN) &
CRT_HOTPLUG_FORCE_DETECT) == 0,
/* clear the interrupt we just generated, if any */
I915_WRITE(PORT_HOTPLUG_STAT, CRT_HOTPLUG_INT_STATUS);
- /* and put the bits back */
- I915_WRITE(PORT_HOTPLUG_EN, orig);
+ i915_hotplug_interrupt_update(dev_priv, CRT_HOTPLUG_FORCE_DETECT, 0);
return ret;
}
u32 fdi_config = FDI_RX_POLARITY_REVERSED_LPT |
FDI_RX_LINK_REVERSAL_OVERRIDE;
- dev_priv->fdi_rx_config = I915_READ(_FDI_RXA_CTL) & fdi_config;
+ dev_priv->fdi_rx_config = I915_READ(FDI_RX_CTL(PIPE_A)) & fdi_config;
}
intel_crt_reset(connector);
*/
#define I915_CSR_SKL "i915/skl_dmc_ver1.bin"
+#define I915_CSR_BXT "i915/bxt_dmc_ver1.bin"
MODULE_FIRMWARE(I915_CSR_SKL);
+MODULE_FIRMWARE(I915_CSR_BXT);
/*
* SKL CSR registers for DC5 and DC6
*/
-#define CSR_PROGRAM_BASE 0x80000
+#define CSR_PROGRAM(i) (0x80000 + (i) * 4)
#define CSR_SSP_BASE_ADDR_GEN9 0x00002FC0
#define CSR_HTP_ADDR_SKL 0x00500034
#define CSR_SSP_BASE 0x8F074
{'G', '0'}, {'H', '0'}, {'I', '0'}
};
+static struct stepping_info bxt_stepping_info[] = {
+ {'A', '0'}, {'A', '1'}, {'A', '2'},
+ {'B', '0'}, {'B', '1'}, {'B', '2'}
+};
+
static char intel_get_stepping(struct drm_device *dev)
{
if (IS_SKYLAKE(dev) && (dev->pdev->revision <
ARRAY_SIZE(skl_stepping_info)))
return skl_stepping_info[dev->pdev->revision].stepping;
+ else if (IS_BROXTON(dev) && (dev->pdev->revision <
+ ARRAY_SIZE(bxt_stepping_info)))
+ return bxt_stepping_info[dev->pdev->revision].stepping;
else
return -ENODATA;
}
if (IS_SKYLAKE(dev) && (dev->pdev->revision <
ARRAY_SIZE(skl_stepping_info)))
return skl_stepping_info[dev->pdev->revision].substepping;
+ else if (IS_BROXTON(dev) && (dev->pdev->revision <
+ ARRAY_SIZE(bxt_stepping_info)))
+ return bxt_stepping_info[dev->pdev->revision].substepping;
else
return -ENODATA;
}
return;
}
+ /*
+ * FIXME: Firmware gets lost on S3/S4, but not when entering system
+ * standby or suspend-to-idle (which is just like forced runtime pm).
+ * Unfortunately the ACPI subsystem doesn't yet give us a way to
+ * differentiate this, hence figure it out with this hack.
+ */
+ if (I915_READ(CSR_PROGRAM(0)))
+ return;
+
mutex_lock(&dev_priv->csr_lock);
fw_size = dev_priv->csr.dmc_fw_size;
for (i = 0; i < fw_size; i++)
- I915_WRITE(CSR_PROGRAM_BASE + i * 4,
- payload[i]);
+ I915_WRITE(CSR_PROGRAM(i), payload[i]);
for (i = 0; i < dev_priv->csr.mmio_count; i++) {
I915_WRITE(dev_priv->csr.mmioaddr[i],
if (IS_SKYLAKE(dev))
csr->fw_path = I915_CSR_SKL;
+ else if (IS_BROXTON(dev_priv))
+ csr->fw_path = I915_CSR_BXT;
else {
DRM_ERROR("Unexpected: no known CSR firmware for platform\n");
intel_csr_load_status_set(dev_priv, FW_FAILED);
void assert_csr_loaded(struct drm_i915_private *dev_priv)
{
- WARN(intel_csr_load_status_get(dev_priv) != FW_LOADED,
- "CSR is not loaded.\n");
- WARN(!I915_READ(CSR_PROGRAM_BASE),
- "CSR program storage start is NULL\n");
- WARN(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
- WARN(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
+ WARN_ONCE(intel_csr_load_status_get(dev_priv) != FW_LOADED,
+ "CSR is not loaded.\n");
+ WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
+ "CSR program storage start is NULL\n");
+ WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
+ WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
}
bool default_index; /* true if the entry represents default value */
};
-/* BSpec does not define separate vswing/pre-emphasis values for eDP.
- * Using DP values for eDP as well.
- */
static const struct bxt_ddi_buf_trans bxt_ddi_translations_dp[] = {
/* Idx NT mV diff db */
{ 52, 0x9A, 0, 128, true }, /* 0: 400 0 */
{ 154, 0x9A, 1, 128, false }, /* 9: 1200 0 */
};
+static const struct bxt_ddi_buf_trans bxt_ddi_translations_edp[] = {
+ /* Idx NT mV diff db */
+ { 26, 0, 0, 128, false }, /* 0: 200 0 */
+ { 38, 0, 0, 112, false }, /* 1: 200 1.5 */
+ { 48, 0, 0, 96, false }, /* 2: 200 4 */
+ { 54, 0, 0, 69, false }, /* 3: 200 6 */
+ { 32, 0, 0, 128, false }, /* 4: 250 0 */
+ { 48, 0, 0, 104, false }, /* 5: 250 1.5 */
+ { 54, 0, 0, 85, false }, /* 6: 250 4 */
+ { 43, 0, 0, 128, false }, /* 7: 300 0 */
+ { 54, 0, 0, 101, false }, /* 8: 300 1.5 */
+ { 48, 0, 0, 128, false }, /* 9: 300 0 */
+};
+
/* BSpec has 2 recommended values - entries 0 and 8.
* Using the entry with higher vswing.
*/
enum port *port)
{
struct drm_encoder *encoder = &intel_encoder->base;
- int type = intel_encoder->type;
- if (type == INTEL_OUTPUT_DP_MST) {
+ switch (intel_encoder->type) {
+ case INTEL_OUTPUT_DP_MST:
*dig_port = enc_to_mst(encoder)->primary;
*port = (*dig_port)->port;
- } else if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP ||
- type == INTEL_OUTPUT_HDMI || type == INTEL_OUTPUT_UNKNOWN) {
+ break;
+ case INTEL_OUTPUT_DISPLAYPORT:
+ case INTEL_OUTPUT_EDP:
+ case INTEL_OUTPUT_HDMI:
+ case INTEL_OUTPUT_UNKNOWN:
*dig_port = enc_to_dig_port(encoder);
*port = (*dig_port)->port;
- } else if (type == INTEL_OUTPUT_ANALOG) {
+ break;
+ case INTEL_OUTPUT_ANALOG:
*dig_port = NULL;
*port = PORT_E;
- } else {
- DRM_ERROR("Invalid DDI encoder type %d\n", type);
- BUG();
+ break;
+ default:
+ WARN(1, "Invalid DDI encoder type %d\n", intel_encoder->type);
+ break;
}
}
bool supports_hdmi)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- u32 reg;
u32 iboost_bit = 0;
int i, n_hdmi_entries, n_dp_entries, n_edp_entries, hdmi_default_entry,
size;
BUG();
}
- for (i = 0, reg = DDI_BUF_TRANS(port); i < size; i++) {
- I915_WRITE(reg, ddi_translations[i].trans1 | iboost_bit);
- reg += 4;
- I915_WRITE(reg, ddi_translations[i].trans2);
- reg += 4;
+ for (i = 0; i < size; i++) {
+ I915_WRITE(DDI_BUF_TRANS_LO(port, i),
+ ddi_translations[i].trans1 | iboost_bit);
+ I915_WRITE(DDI_BUF_TRANS_HI(port, i),
+ ddi_translations[i].trans2);
}
if (!supports_hdmi)
hdmi_level = hdmi_default_entry;
/* Entry 9 is for HDMI: */
- I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans1 | iboost_bit);
- reg += 4;
- I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans2);
- reg += 4;
+ I915_WRITE(DDI_BUF_TRANS_LO(port, i),
+ ddi_translations_hdmi[hdmi_level].trans1 | iboost_bit);
+ I915_WRITE(DDI_BUF_TRANS_HI(port, i),
+ ddi_translations_hdmi[hdmi_level].trans2);
}
/* Program DDI buffers translations for DP. By default, program ports A-D in DP
enum port port;
bool supports_hdmi;
- ddi_get_encoder_port(intel_encoder, &intel_dig_port, &port);
+ if (intel_encoder->type == INTEL_OUTPUT_DSI)
+ continue;
+ ddi_get_encoder_port(intel_encoder, &intel_dig_port, &port);
if (visited[port])
continue;
*
* WaFDIAutoLinkSetTimingOverrride:hsw
*/
- I915_WRITE(_FDI_RXA_MISC, FDI_RX_PWRDN_LANE1_VAL(2) |
+ I915_WRITE(FDI_RX_MISC(PIPE_A), FDI_RX_PWRDN_LANE1_VAL(2) |
FDI_RX_PWRDN_LANE0_VAL(2) |
FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
rx_ctl_val = dev_priv->fdi_rx_config | FDI_RX_ENHANCE_FRAME_ENABLE |
FDI_RX_PLL_ENABLE |
FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
- I915_WRITE(_FDI_RXA_CTL, rx_ctl_val);
- POSTING_READ(_FDI_RXA_CTL);
+ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val);
+ POSTING_READ(FDI_RX_CTL(PIPE_A));
udelay(220);
/* Switch from Rawclk to PCDclk */
rx_ctl_val |= FDI_PCDCLK;
- I915_WRITE(_FDI_RXA_CTL, rx_ctl_val);
+ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val);
/* Configure Port Clock Select */
I915_WRITE(PORT_CLK_SEL(PORT_E), intel_crtc->config->ddi_pll_sel);
udelay(600);
/* Program PCH FDI Receiver TU */
- I915_WRITE(_FDI_RXA_TUSIZE1, TU_SIZE(64));
+ I915_WRITE(FDI_RX_TUSIZE1(PIPE_A), TU_SIZE(64));
/* Enable PCH FDI Receiver with auto-training */
rx_ctl_val |= FDI_RX_ENABLE | FDI_LINK_TRAIN_AUTO;
- I915_WRITE(_FDI_RXA_CTL, rx_ctl_val);
- POSTING_READ(_FDI_RXA_CTL);
+ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val);
+ POSTING_READ(FDI_RX_CTL(PIPE_A));
/* Wait for FDI receiver lane calibration */
udelay(30);
/* Unset FDI_RX_MISC pwrdn lanes */
- temp = I915_READ(_FDI_RXA_MISC);
+ temp = I915_READ(FDI_RX_MISC(PIPE_A));
temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
- I915_WRITE(_FDI_RXA_MISC, temp);
- POSTING_READ(_FDI_RXA_MISC);
+ I915_WRITE(FDI_RX_MISC(PIPE_A), temp);
+ POSTING_READ(FDI_RX_MISC(PIPE_A));
/* Wait for FDI auto training time */
udelay(5);
intel_wait_ddi_buf_idle(dev_priv, PORT_E);
rx_ctl_val &= ~FDI_RX_ENABLE;
- I915_WRITE(_FDI_RXA_CTL, rx_ctl_val);
- POSTING_READ(_FDI_RXA_CTL);
+ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val);
+ POSTING_READ(FDI_RX_CTL(PIPE_A));
/* Reset FDI_RX_MISC pwrdn lanes */
- temp = I915_READ(_FDI_RXA_MISC);
+ temp = I915_READ(FDI_RX_MISC(PIPE_A));
temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
temp |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2);
- I915_WRITE(_FDI_RXA_MISC, temp);
- POSTING_READ(_FDI_RXA_MISC);
+ I915_WRITE(FDI_RX_MISC(PIPE_A), temp);
+ POSTING_READ(FDI_RX_MISC(PIPE_A));
}
DRM_ERROR("FDI link training failed!\n");
intel_dp->DP = intel_dig_port->saved_port_bits |
DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0);
intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count);
-
}
static struct intel_encoder *
uint32_t cfgcr1_val, cfgcr2_val;
uint32_t p0, p1, p2, dco_freq;
- cfgcr1_reg = GET_CFG_CR1_REG(dpll);
- cfgcr2_reg = GET_CFG_CR2_REG(dpll);
+ cfgcr1_reg = DPLL_CFGCR1(dpll);
+ cfgcr2_reg = DPLL_CFGCR2(dpll);
cfgcr1_val = I915_READ(cfgcr1_reg);
cfgcr2_val = I915_READ(cfgcr2_reg);
static bool
hsw_ddi_pll_select(struct intel_crtc *intel_crtc,
struct intel_crtc_state *crtc_state,
- struct intel_encoder *intel_encoder,
- int clock)
+ struct intel_encoder *intel_encoder)
{
+ int clock = crtc_state->port_clock;
+
if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
struct intel_shared_dpll *pll;
uint32_t val;
static bool
skl_ddi_pll_select(struct intel_crtc *intel_crtc,
struct intel_crtc_state *crtc_state,
- struct intel_encoder *intel_encoder,
- int clock)
+ struct intel_encoder *intel_encoder)
{
struct intel_shared_dpll *pll;
uint32_t ctrl1, cfgcr1, cfgcr2;
+ int clock = crtc_state->port_clock;
/*
* See comment in intel_dpll_hw_state to understand why we always use 0
static bool
bxt_ddi_pll_select(struct intel_crtc *intel_crtc,
struct intel_crtc_state *crtc_state,
- struct intel_encoder *intel_encoder,
- int clock)
+ struct intel_encoder *intel_encoder)
{
struct intel_shared_dpll *pll;
struct bxt_clk_div clk_div = {0};
int vco = 0;
uint32_t prop_coef, int_coef, gain_ctl, targ_cnt;
uint32_t lanestagger;
+ int clock = crtc_state->port_clock;
if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
intel_clock_t best_clock;
struct drm_device *dev = intel_crtc->base.dev;
struct intel_encoder *intel_encoder =
intel_ddi_get_crtc_new_encoder(crtc_state);
- int clock = crtc_state->port_clock;
if (IS_SKYLAKE(dev))
return skl_ddi_pll_select(intel_crtc, crtc_state,
- intel_encoder, clock);
+ intel_encoder);
else if (IS_BROXTON(dev))
return bxt_ddi_pll_select(intel_crtc, crtc_state,
- intel_encoder, clock);
+ intel_encoder);
else
return hsw_ddi_pll_select(intel_crtc, crtc_state,
- intel_encoder, clock);
+ intel_encoder);
}
void intel_ddi_set_pipe_settings(struct drm_crtc *crtc)
} else
temp |= TRANS_DDI_MODE_SELECT_DP_SST;
- temp |= DDI_PORT_WIDTH(intel_dp->lane_count);
+ temp |= DDI_PORT_WIDTH(intel_crtc->config->lane_count);
} else if (type == INTEL_OUTPUT_DP_MST) {
struct intel_dp *intel_dp = &enc_to_mst(encoder)->primary->dp;
} else
temp |= TRANS_DDI_MODE_SELECT_DP_SST;
- temp |= DDI_PORT_WIDTH(intel_dp->lane_count);
+ temp |= DDI_PORT_WIDTH(intel_crtc->config->lane_count);
} else {
WARN(1, "Invalid encoder type %d for pipe %c\n",
intel_encoder->type, pipe_name(pipe));
void intel_ddi_enable_pipe_clock(struct intel_crtc *intel_crtc)
{
struct drm_crtc *crtc = &intel_crtc->base;
- struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+ struct drm_device *dev = crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
enum port port = intel_ddi_get_encoder_port(intel_encoder);
enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
u32 n_entries, i;
uint32_t val;
- if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) {
+ if (type == INTEL_OUTPUT_EDP && dev_priv->edp_low_vswing) {
+ n_entries = ARRAY_SIZE(bxt_ddi_translations_edp);
+ ddi_translations = bxt_ddi_translations_edp;
+ } else if (type == INTEL_OUTPUT_DISPLAYPORT
+ || type == INTEL_OUTPUT_EDP) {
n_entries = ARRAY_SIZE(bxt_ddi_translations_dp);
ddi_translations = bxt_ddi_translations_dp;
} else if (type == INTEL_OUTPUT_HDMI) {
I915_WRITE(BXT_PORT_TX_DW2_GRP(port), val);
val = I915_READ(BXT_PORT_TX_DW3_LN0(port));
- val &= ~UNIQE_TRANGE_EN_METHOD;
+ val &= ~SCALE_DCOMP_METHOD;
if (ddi_translations[level].enable)
- val |= UNIQE_TRANGE_EN_METHOD;
+ val |= SCALE_DCOMP_METHOD;
+
+ if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD))
+ DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set");
+
I915_WRITE(BXT_PORT_TX_DW3_GRP(port), val);
val = I915_READ(BXT_PORT_TX_DW4_LN0(port));
if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) {
struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+ intel_dp_set_link_params(intel_dp, crtc->config);
+
intel_ddi_init_dp_buf_reg(intel_encoder);
intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
intel_dp_start_link_train(intel_dp);
- intel_dp_complete_link_train(intel_dp);
if (port != PORT_A || INTEL_INFO(dev)->gen >= 9)
intel_dp_stop_link_train(intel_dp);
} else if (type == INTEL_OUTPUT_HDMI) {
{
/* DPLL 1 */
.ctl = LCPLL2_CTL,
- .cfgcr1 = DPLL1_CFGCR1,
- .cfgcr2 = DPLL1_CFGCR2,
+ .cfgcr1 = DPLL_CFGCR1(SKL_DPLL1),
+ .cfgcr2 = DPLL_CFGCR2(SKL_DPLL1),
},
{
/* DPLL 2 */
.ctl = WRPLL_CTL1,
- .cfgcr1 = DPLL2_CFGCR1,
- .cfgcr2 = DPLL2_CFGCR2,
+ .cfgcr1 = DPLL_CFGCR1(SKL_DPLL2),
+ .cfgcr2 = DPLL_CFGCR2(SKL_DPLL2),
},
{
/* DPLL 3 */
.ctl = WRPLL_CTL2,
- .cfgcr1 = DPLL3_CFGCR1,
- .cfgcr2 = DPLL3_CFGCR2,
+ .cfgcr1 = DPLL_CFGCR1(SKL_DPLL3),
+ .cfgcr2 = DPLL_CFGCR2(SKL_DPLL3),
},
};
* here just read out lanes 0/1 and output a note if lanes 2/3 differ.
*/
hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(port));
- if (I915_READ(BXT_PORT_PCS_DW12_LN23(port) != hw_state->pcsdw12))
+ if (I915_READ(BXT_PORT_PCS_DW12_LN23(port)) != hw_state->pcsdw12)
DRM_DEBUG_DRIVER("lane stagger config different for lane 01 (%08x) and 23 (%08x)\n",
hw_state->pcsdw12,
I915_READ(BXT_PORT_PCS_DW12_LN23(port)));
intel_ddi_post_disable(intel_encoder);
- val = I915_READ(_FDI_RXA_CTL);
+ val = I915_READ(FDI_RX_CTL(PIPE_A));
val &= ~FDI_RX_ENABLE;
- I915_WRITE(_FDI_RXA_CTL, val);
+ I915_WRITE(FDI_RX_CTL(PIPE_A), val);
- val = I915_READ(_FDI_RXA_MISC);
+ val = I915_READ(FDI_RX_MISC(PIPE_A));
val &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
val |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2);
- I915_WRITE(_FDI_RXA_MISC, val);
+ I915_WRITE(FDI_RX_MISC(PIPE_A), val);
- val = I915_READ(_FDI_RXA_CTL);
+ val = I915_READ(FDI_RX_CTL(PIPE_A));
val &= ~FDI_PCDCLK;
- I915_WRITE(_FDI_RXA_CTL, val);
+ I915_WRITE(FDI_RX_CTL(PIPE_A), val);
- val = I915_READ(_FDI_RXA_CTL);
+ val = I915_READ(FDI_RX_CTL(PIPE_A));
val &= ~FDI_RX_PLL_ENABLE;
- I915_WRITE(_FDI_RXA_CTL, val);
+ I915_WRITE(FDI_RX_CTL(PIPE_A), val);
}
void intel_ddi_get_config(struct intel_encoder *encoder,
case TRANS_DDI_MODE_SELECT_DP_SST:
case TRANS_DDI_MODE_SELECT_DP_MST:
pipe_config->has_dp_encoder = true;
+ pipe_config->lane_count =
+ ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
intel_dp_get_m_n(intel_crtc, pipe_config);
break;
default:
goto err;
intel_dig_port->hpd_pulse = intel_dp_hpd_pulse;
- dev_priv->hotplug.irq_port[port] = intel_dig_port;
+ /*
+ * On BXT A0/A1, sw needs to activate DDIA HPD logic and
+ * interrupts to check the external panel connection.
+ */
+ if (IS_BROXTON(dev_priv) && (INTEL_REVID(dev) < BXT_REVID_B0)
+ && port == PORT_B)
+ dev_priv->hotplug.irq_port[PORT_A] = intel_dig_port;
+ else
+ dev_priv->hotplug.irq_port[port] = intel_dig_port;
}
/* In theory we don't need the encoder->type check, but leave it just in
DRM_FORMAT_ABGR8888,
DRM_FORMAT_XRGB2101010,
DRM_FORMAT_XBGR2101010,
+ DRM_FORMAT_YUYV,
+ DRM_FORMAT_YVYU,
+ DRM_FORMAT_UYVY,
+ DRM_FORMAT_VYUY,
};
/* Cursor formats */
struct intel_crtc_state *crtc_state);
static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
int num_connectors);
+static void skylake_pfit_enable(struct intel_crtc *crtc);
+static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
+static void ironlake_pfit_enable(struct intel_crtc *crtc);
static void intel_modeset_setup_hw_state(struct drm_device *dev);
typedef struct {
intel_p2_t p2;
};
+/* returns HPLL frequency in kHz */
+static int valleyview_get_vco(struct drm_i915_private *dev_priv)
+{
+ int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
+
+ /* Obtain SKU information */
+ mutex_lock(&dev_priv->sb_lock);
+ hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
+ CCK_FUSE_HPLL_FREQ_MASK;
+ mutex_unlock(&dev_priv->sb_lock);
+
+ return vco_freq[hpll_freq] * 1000;
+}
+
+static int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
+ const char *name, u32 reg)
+{
+ u32 val;
+ int divider;
+
+ if (dev_priv->hpll_freq == 0)
+ dev_priv->hpll_freq = valleyview_get_vco(dev_priv);
+
+ mutex_lock(&dev_priv->sb_lock);
+ val = vlv_cck_read(dev_priv, reg);
+ mutex_unlock(&dev_priv->sb_lock);
+
+ divider = val & CCK_FREQUENCY_VALUES;
+
+ WARN((val & CCK_FREQUENCY_STATUS) !=
+ (divider << CCK_FREQUENCY_STATUS_SHIFT),
+ "%s change in progress\n", name);
+
+ return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, divider + 1);
+}
+
int
intel_pch_rawclk(struct drm_device *dev)
{
return I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK;
}
+/* hrawclock is 1/4 the FSB frequency */
+int intel_hrawclk(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ uint32_t clkcfg;
+
+ /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */
+ if (IS_VALLEYVIEW(dev))
+ return 200;
+
+ clkcfg = I915_READ(CLKCFG);
+ switch (clkcfg & CLKCFG_FSB_MASK) {
+ case CLKCFG_FSB_400:
+ return 100;
+ case CLKCFG_FSB_533:
+ return 133;
+ case CLKCFG_FSB_667:
+ return 166;
+ case CLKCFG_FSB_800:
+ return 200;
+ case CLKCFG_FSB_1067:
+ return 266;
+ case CLKCFG_FSB_1333:
+ return 333;
+ /* these two are just a guess; one of them might be right */
+ case CLKCFG_FSB_1600:
+ case CLKCFG_FSB_1600_ALT:
+ return 400;
+ default:
+ return 133;
+ }
+}
+
+static void intel_update_czclk(struct drm_i915_private *dev_priv)
+{
+ if (!IS_VALLEYVIEW(dev_priv))
+ return;
+
+ dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
+ CCK_CZ_CLOCK_CONTROL);
+
+ DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
+}
+
static inline u32 /* units of 100MHz */
intel_fdi_link_freq(struct drm_device *dev)
{
}
}
-/*
- * ibx_digital_port_connected - is the specified port connected?
- * @dev_priv: i915 private structure
- * @port: the port to test
- *
- * Returns true if @port is connected, false otherwise.
- */
-bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
- struct intel_digital_port *port)
-{
- u32 bit;
-
- if (HAS_PCH_IBX(dev_priv->dev)) {
- switch (port->port) {
- case PORT_B:
- bit = SDE_PORTB_HOTPLUG;
- break;
- case PORT_C:
- bit = SDE_PORTC_HOTPLUG;
- break;
- case PORT_D:
- bit = SDE_PORTD_HOTPLUG;
- break;
- default:
- return true;
- }
- } else {
- switch (port->port) {
- case PORT_B:
- bit = SDE_PORTB_HOTPLUG_CPT;
- break;
- case PORT_C:
- bit = SDE_PORTC_HOTPLUG_CPT;
- break;
- case PORT_D:
- bit = SDE_PORTD_HOTPLUG_CPT;
- break;
- case PORT_E:
- bit = SDE_PORTE_HOTPLUG_SPT;
- break;
- default:
- return true;
- }
- }
-
- return I915_READ(SDEISR) & bit;
-}
-
static const char *state_string(bool enabled)
{
return enabled ? "on" : "off";
void assert_pll(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
- int reg;
u32 val;
bool cur_state;
- reg = DPLL(pipe);
- val = I915_READ(reg);
+ val = I915_READ(DPLL(pipe));
cur_state = !!(val & DPLL_VCO_ENABLE);
I915_STATE_WARN(cur_state != state,
"PLL state assertion failure (expected %s, current %s)\n",
static void assert_fdi_tx(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
- int reg;
- u32 val;
bool cur_state;
enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
pipe);
if (HAS_DDI(dev_priv->dev)) {
/* DDI does not have a specific FDI_TX register */
- reg = TRANS_DDI_FUNC_CTL(cpu_transcoder);
- val = I915_READ(reg);
+ u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
} else {
- reg = FDI_TX_CTL(pipe);
- val = I915_READ(reg);
+ u32 val = I915_READ(FDI_TX_CTL(pipe));
cur_state = !!(val & FDI_TX_ENABLE);
}
I915_STATE_WARN(cur_state != state,
static void assert_fdi_rx(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
- int reg;
u32 val;
bool cur_state;
- reg = FDI_RX_CTL(pipe);
- val = I915_READ(reg);
+ val = I915_READ(FDI_RX_CTL(pipe));
cur_state = !!(val & FDI_RX_ENABLE);
I915_STATE_WARN(cur_state != state,
"FDI RX state assertion failure (expected %s, current %s)\n",
static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
- int reg;
u32 val;
/* ILK FDI PLL is always enabled */
if (HAS_DDI(dev_priv->dev))
return;
- reg = FDI_TX_CTL(pipe);
- val = I915_READ(reg);
+ val = I915_READ(FDI_TX_CTL(pipe));
I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
}
void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
- int reg;
u32 val;
bool cur_state;
- reg = FDI_RX_CTL(pipe);
- val = I915_READ(reg);
+ val = I915_READ(FDI_RX_CTL(pipe));
cur_state = !!(val & FDI_RX_PLL_ENABLE);
I915_STATE_WARN(cur_state != state,
"FDI RX PLL assertion failure (expected %s, current %s)\n",
bool cur_state;
if (IS_845G(dev) || IS_I865G(dev))
- cur_state = I915_READ(_CURACNTR) & CURSOR_ENABLE;
+ cur_state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
else
cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
void assert_pipe(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
- int reg;
- u32 val;
bool cur_state;
enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
pipe);
POWER_DOMAIN_TRANSCODER(cpu_transcoder))) {
cur_state = false;
} else {
- reg = PIPECONF(cpu_transcoder);
- val = I915_READ(reg);
+ u32 val = I915_READ(PIPECONF(cpu_transcoder));
cur_state = !!(val & PIPECONF_ENABLE);
}
static void assert_plane(struct drm_i915_private *dev_priv,
enum plane plane, bool state)
{
- int reg;
u32 val;
bool cur_state;
- reg = DSPCNTR(plane);
- val = I915_READ(reg);
+ val = I915_READ(DSPCNTR(plane));
cur_state = !!(val & DISPLAY_PLANE_ENABLE);
I915_STATE_WARN(cur_state != state,
"plane %c assertion failure (expected %s, current %s)\n",
enum pipe pipe)
{
struct drm_device *dev = dev_priv->dev;
- int reg, i;
- u32 val;
- int cur_pipe;
+ int i;
/* Primary planes are fixed to pipes on gen4+ */
if (INTEL_INFO(dev)->gen >= 4) {
- reg = DSPCNTR(pipe);
- val = I915_READ(reg);
+ u32 val = I915_READ(DSPCNTR(pipe));
I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE,
"plane %c assertion failure, should be disabled but not\n",
plane_name(pipe));
/* Need to check both planes against the pipe */
for_each_pipe(dev_priv, i) {
- reg = DSPCNTR(i);
- val = I915_READ(reg);
- cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
+ u32 val = I915_READ(DSPCNTR(i));
+ enum pipe cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
DISPPLANE_SEL_PIPE_SHIFT;
I915_STATE_WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
"plane %c assertion failure, should be off on pipe %c but is still active\n",
enum pipe pipe)
{
struct drm_device *dev = dev_priv->dev;
- int reg, sprite;
- u32 val;
+ int sprite;
if (INTEL_INFO(dev)->gen >= 9) {
for_each_sprite(dev_priv, pipe, sprite) {
- val = I915_READ(PLANE_CTL(pipe, sprite));
+ u32 val = I915_READ(PLANE_CTL(pipe, sprite));
I915_STATE_WARN(val & PLANE_CTL_ENABLE,
"plane %d assertion failure, should be off on pipe %c but is still active\n",
sprite, pipe_name(pipe));
}
} else if (IS_VALLEYVIEW(dev)) {
for_each_sprite(dev_priv, pipe, sprite) {
- reg = SPCNTR(pipe, sprite);
- val = I915_READ(reg);
+ u32 val = I915_READ(SPCNTR(pipe, sprite));
I915_STATE_WARN(val & SP_ENABLE,
"sprite %c assertion failure, should be off on pipe %c but is still active\n",
sprite_name(pipe, sprite), pipe_name(pipe));
}
} else if (INTEL_INFO(dev)->gen >= 7) {
- reg = SPRCTL(pipe);
- val = I915_READ(reg);
+ u32 val = I915_READ(SPRCTL(pipe));
I915_STATE_WARN(val & SPRITE_ENABLE,
"sprite %c assertion failure, should be off on pipe %c but is still active\n",
plane_name(pipe), pipe_name(pipe));
} else if (INTEL_INFO(dev)->gen >= 5) {
- reg = DVSCNTR(pipe);
- val = I915_READ(reg);
+ u32 val = I915_READ(DVSCNTR(pipe));
I915_STATE_WARN(val & DVS_ENABLE,
"sprite %c assertion failure, should be off on pipe %c but is still active\n",
plane_name(pipe), pipe_name(pipe));
static void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
- int reg;
u32 val;
bool enabled;
- reg = PCH_TRANSCONF(pipe);
- val = I915_READ(reg);
+ val = I915_READ(PCH_TRANSCONF(pipe));
enabled = !!(val & TRANS_ENABLE);
I915_STATE_WARN(enabled,
"transcoder assertion failed, should be off on pipe %c but is still active\n",
static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
- int reg;
u32 val;
assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
- reg = PCH_ADPA;
- val = I915_READ(reg);
+ val = I915_READ(PCH_ADPA);
I915_STATE_WARN(adpa_pipe_enabled(dev_priv, pipe, val),
"PCH VGA enabled on transcoder %c, should be disabled\n",
pipe_name(pipe));
- reg = PCH_LVDS;
- val = I915_READ(reg);
+ val = I915_READ(PCH_LVDS);
I915_STATE_WARN(lvds_pipe_enabled(dev_priv, pipe, val),
"PCH LVDS enabled on transcoder %c, should be disabled\n",
pipe_name(pipe));
assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMID);
}
-static void intel_init_dpio(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- if (!IS_VALLEYVIEW(dev))
- return;
-
- /*
- * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
- * CHV x1 PHY (DP/HDMI D)
- * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
- */
- if (IS_CHERRYVIEW(dev)) {
- DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
- DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
- } else {
- DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
- }
-}
-
static void vlv_enable_pll(struct intel_crtc *crtc,
const struct intel_crtc_state *pipe_config)
{
val &= ~DPIO_DCLKP_EN;
vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
- /* disable left/right clock distribution */
- if (pipe != PIPE_B) {
- val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
- val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
- vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
- } else {
- val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
- val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
- vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
- }
-
mutex_unlock(&dev_priv->sb_lock);
}
assert_fdi_rx_enabled(dev_priv, TRANSCODER_A);
/* Workaround: set timing override bit. */
- val = I915_READ(_TRANSA_CHICKEN2);
+ val = I915_READ(TRANS_CHICKEN2(PIPE_A));
val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
- I915_WRITE(_TRANSA_CHICKEN2, val);
+ I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
val = TRANS_ENABLE;
pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
DRM_ERROR("Failed to disable PCH transcoder\n");
/* Workaround: clear timing override bit. */
- val = I915_READ(_TRANSA_CHICKEN2);
+ val = I915_READ(TRANS_CHICKEN2(PIPE_A));
val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
- I915_WRITE(_TRANSA_CHICKEN2, val);
+ I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
}
/**
unsigned int
intel_tile_height(struct drm_device *dev, uint32_t pixel_format,
- uint64_t fb_format_modifier)
+ uint64_t fb_format_modifier, unsigned int plane)
{
unsigned int tile_height;
uint32_t pixel_bytes;
tile_height = 32;
break;
case I915_FORMAT_MOD_Yf_TILED:
- pixel_bytes = drm_format_plane_cpp(pixel_format, 0);
+ pixel_bytes = drm_format_plane_cpp(pixel_format, plane);
switch (pixel_bytes) {
default:
case 1:
uint32_t pixel_format, uint64_t fb_format_modifier)
{
return ALIGN(height, intel_tile_height(dev, pixel_format,
- fb_format_modifier));
+ fb_format_modifier, 0));
}
static int
info->height = fb->height;
info->pixel_format = fb->pixel_format;
info->pitch = fb->pitches[0];
+ info->uv_offset = fb->offsets[1];
info->fb_modifier = fb->modifier[0];
tile_height = intel_tile_height(fb->dev, fb->pixel_format,
- fb->modifier[0]);
+ fb->modifier[0], 0);
tile_pitch = PAGE_SIZE / tile_height;
info->width_pages = DIV_ROUND_UP(fb->pitches[0], tile_pitch);
info->height_pages = DIV_ROUND_UP(fb->height, tile_height);
info->size = info->width_pages * info->height_pages * PAGE_SIZE;
+ if (info->pixel_format == DRM_FORMAT_NV12) {
+ tile_height = intel_tile_height(fb->dev, fb->pixel_format,
+ fb->modifier[0], 1);
+ tile_pitch = PAGE_SIZE / tile_height;
+ info->width_pages_uv = DIV_ROUND_UP(fb->pitches[0], tile_pitch);
+ info->height_pages_uv = DIV_ROUND_UP(fb->height / 2,
+ tile_height);
+ info->size_uv = info->width_pages_uv * info->height_pages_uv *
+ PAGE_SIZE;
+ }
+
return 0;
}
struct intel_initial_plane_config *plane_config)
{
struct drm_device *dev = crtc->base.dev;
+ struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_gem_object *obj = NULL;
struct drm_mode_fb_cmd2 mode_cmd = { 0 };
struct drm_framebuffer *fb = &plane_config->fb->base;
if (plane_config->size == 0)
return false;
+ /* If the FB is too big, just don't use it since fbdev is not very
+ * important and we should probably use that space with FBC or other
+ * features. */
+ if (size_aligned * 2 > dev_priv->gtt.stolen_usable_size)
+ return false;
+
obj = i915_gem_object_create_stolen_for_preallocated(dev,
base_aligned,
base_aligned,
(intel_crtc->config->pipe_src_w - 1) * pixel_size;
}
+ intel_crtc->adjusted_x = x;
+ intel_crtc->adjusted_y = y;
+
I915_WRITE(reg, dspcntr);
I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
}
}
+ intel_crtc->adjusted_x = x;
+ intel_crtc->adjusted_y = y;
+
I915_WRITE(reg, dspcntr);
I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
}
unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane,
- struct drm_i915_gem_object *obj)
+ struct drm_i915_gem_object *obj,
+ unsigned int plane)
{
const struct i915_ggtt_view *view = &i915_ggtt_view_normal;
+ struct i915_vma *vma;
+ unsigned char *offset;
if (intel_rotation_90_or_270(intel_plane->base.state->rotation))
view = &i915_ggtt_view_rotated;
- return i915_gem_obj_ggtt_offset_view(obj, view);
+ vma = i915_gem_obj_to_ggtt_view(obj, view);
+ if (WARN(!vma, "ggtt vma for display object not found! (view=%u)\n",
+ view->type))
+ return -1;
+
+ offset = (unsigned char *)vma->node.start;
+
+ if (plane == 1) {
+ offset += vma->ggtt_view.rotation_info.uv_start_page *
+ PAGE_SIZE;
+ }
+
+ return (unsigned long)offset;
}
static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
- DRM_DEBUG_KMS("CRTC:%d Disabled scaler id %u.%u\n",
- intel_crtc->base.base.id, intel_crtc->pipe, id);
}
/*
obj = intel_fb_obj(fb);
stride_div = intel_fb_stride_alignment(dev, fb->modifier[0],
fb->pixel_format);
- surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj);
+ surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj, 0);
- /*
- * FIXME: intel_plane_state->src, dst aren't set when transitional
- * update_plane helpers are called from legacy paths.
- * Once full atomic crtc is available, below check can be avoided.
- */
- if (drm_rect_width(&plane_state->src)) {
- scaler_id = plane_state->scaler_id;
- src_x = plane_state->src.x1 >> 16;
- src_y = plane_state->src.y1 >> 16;
- src_w = drm_rect_width(&plane_state->src) >> 16;
- src_h = drm_rect_height(&plane_state->src) >> 16;
- dst_x = plane_state->dst.x1;
- dst_y = plane_state->dst.y1;
- dst_w = drm_rect_width(&plane_state->dst);
- dst_h = drm_rect_height(&plane_state->dst);
-
- WARN_ON(x != src_x || y != src_y);
- } else {
- src_w = intel_crtc->config->pipe_src_w;
- src_h = intel_crtc->config->pipe_src_h;
- }
+ WARN_ON(drm_rect_width(&plane_state->src) == 0);
+
+ scaler_id = plane_state->scaler_id;
+ src_x = plane_state->src.x1 >> 16;
+ src_y = plane_state->src.y1 >> 16;
+ src_w = drm_rect_width(&plane_state->src) >> 16;
+ src_h = drm_rect_height(&plane_state->src) >> 16;
+ dst_x = plane_state->dst.x1;
+ dst_y = plane_state->dst.y1;
+ dst_w = drm_rect_width(&plane_state->dst);
+ dst_h = drm_rect_height(&plane_state->dst);
+
+ WARN_ON(x != src_x || y != src_y);
if (intel_rotation_90_or_270(rotation)) {
/* stride = Surface height in tiles */
tile_height = intel_tile_height(dev, fb->pixel_format,
- fb->modifier[0]);
+ fb->modifier[0], 0);
stride = DIV_ROUND_UP(fb->height, tile_height);
x_offset = stride * tile_height - y - src_h;
y_offset = x;
}
plane_offset = y_offset << 16 | x_offset;
+ intel_crtc->adjusted_x = x_offset;
+ intel_crtc->adjusted_y = y_offset;
+
I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
I915_WRITE(PLANE_OFFSET(pipe, 0), plane_offset);
I915_WRITE(PLANE_SIZE(pipe, 0), plane_size);
static void intel_update_primary_planes(struct drm_device *dev)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_crtc *crtc;
for_each_crtc(dev, crtc) {
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct intel_plane *plane = to_intel_plane(crtc->primary);
+ struct intel_plane_state *plane_state;
- drm_modeset_lock(&crtc->mutex, NULL);
- /*
- * FIXME: Once we have proper support for primary planes (and
- * disabling them without disabling the entire crtc) allow again
- * a NULL crtc->primary->fb.
- */
- if (intel_crtc->active && crtc->primary->fb)
- dev_priv->display.update_primary_plane(crtc,
- crtc->primary->fb,
- crtc->x,
- crtc->y);
- drm_modeset_unlock(&crtc->mutex);
+ drm_modeset_lock_crtc(crtc, &plane->base);
+
+ plane_state = to_intel_plane_state(plane->base.state);
+
+ if (plane_state->base.fb)
+ plane->commit_plane(&plane->base, plane_state);
+
+ drm_modeset_unlock_crtc(crtc);
}
}
* so update the base address of all primary
* planes to the the last fb to make sure we're
* showing the correct fb after a reset.
+ *
+ * FIXME: Atomic will make this obsolete since we won't schedule
+ * CS-based flips (which might get lost in gpu resets) any more.
*/
intel_update_primary_planes(dev);
return;
return pending;
}
-static void intel_update_pipe_size(struct intel_crtc *crtc)
+static void intel_update_pipe_config(struct intel_crtc *crtc,
+ struct intel_crtc_state *old_crtc_state)
{
struct drm_device *dev = crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- const struct drm_display_mode *adjusted_mode;
+ struct intel_crtc_state *pipe_config =
+ to_intel_crtc_state(crtc->base.state);
- if (!i915.fastboot)
- return;
+ /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
+ crtc->base.mode = crtc->base.state->mode;
+
+ DRM_DEBUG_KMS("Updating pipe size %ix%i -> %ix%i\n",
+ old_crtc_state->pipe_src_w, old_crtc_state->pipe_src_h,
+ pipe_config->pipe_src_w, pipe_config->pipe_src_h);
+
+ if (HAS_DDI(dev))
+ intel_set_pipe_csc(&crtc->base);
/*
* Update pipe size and adjust fitter if needed: the reason for this is
* fastboot case, we'll flip, but if we don't update the pipesrc and
* pfit state, we'll end up with a big fb scanned out into the wrong
* sized surface.
- *
- * To fix this properly, we need to hoist the checks up into
- * compute_mode_changes (or above), check the actual pfit state and
- * whether the platform allows pfit disable with pipe active, and only
- * then update the pipesrc and pfit state, even on the flip path.
*/
- adjusted_mode = &crtc->config->base.adjusted_mode;
-
I915_WRITE(PIPESRC(crtc->pipe),
- ((adjusted_mode->crtc_hdisplay - 1) << 16) |
- (adjusted_mode->crtc_vdisplay - 1));
- if (!crtc->config->pch_pfit.enabled &&
- (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) ||
- intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) {
- I915_WRITE(PF_CTL(crtc->pipe), 0);
- I915_WRITE(PF_WIN_POS(crtc->pipe), 0);
- I915_WRITE(PF_WIN_SZ(crtc->pipe), 0);
+ ((pipe_config->pipe_src_w - 1) << 16) |
+ (pipe_config->pipe_src_h - 1));
+
+ /* on skylake this is done by detaching scalers */
+ if (INTEL_INFO(dev)->gen >= 9) {
+ skl_detach_scalers(crtc);
+
+ if (pipe_config->pch_pfit.enabled)
+ skylake_pfit_enable(crtc);
+ } else if (HAS_PCH_SPLIT(dev)) {
+ if (pipe_config->pch_pfit.enabled)
+ ironlake_pfit_enable(crtc);
+ else if (old_crtc_state->pch_pfit.enabled)
+ ironlake_pfit_disable(crtc, true);
}
- crtc->config->pipe_src_w = adjusted_mode->crtc_hdisplay;
- crtc->config->pipe_src_h = adjusted_mode->crtc_vdisplay;
}
static void intel_fdi_normal_train(struct drm_crtc *crtc)
int skl_update_scaler_crtc(struct intel_crtc_state *state)
{
struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
- struct drm_display_mode *adjusted_mode =
- &state->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
DRM_DEBUG_KMS("Updating scaler for [CRTC:%i] scaler_user index %u.%u\n",
intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX);
return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
&state->scaler_state.scaler_id, DRM_ROTATE_0,
state->pipe_src_w, state->pipe_src_h,
- adjusted_mode->hdisplay, adjusted_mode->vdisplay);
+ adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay);
}
/**
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
enum pipe pipe = intel_crtc->pipe;
- int palreg = PALETTE(pipe);
int i;
bool reenable_ips = false;
assert_pll_enabled(dev_priv, pipe);
}
- /* use legacy palette for Ironlake */
- if (!HAS_GMCH_DISPLAY(dev))
- palreg = LGC_PALETTE(pipe);
-
/* Workaround : Do not read or write the pipe palette/gamma data while
* GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
*/
}
for (i = 0; i < 256; i++) {
- I915_WRITE(palreg + 4 * i,
+ u32 palreg;
+
+ if (HAS_GMCH_DISPLAY(dev))
+ palreg = PALETTE(pipe, i);
+ else
+ palreg = LGC_PALETTE(pipe, i);
+
+ I915_WRITE(palreg,
(intel_crtc->lut_r[i] << 16) |
(intel_crtc->lut_g[i] << 8) |
intel_crtc->lut_b[i]);
int pipe = intel_crtc->pipe, hsw_workaround_pipe;
struct intel_crtc_state *pipe_config =
to_intel_crtc_state(crtc->state);
+ bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
if (WARN_ON(intel_crtc->active))
return;
intel_crtc->active = true;
intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
- for_each_encoder_on_crtc(dev, crtc, encoder)
+ for_each_encoder_on_crtc(dev, crtc, encoder) {
+ if (encoder->pre_pll_enable)
+ encoder->pre_pll_enable(encoder);
if (encoder->pre_enable)
encoder->pre_enable(encoder);
+ }
if (intel_crtc->config->has_pch_encoder) {
intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
dev_priv->display.fdi_link_train(crtc);
}
- intel_ddi_enable_pipe_clock(intel_crtc);
+ if (!is_dsi)
+ intel_ddi_enable_pipe_clock(intel_crtc);
- if (INTEL_INFO(dev)->gen == 9)
+ if (INTEL_INFO(dev)->gen >= 9)
skylake_pfit_enable(intel_crtc);
- else if (INTEL_INFO(dev)->gen < 9)
- ironlake_pfit_enable(intel_crtc);
else
- MISSING_CASE(INTEL_INFO(dev)->gen);
+ ironlake_pfit_enable(intel_crtc);
/*
* On ILK+ LUT must be loaded before the pipe is running but with
intel_crtc_load_lut(crtc);
intel_ddi_set_pipe_settings(crtc);
- intel_ddi_enable_transcoder_func(crtc);
+ if (!is_dsi)
+ intel_ddi_enable_transcoder_func(crtc);
intel_update_watermarks(crtc);
intel_enable_pipe(intel_crtc);
if (intel_crtc->config->has_pch_encoder)
lpt_pch_enable(crtc);
- if (intel_crtc->config->dp_encoder_is_mst)
+ if (intel_crtc->config->dp_encoder_is_mst && !is_dsi)
intel_ddi_set_vc_payload_alloc(crtc, true);
assert_vblank_disabled(crtc);
}
}
-static void ironlake_pfit_disable(struct intel_crtc *crtc)
+static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
{
struct drm_device *dev = crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
/* To avoid upsetting the power well on haswell only disable the pfit if
* it's in use. The hw state code will make sure we get this right. */
- if (crtc->config->pch_pfit.enabled) {
+ if (force || crtc->config->pch_pfit.enabled) {
I915_WRITE(PF_CTL(pipe), 0);
I915_WRITE(PF_WIN_POS(pipe), 0);
I915_WRITE(PF_WIN_SZ(pipe), 0);
intel_disable_pipe(intel_crtc);
- ironlake_pfit_disable(intel_crtc);
+ ironlake_pfit_disable(intel_crtc, false);
if (intel_crtc->config->has_pch_encoder)
ironlake_fdi_disable(crtc);
ironlake_fdi_pll_disable(intel_crtc);
}
-
- intel_crtc->active = false;
- intel_update_watermarks(crtc);
}
static void haswell_crtc_disable(struct drm_crtc *crtc)
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_encoder *encoder;
enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
+ bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
for_each_encoder_on_crtc(dev, crtc, encoder) {
intel_opregion_notify_encoder(encoder, false);
if (intel_crtc->config->dp_encoder_is_mst)
intel_ddi_set_vc_payload_alloc(crtc, false);
- intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
+ if (!is_dsi)
+ intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
- if (INTEL_INFO(dev)->gen == 9)
+ if (INTEL_INFO(dev)->gen >= 9)
skylake_scaler_disable(intel_crtc);
- else if (INTEL_INFO(dev)->gen < 9)
- ironlake_pfit_disable(intel_crtc);
else
- MISSING_CASE(INTEL_INFO(dev)->gen);
+ ironlake_pfit_disable(intel_crtc, false);
- intel_ddi_disable_pipe_clock(intel_crtc);
+ if (!is_dsi)
+ intel_ddi_disable_pipe_clock(intel_crtc);
if (intel_crtc->config->has_pch_encoder) {
lpt_disable_pch_transcoder(dev_priv);
for_each_encoder_on_crtc(dev, crtc, encoder)
if (encoder->post_disable)
encoder->post_disable(encoder);
-
- intel_crtc->active = false;
- intel_update_watermarks(crtc);
}
static void i9xx_pfit_enable(struct intel_crtc *crtc)
modeset_put_power_domains(dev_priv, put_domains[i]);
}
+static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
+{
+ int max_cdclk_freq = dev_priv->max_cdclk_freq;
+
+ if (INTEL_INFO(dev_priv)->gen >= 9 ||
+ IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
+ return max_cdclk_freq;
+ else if (IS_CHERRYVIEW(dev_priv))
+ return max_cdclk_freq*95/100;
+ else if (INTEL_INFO(dev_priv)->gen < 4)
+ return 2*max_cdclk_freq*90/100;
+ else
+ return max_cdclk_freq*90/100;
+}
+
static void intel_update_max_cdclk(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
dev_priv->max_cdclk_freq = dev_priv->cdclk_freq;
}
+ dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
+
DRM_DEBUG_DRIVER("Max CD clock rate: %d kHz\n",
dev_priv->max_cdclk_freq);
+
+ DRM_DEBUG_DRIVER("Max dotclock rate: %d kHz\n",
+ dev_priv->max_dotclk_freq);
}
static void intel_update_cdclk(struct drm_device *dev)
if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
DRM_ERROR("DBuf power disable timeout\n");
- /* disable DPLL0 */
- I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
- if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
- DRM_ERROR("Couldn't disable DPLL0\n");
+ /*
+ * DMC assumes ownership of LCPLL and will get confused if we touch it.
+ */
+ if (dev_priv->csr.dmc_payload) {
+ /* disable DPLL0 */
+ I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) &
+ ~LCPLL_PLL_ENABLE);
+ if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
+ DRM_ERROR("Couldn't disable DPLL0\n");
+ }
intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
}
DRM_ERROR("DBuf power enable timeout\n");
}
-/* returns HPLL frequency in kHz */
-static int valleyview_get_vco(struct drm_i915_private *dev_priv)
-{
- int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
-
- /* Obtain SKU information */
- mutex_lock(&dev_priv->sb_lock);
- hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
- CCK_FUSE_HPLL_FREQ_MASK;
- mutex_unlock(&dev_priv->sb_lock);
-
- return vco_freq[hpll_freq] * 1000;
-}
-
/* Adjust CDclk dividers to allow high res or save power if possible */
static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
{
/* adjust cdclk divider */
val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
- val &= ~DISPLAY_FREQUENCY_VALUES;
+ val &= ~CCK_FREQUENCY_VALUES;
val |= divider;
vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
- DISPLAY_FREQUENCY_STATUS) == (divider << DISPLAY_FREQUENCY_STATUS_SHIFT),
+ CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
50))
DRM_ERROR("timed out waiting for CDclk change\n");
}
else
default_credits = PFI_CREDIT(8);
- if (DIV_ROUND_CLOSEST(dev_priv->cdclk_freq, 1000) >= dev_priv->rps.cz_freq) {
+ if (dev_priv->cdclk_freq >= dev_priv->czclk_freq) {
/* CHV suggested value is 31 or 63 */
if (IS_CHERRYVIEW(dev_priv))
credits = PFI_CREDIT_63;
is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
- if (!is_dsi) {
- if (IS_CHERRYVIEW(dev))
- chv_prepare_pll(intel_crtc, intel_crtc->config);
- else
- vlv_prepare_pll(intel_crtc, intel_crtc->config);
- }
-
if (intel_crtc->config->has_dp_encoder)
intel_dp_set_m_n(intel_crtc, M1_N1);
encoder->pre_pll_enable(encoder);
if (!is_dsi) {
- if (IS_CHERRYVIEW(dev))
+ if (IS_CHERRYVIEW(dev)) {
+ chv_prepare_pll(intel_crtc, intel_crtc->config);
chv_enable_pll(intel_crtc, intel_crtc->config);
- else
+ } else {
+ vlv_prepare_pll(intel_crtc, intel_crtc->config);
vlv_enable_pll(intel_crtc, intel_crtc->config);
+ }
}
for_each_encoder_on_crtc(dev, crtc, encoder)
i9xx_disable_pll(intel_crtc);
}
+ for_each_encoder_on_crtc(dev, crtc, encoder)
+ if (encoder->post_pll_disable)
+ encoder->post_pll_disable(encoder);
+
if (!IS_GEN2(dev))
intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
-
- intel_crtc->active = false;
- intel_update_watermarks(crtc);
}
static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
intel_crtc_disable_planes(crtc, crtc->state->plane_mask);
dev_priv->display.crtc_disable(crtc);
+ intel_crtc->active = false;
+ intel_update_watermarks(crtc);
intel_disable_shared_dpll(intel_crtc);
domains = intel_crtc->enabled_power_domains;
struct intel_crtc_state *pipe_config)
{
struct drm_device *dev = intel_crtc->base.dev;
- struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
int lane, link_bw, fdi_dotclock, ret;
bool needs_recompute = false;
{
struct drm_device *dev = crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
/* FIXME should check pixel clock limits on all platforms */
if (INTEL_INFO(dev)->gen < 4) {
* WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
*/
if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
- adjusted_mode->hsync_start == adjusted_mode->hdisplay)
+ adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
return -EINVAL;
if (HAS_IPS(dev))
static int valleyview_get_display_clock_speed(struct drm_device *dev)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
- u32 val;
- int divider;
-
- if (dev_priv->hpll_freq == 0)
- dev_priv->hpll_freq = valleyview_get_vco(dev_priv);
-
- mutex_lock(&dev_priv->sb_lock);
- val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
- mutex_unlock(&dev_priv->sb_lock);
-
- divider = val & DISPLAY_FREQUENCY_VALUES;
-
- WARN((val & DISPLAY_FREQUENCY_STATUS) !=
- (divider << DISPLAY_FREQUENCY_STATUS_SHIFT),
- "cdclk change in progress\n");
-
- return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, divider + 1);
+ return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk",
+ CCK_DISPLAY_CLOCK_CONTROL);
}
static int ilk_get_display_clock_speed(struct drm_device *dev)
1 << DPIO_CHV_N_DIV_SHIFT);
/* M2 fraction division */
- if (bestm2_frac)
- vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
+ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
/* M2 fraction division enable */
dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
struct drm_i915_private *dev_priv = dev->dev_private;
enum pipe pipe = intel_crtc->pipe;
enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
- struct drm_display_mode *adjusted_mode =
- &intel_crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
uint32_t crtc_vtotal, crtc_vblank_end;
int vsyncshift = 0;
else
i9xx_crtc_clock_get(crtc, pipe_config);
+ /*
+ * Normally the dotclock is filled in by the encoder .get_config()
+ * but in case the pipe is enabled w/o any ports we need a sane
+ * default.
+ */
+ pipe_config->base.adjusted_mode.crtc_clock =
+ pipe_config->port_clock / pipe_config->pixel_multiplier;
+
return true;
}
if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
with_spread = true;
- if (WARN(dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE &&
- with_fdi, "LP PCH doesn't have FDI\n"))
+ if (WARN(HAS_PCH_LPT_LP(dev) && with_fdi, "LP PCH doesn't have FDI\n"))
with_fdi = false;
mutex_lock(&dev_priv->sb_lock);
}
}
- reg = (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) ?
- SBI_GEN0 : SBI_DBUFF0;
+ reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
mutex_lock(&dev_priv->sb_lock);
- reg = (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) ?
- SBI_GEN0 : SBI_DBUFF0;
+ reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
DRM_DEBUG_KMS("Enabling package C8+\n");
- if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
+ if (HAS_PCH_LPT_LP(dev)) {
val = I915_READ(SOUTH_DSPCLK_GATE_D);
val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
hsw_restore_lcpll(dev_priv);
lpt_init_pch_refclk(dev);
- if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
+ if (HAS_PCH_LPT_LP(dev)) {
val = I915_READ(SOUTH_DSPCLK_GATE_D);
val |= PCH_LP_PARTITION_LEVEL_DISABLE;
I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
}
if (intel_display_power_is_enabled(dev_priv, pfit_domain)) {
- if (INTEL_INFO(dev)->gen == 9)
+ if (INTEL_INFO(dev)->gen >= 9)
skylake_get_pfit_config(crtc, pipe_config);
- else if (INTEL_INFO(dev)->gen < 9)
- ironlake_get_pfit_config(crtc, pipe_config);
else
- MISSING_CASE(INTEL_INFO(dev)->gen);
+ ironlake_get_pfit_config(crtc, pipe_config);
}
if (IS_HASWELL(dev))
/* On these chipsets we can only modify the base/size/stride
* whilst the cursor is disabled.
*/
- I915_WRITE(_CURACNTR, 0);
- POSTING_READ(_CURACNTR);
+ I915_WRITE(CURCNTR(PIPE_A), 0);
+ POSTING_READ(CURCNTR(PIPE_A));
intel_crtc->cursor_cntl = 0;
}
if (intel_crtc->cursor_base != base) {
- I915_WRITE(_CURABASE, base);
+ I915_WRITE(CURBASE(PIPE_A), base);
intel_crtc->cursor_base = base;
}
}
if (intel_crtc->cursor_cntl != cntl) {
- I915_WRITE(_CURACNTR, cntl);
- POSTING_READ(_CURACNTR);
+ I915_WRITE(CURCNTR(PIPE_A), cntl);
+ POSTING_READ(CURCNTR(PIPE_A));
intel_crtc->cursor_cntl = cntl;
}
}
}
cntl |= pipe << 28; /* Connect to correct pipe */
- if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+ if (HAS_DDI(dev))
cntl |= CURSOR_PIPE_CSC_ENABLE;
}
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
- int x = crtc->cursor_x;
- int y = crtc->cursor_y;
+ struct drm_plane_state *cursor_state = crtc->cursor->state;
+ int x = cursor_state->crtc_x;
+ int y = cursor_state->crtc_y;
u32 base = 0, pos = 0;
if (on)
base = 0;
if (x < 0) {
- if (x + intel_crtc->base.cursor->state->crtc_w <= 0)
+ if (x + cursor_state->crtc_w <= 0)
base = 0;
pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
pos |= x << CURSOR_X_SHIFT;
if (y < 0) {
- if (y + intel_crtc->base.cursor->state->crtc_h <= 0)
+ if (y + cursor_state->crtc_h <= 0)
base = 0;
pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
/* ILK+ do this automagically */
if (HAS_GMCH_DISPLAY(dev) &&
crtc->cursor->state->rotation == BIT(DRM_ROTATE_180)) {
- base += (intel_crtc->base.cursor->state->crtc_h *
- intel_crtc->base.cursor->state->crtc_w - 1) * 4;
+ base += (cursor_state->crtc_h *
+ cursor_state->crtc_w - 1) * 4;
}
if (IS_845G(dev) || IS_I865G(dev))
*/
return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) ==
crtc->unpin_work->gtt_offset &&
- g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_GM45(crtc->pipe)),
+ g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)),
crtc->unpin_work->flip_count);
}
spin_unlock_irqrestore(&dev->event_lock, flags);
}
-static inline void intel_mark_page_flip_active(struct intel_crtc *intel_crtc)
+static inline void intel_mark_page_flip_active(struct intel_unpin_work *work)
{
/* Ensure that the work item is consistent when activating it ... */
smp_wmb();
- atomic_set(&intel_crtc->unpin_work->pending, INTEL_FLIP_PENDING);
+ atomic_set(&work->pending, INTEL_FLIP_PENDING);
/* and that it is marked active as soon as the irq could fire. */
smp_wmb();
}
intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
intel_ring_emit(ring, 0); /* aux display base address, unused */
- intel_mark_page_flip_active(intel_crtc);
+ intel_mark_page_flip_active(intel_crtc->unpin_work);
return 0;
}
intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
intel_ring_emit(ring, MI_NOOP);
- intel_mark_page_flip_active(intel_crtc);
+ intel_mark_page_flip_active(intel_crtc->unpin_work);
return 0;
}
pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
intel_ring_emit(ring, pf | pipesrc);
- intel_mark_page_flip_active(intel_crtc);
+ intel_mark_page_flip_active(intel_crtc->unpin_work);
return 0;
}
pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
intel_ring_emit(ring, pf | pipesrc);
- intel_mark_page_flip_active(intel_crtc);
+ intel_mark_page_flip_active(intel_crtc->unpin_work);
return 0;
}
DERRMR_PIPEB_PRI_FLIP_DONE |
DERRMR_PIPEC_PRI_FLIP_DONE));
if (IS_GEN8(dev))
- intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8(1) |
+ intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8 |
MI_SRM_LRM_GLOBAL_GTT);
else
- intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) |
+ intel_ring_emit(ring, MI_STORE_REGISTER_MEM |
MI_SRM_LRM_GLOBAL_GTT);
intel_ring_emit(ring, DERRMR);
intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
intel_ring_emit(ring, (MI_NOOP));
- intel_mark_page_flip_active(intel_crtc);
+ intel_mark_page_flip_active(intel_crtc->unpin_work);
return 0;
}
return ring != i915_gem_request_get_ring(obj->last_write_req);
}
-static void skl_do_mmio_flip(struct intel_crtc *intel_crtc)
+static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
+ struct intel_unpin_work *work)
{
struct drm_device *dev = intel_crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
I915_WRITE(PLANE_CTL(pipe, 0), ctl);
I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
- I915_WRITE(PLANE_SURF(pipe, 0), intel_crtc->unpin_work->gtt_offset);
+ I915_WRITE(PLANE_SURF(pipe, 0), work->gtt_offset);
POSTING_READ(PLANE_SURF(pipe, 0));
}
-static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc)
+static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
+ struct intel_unpin_work *work)
{
struct drm_device *dev = intel_crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
I915_WRITE(reg, dspcntr);
- I915_WRITE(DSPSURF(intel_crtc->plane),
- intel_crtc->unpin_work->gtt_offset);
+ I915_WRITE(DSPSURF(intel_crtc->plane), work->gtt_offset);
POSTING_READ(DSPSURF(intel_crtc->plane));
-
}
/*
* XXX: This is the temporary way to update the plane registers until we get
* around to using the usual plane update functions for MMIO flips
*/
-static void intel_do_mmio_flip(struct intel_crtc *intel_crtc)
+static void intel_do_mmio_flip(struct intel_mmio_flip *mmio_flip)
{
- struct drm_device *dev = intel_crtc->base.dev;
- u32 start_vbl_count;
+ struct intel_crtc *crtc = mmio_flip->crtc;
+ struct intel_unpin_work *work;
- intel_mark_page_flip_active(intel_crtc);
+ spin_lock_irq(&crtc->base.dev->event_lock);
+ work = crtc->unpin_work;
+ spin_unlock_irq(&crtc->base.dev->event_lock);
+ if (work == NULL)
+ return;
- intel_pipe_update_start(intel_crtc, &start_vbl_count);
+ intel_mark_page_flip_active(work);
- if (INTEL_INFO(dev)->gen >= 9)
- skl_do_mmio_flip(intel_crtc);
+ intel_pipe_update_start(crtc);
+
+ if (INTEL_INFO(mmio_flip->i915)->gen >= 9)
+ skl_do_mmio_flip(crtc, work);
else
/* use_mmio_flip() retricts MMIO flips to ilk+ */
- ilk_do_mmio_flip(intel_crtc);
+ ilk_do_mmio_flip(crtc, work);
- intel_pipe_update_end(intel_crtc, start_vbl_count);
+ intel_pipe_update_end(crtc);
}
static void intel_mmio_flip_work_func(struct work_struct *work)
struct intel_mmio_flip *mmio_flip =
container_of(work, struct intel_mmio_flip, work);
- if (mmio_flip->req)
+ if (mmio_flip->req) {
WARN_ON(__i915_wait_request(mmio_flip->req,
mmio_flip->crtc->reset_counter,
false, NULL,
&mmio_flip->i915->rps.mmioflips));
+ i915_gem_request_unreference__unlocked(mmio_flip->req);
+ }
- intel_do_mmio_flip(mmio_flip->crtc);
-
- i915_gem_request_unreference__unlocked(mmio_flip->req);
+ intel_do_mmio_flip(mmio_flip);
kfree(mmio_flip);
}
if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE)
return true;
+ if (atomic_read(&work->pending) < INTEL_FLIP_PENDING)
+ return false;
+
if (!work->enable_stall_check)
return false;
intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
- work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(pipe)) + 1;
+ work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
if (IS_VALLEYVIEW(dev)) {
ring = &dev_priv->ring[BCS];
if (ret)
goto cleanup_pending;
- work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary), obj)
- + intel_crtc->dspaddr_offset;
+ work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary),
+ obj, 0);
+ work->gtt_offset += intel_crtc->dspaddr_offset;
if (mmio_flip) {
ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
intel_crtc->atomic.update_wm_pre = true;
}
- if (visible)
+ if (visible || was_visible)
intel_crtc->atomic.fb_bits |=
to_intel_plane(plane)->frontbuffer_bit;
pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n,
pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
pipe_config->fdi_m_n.tu);
- DRM_DEBUG_KMS("dp: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
+ DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
pipe_config->has_dp_encoder,
+ pipe_config->lane_count,
pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
pipe_config->dp_m_n.tu);
- DRM_DEBUG_KMS("dp: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
+ DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
pipe_config->has_dp_encoder,
+ pipe_config->lane_count,
pipe_config->dp_m2_n2.gmch_m,
pipe_config->dp_m2_n2.gmch_n,
pipe_config->dp_m2_n2.link_m,
(DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
- /* Compute a starting value for pipe_config->pipe_bpp taking the source
- * plane pixel format and any sink constraints into account. Returns the
- * source plane bpp so that dithering can be selected on mismatches
- * after encoders and crtc also have had their say. */
base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
pipe_config);
if (base_bpp < 0)
/* Dithering seems to not pass-through bits correctly when it should, so
* only enable it on 6bpc panels. */
pipe_config->dither = pipe_config->pipe_bpp == 6*3;
- DRM_DEBUG_KMS("plane bpp: %i, pipe bpp: %i, dithering: %i\n",
+ DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
fail:
base.head) \
if (mask & (1 <<(intel_crtc)->pipe))
-
static bool
intel_compare_m_n(unsigned int m, unsigned int n,
unsigned int m2, unsigned int n2,
PIPE_CONF_CHECK_M_N(fdi_m_n);
PIPE_CONF_CHECK_I(has_dp_encoder);
+ PIPE_CONF_CHECK_I(lane_count);
if (INTEL_INFO(dev)->gen < 8) {
PIPE_CONF_CHECK_M_N(dp_m_n);
DRM_MODE_FLAG_NVSYNC);
}
- PIPE_CONF_CHECK_I(pipe_src_w);
- PIPE_CONF_CHECK_I(pipe_src_h);
-
- PIPE_CONF_CHECK_I(gmch_pfit.control);
+ PIPE_CONF_CHECK_X(gmch_pfit.control);
/* pfit ratios are autocomputed by the hw on gen4+ */
if (INTEL_INFO(dev)->gen < 4)
PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios);
- PIPE_CONF_CHECK_I(gmch_pfit.lvds_border_bits);
+ PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
- PIPE_CONF_CHECK_I(pch_pfit.enabled);
- if (current_config->pch_pfit.enabled) {
- PIPE_CONF_CHECK_I(pch_pfit.pos);
- PIPE_CONF_CHECK_I(pch_pfit.size);
- }
+ if (!adjust) {
+ PIPE_CONF_CHECK_I(pipe_src_w);
+ PIPE_CONF_CHECK_I(pipe_src_h);
- PIPE_CONF_CHECK_I(scaler_state.scaler_id);
+ PIPE_CONF_CHECK_I(pch_pfit.enabled);
+ if (current_config->pch_pfit.enabled) {
+ PIPE_CONF_CHECK_X(pch_pfit.pos);
+ PIPE_CONF_CHECK_X(pch_pfit.size);
+ }
+
+ PIPE_CONF_CHECK_I(scaler_state.scaler_id);
+ }
/* BDW+ don't expose a synchronous way to read the state */
if (IS_HASWELL(dev))
}
/* cursor */
- hw_entry = &hw_ddb.cursor[pipe];
- sw_entry = &sw_ddb->cursor[pipe];
+ hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
+ sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
if (skl_ddb_entry_equal(hw_entry, sw_entry))
continue;
struct intel_crtc_state *pipe_config, *sw_config;
bool active;
- if (!needs_modeset(crtc->state))
+ if (!needs_modeset(crtc->state) &&
+ !to_intel_crtc_state(crtc->state)->update_pipe)
continue;
__drm_atomic_helper_crtc_destroy_state(crtc, old_crtc_state);
* one to the value.
*/
if (IS_GEN2(dev)) {
- const struct drm_display_mode *mode = &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
int vtotal;
- vtotal = mode->crtc_vtotal;
- if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+ vtotal = adjusted_mode->crtc_vtotal;
+ if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
vtotal /= 2;
crtc->scanline_offset = vtotal - 1;
return ret;
}
-
static int intel_modeset_checks(struct drm_atomic_state *state)
{
struct drm_device *dev = state->dev;
if (ret)
return ret;
- if (i915.fastboot &&
- intel_pipe_config_compare(state->dev,
+ if (intel_pipe_config_compare(state->dev,
to_intel_crtc_state(crtc->state),
pipe_config, true)) {
crtc_state->mode_changed = false;
+ to_intel_crtc_state(crtc_state)->update_pipe = true;
}
if (needs_modeset(crtc_state)) {
for_each_crtc_in_state(state, crtc, crtc_state, i) {
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
bool modeset = needs_modeset(crtc->state);
+ bool update_pipe = !modeset &&
+ to_intel_crtc_state(crtc->state)->update_pipe;
+ unsigned long put_domains = 0;
if (modeset && crtc->state->active) {
update_scanline_offset(to_intel_crtc(crtc));
dev_priv->display.crtc_enable(crtc);
}
+ if (update_pipe) {
+ put_domains = modeset_get_crtc_power_domains(crtc);
+
+ /* make sure intel_modeset_check_state runs */
+ any_ms = true;
+ }
+
if (!modeset)
intel_pre_plane_update(intel_crtc);
drm_atomic_helper_commit_planes_on_crtc(crtc_state);
+
+ if (put_domains)
+ modeset_put_power_domains(dev_priv, put_domains);
+
intel_post_plane_update(intel_crtc);
}
{
struct drm_i915_private *dev_priv = dev->dev_private;
- intel_update_cdclk(dev);
-
if (HAS_DDI(dev))
intel_ddi_pll_init(dev);
else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
*/
int
intel_prepare_plane_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
struct drm_device *dev = plane->dev;
+ struct drm_framebuffer *fb = new_state->fb;
struct intel_plane *intel_plane = to_intel_plane(plane);
struct drm_i915_gem_object *obj = intel_fb_obj(fb);
struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb);
*/
void
intel_cleanup_plane_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *old_state)
{
struct drm_device *dev = plane->dev;
- struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+ struct drm_i915_gem_object *obj = intel_fb_obj(old_state->fb);
- if (WARN_ON(!obj))
+ if (!obj)
return;
if (plane->type != DRM_PLANE_TYPE_CURSOR ||
!INTEL_INFO(dev)->cursor_needs_physical) {
mutex_lock(&dev->struct_mutex);
- intel_unpin_fb_obj(fb, old_state);
+ intel_unpin_fb_obj(old_state->fb, old_state);
mutex_unlock(&dev->struct_mutex);
}
}
if (!crtc->state->active)
return;
- if (state->visible)
- /* FIXME: kill this fastboot hack */
- intel_update_pipe_size(intel_crtc);
-
- dev_priv->display.update_primary_plane(crtc, fb, crtc->x, crtc->y);
+ dev_priv->display.update_primary_plane(crtc, fb,
+ state->src.x1 >> 16,
+ state->src.y1 >> 16);
}
static void
{
struct drm_device *dev = crtc->dev;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct intel_crtc_state *old_intel_state =
+ to_intel_crtc_state(old_crtc_state);
+ bool modeset = needs_modeset(crtc->state);
if (intel_crtc->atomic.update_wm_pre)
intel_update_watermarks(crtc);
/* Perform vblank evasion around commit operation */
if (crtc->state->active)
- intel_pipe_update_start(intel_crtc, &intel_crtc->start_vbl_count);
+ intel_pipe_update_start(intel_crtc);
- if (!needs_modeset(crtc->state) && INTEL_INFO(dev)->gen >= 9)
+ if (modeset)
+ return;
+
+ if (to_intel_crtc_state(crtc->state)->update_pipe)
+ intel_update_pipe_config(intel_crtc, old_intel_state);
+ else if (INTEL_INFO(dev)->gen >= 9)
skl_detach_scalers(intel_crtc);
}
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
if (crtc->state->active)
- intel_pipe_update_end(intel_crtc, intel_crtc->start_vbl_count);
+ intel_pipe_update_end(intel_crtc);
}
/**
crtc = crtc ? crtc : plane->crtc;
intel_crtc = to_intel_crtc(crtc);
- plane->fb = state->base.fb;
- crtc->cursor_x = state->base.crtc_x;
- crtc->cursor_y = state->base.crtc_y;
-
if (intel_crtc->cursor_bo == obj)
goto update;
* On SKL pre-D0 the strap isn't connected, so we assume
* it's there.
*/
- found = I915_READ(DDI_BUF_CTL_A) & DDI_INIT_DISPLAY_DETECTED;
+ found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
/* WaIgnoreDDIAStrap: skl */
if (found || IS_SKYLAKE(dev))
intel_ddi_init(dev, PORT_A);
* eDP ports. Consult the VBT as well as DP_DETECTED to
* detect eDP ports.
*/
- if (I915_READ(VLV_DISPLAY_BASE + GEN4_HDMIB) & SDVO_DETECTED &&
+ if (I915_READ(VLV_HDMIB) & SDVO_DETECTED &&
!intel_dp_is_edp(dev, PORT_B))
- intel_hdmi_init(dev, VLV_DISPLAY_BASE + GEN4_HDMIB,
- PORT_B);
- if (I915_READ(VLV_DISPLAY_BASE + DP_B) & DP_DETECTED ||
+ intel_hdmi_init(dev, VLV_HDMIB, PORT_B);
+ if (I915_READ(VLV_DP_B) & DP_DETECTED ||
intel_dp_is_edp(dev, PORT_B))
- intel_dp_init(dev, VLV_DISPLAY_BASE + DP_B, PORT_B);
+ intel_dp_init(dev, VLV_DP_B, PORT_B);
- if (I915_READ(VLV_DISPLAY_BASE + GEN4_HDMIC) & SDVO_DETECTED &&
+ if (I915_READ(VLV_HDMIC) & SDVO_DETECTED &&
!intel_dp_is_edp(dev, PORT_C))
- intel_hdmi_init(dev, VLV_DISPLAY_BASE + GEN4_HDMIC,
- PORT_C);
- if (I915_READ(VLV_DISPLAY_BASE + DP_C) & DP_DETECTED ||
+ intel_hdmi_init(dev, VLV_HDMIC, PORT_C);
+ if (I915_READ(VLV_DP_C) & DP_DETECTED ||
intel_dp_is_edp(dev, PORT_C))
- intel_dp_init(dev, VLV_DISPLAY_BASE + DP_C, PORT_C);
+ intel_dp_init(dev, VLV_DP_C, PORT_C);
if (IS_CHERRYVIEW(dev)) {
- if (I915_READ(VLV_DISPLAY_BASE + CHV_HDMID) & SDVO_DETECTED)
- intel_hdmi_init(dev, VLV_DISPLAY_BASE + CHV_HDMID,
- PORT_D);
/* eDP not supported on port D, so don't check VBT */
- if (I915_READ(VLV_DISPLAY_BASE + DP_D) & DP_DETECTED)
- intel_dp_init(dev, VLV_DISPLAY_BASE + DP_D, PORT_D);
+ if (I915_READ(CHV_HDMID) & SDVO_DETECTED)
+ intel_hdmi_init(dev, CHV_HDMID, PORT_D);
+ if (I915_READ(CHV_DP_D) & DP_DETECTED)
+ intel_dp_init(dev, CHV_DP_D, PORT_D);
}
intel_dsi_init(dev);
dev_priv->display.queue_flip = intel_default_queue_flip;
}
- intel_panel_init_backlight_funcs(dev);
-
mutex_init(&dev_priv->pps_mutex);
}
}
}
- intel_init_dpio(dev);
+ intel_update_czclk(dev_priv);
+ intel_update_cdclk(dev);
intel_shared_dpll_init(dev);
{
struct drm_device *dev = crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- u32 reg, val;
+ u32 val;
if (INTEL_INFO(dev)->num_pipes == 1)
return true;
- reg = DSPCNTR(!crtc->plane);
- val = I915_READ(reg);
+ val = I915_READ(DSPCNTR(!crtc->plane));
if ((val & DISPLAY_PLANE_ENABLE) &&
(!!(val & DISPPLANE_SEL_PIPE_MASK) == crtc->pipe))
return true;
}
+static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
+{
+ struct drm_device *dev = crtc->base.dev;
+ struct intel_encoder *encoder;
+
+ for_each_encoder_on_crtc(dev, &crtc->base, encoder)
+ return true;
+
+ return false;
+}
+
static void intel_sanitize_crtc(struct intel_crtc *crtc)
{
struct drm_device *dev = crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_encoder *encoder;
u32 reg;
- bool enable;
/* Clear any frame start delays used for debugging left by the BIOS */
reg = PIPECONF(crtc->config->cpu_transcoder);
/* restore vblank interrupts to correct state */
drm_crtc_vblank_reset(&crtc->base);
if (crtc->active) {
- drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode);
- update_scanline_offset(crtc);
+ struct intel_plane *plane;
+
drm_crtc_vblank_on(&crtc->base);
+
+ /* Disable everything but the primary plane */
+ for_each_intel_plane_on_crtc(dev, crtc, plane) {
+ if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
+ continue;
+
+ plane->disable_plane(&plane->base, &crtc->base);
+ }
}
/* We need to sanitize the plane -> pipe mapping first because this will
/* Adjust the state of the output pipe according to whether we
* have active connectors/encoders. */
- enable = false;
- for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
- enable = true;
- break;
- }
-
- if (!enable)
+ if (!intel_crtc_has_encoders(crtc))
intel_crtc_disable_noatomic(&crtc->base);
if (crtc->active != crtc->base.state->active) {
+ struct intel_encoder *encoder;
/* This can happen either due to bugs in the get_hw_state
* functions or because of calls to intel_crtc_disable_noatomic,
i915_redisable_vga_power_on(dev);
}
-static bool primary_get_hw_state(struct intel_crtc *crtc)
+static bool primary_get_hw_state(struct intel_plane *plane)
{
- struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+ struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
- return !!(I915_READ(DSPCNTR(crtc->plane)) & DISPLAY_PLANE_ENABLE);
+ return I915_READ(DSPCNTR(plane->plane)) & DISPLAY_PLANE_ENABLE;
}
-static void readout_plane_state(struct intel_crtc *crtc,
- struct intel_crtc_state *crtc_state)
+/* FIXME read out full plane state for all planes */
+static void readout_plane_state(struct intel_crtc *crtc)
{
- struct intel_plane *p;
- struct intel_plane_state *plane_state;
- bool active = crtc_state->base.active;
-
- for_each_intel_plane(crtc->base.dev, p) {
- if (crtc->pipe != p->pipe)
- continue;
-
- plane_state = to_intel_plane_state(p->base.state);
+ struct drm_plane *primary = crtc->base.primary;
+ struct intel_plane_state *plane_state =
+ to_intel_plane_state(primary->state);
- if (p->base.type == DRM_PLANE_TYPE_PRIMARY)
- plane_state->visible = primary_get_hw_state(crtc);
- else {
- if (active)
- p->disable_plane(&p->base, &crtc->base);
+ plane_state->visible =
+ primary_get_hw_state(to_intel_plane(primary));
- plane_state->visible = false;
- }
- }
+ if (plane_state->visible)
+ crtc->base.state->plane_mask |= 1 << drm_plane_index(primary);
}
static void intel_modeset_readout_hw_state(struct drm_device *dev)
crtc->base.state->active = crtc->active;
crtc->base.enabled = crtc->active;
- memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
- if (crtc->base.state->active) {
- intel_mode_from_pipe_config(&crtc->base.mode, crtc->config);
- intel_mode_from_pipe_config(&crtc->base.state->adjusted_mode, crtc->config);
- WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
-
- /*
- * The initial mode needs to be set in order to keep
- * the atomic core happy. It wants a valid mode if the
- * crtc's enabled, so we do the above call.
- *
- * At this point some state updated by the connectors
- * in their ->detect() callback has not run yet, so
- * no recalculation can be done yet.
- *
- * Even if we could do a recalculation and modeset
- * right now it would cause a double modeset if
- * fbdev or userspace chooses a different initial mode.
- *
- * If that happens, someone indicated they wanted a
- * mode change, which means it's safe to do a full
- * recalculation.
- */
- crtc->base.state->mode.private_flags = I915_MODE_FLAG_INHERITED;
- }
-
- crtc->base.hwmode = crtc->config->base.adjusted_mode;
- readout_plane_state(crtc, to_intel_crtc_state(crtc->base.state));
+ readout_plane_state(crtc);
DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n",
crtc->base.base.id,
connector->base.name,
connector->base.encoder ? "enabled" : "disabled");
}
+
+ for_each_intel_crtc(dev, crtc) {
+ crtc->base.hwmode = crtc->config->base.adjusted_mode;
+
+ memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
+ if (crtc->base.state->active) {
+ intel_mode_from_pipe_config(&crtc->base.mode, crtc->config);
+ intel_mode_from_pipe_config(&crtc->base.state->adjusted_mode, crtc->config);
+ WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
+
+ /*
+ * The initial mode needs to be set in order to keep
+ * the atomic core happy. It wants a valid mode if the
+ * crtc's enabled, so we do the above call.
+ *
+ * At this point some state updated by the connectors
+ * in their ->detect() callback has not run yet, so
+ * no recalculation can be done yet.
+ *
+ * Even if we could do a recalculation and modeset
+ * right now it would cause a double modeset if
+ * fbdev or userspace chooses a different initial mode.
+ *
+ * If that happens, someone indicated they wanted a
+ * mode change, which means it's safe to do a full
+ * recalculation.
+ */
+ crtc->base.state->mode.private_flags = I915_MODE_FLAG_INHERITED;
+
+ drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode);
+ update_scanline_offset(crtc);
+ }
+ }
}
/* Scan out the current hw modeset state,
static void vlv_steal_power_sequencer(struct drm_device *dev,
enum pipe pipe);
+static unsigned int intel_dp_unused_lane_mask(int lane_count)
+{
+ return ~((1 << lane_count) - 1) & 0xf;
+}
+
static int
intel_dp_max_link_bw(struct intel_dp *intel_dp)
{
dst[i] = src >> ((3-i) * 8);
}
-/* hrawclock is 1/4 the FSB frequency */
-static int
-intel_hrawclk(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- uint32_t clkcfg;
-
- /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */
- if (IS_VALLEYVIEW(dev))
- return 200;
-
- clkcfg = I915_READ(CLKCFG);
- switch (clkcfg & CLKCFG_FSB_MASK) {
- case CLKCFG_FSB_400:
- return 100;
- case CLKCFG_FSB_533:
- return 133;
- case CLKCFG_FSB_667:
- return 166;
- case CLKCFG_FSB_800:
- return 200;
- case CLKCFG_FSB_1067:
- return 266;
- case CLKCFG_FSB_1333:
- return 333;
- /* these two are just a guess; one of them might be right */
- case CLKCFG_FSB_1600:
- case CLKCFG_FSB_1600_ALT:
- return 400;
- default:
- return 133;
- }
-}
-
static void
intel_dp_init_panel_power_sequencer(struct drm_device *dev,
struct intel_dp *intel_dp);
struct drm_device *dev = intel_dig_port->base.base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
enum pipe pipe = intel_dp->pps_pipe;
- bool pll_enabled;
+ bool pll_enabled, release_cl_override = false;
+ enum dpio_phy phy = DPIO_PHY(pipe);
+ enum dpio_channel ch = vlv_pipe_to_channel(pipe);
uint32_t DP;
if (WARN(I915_READ(intel_dp->output_reg) & DP_PORT_EN,
* The DPLL for the pipe must be enabled for this to work.
* So enable temporarily it if it's not already enabled.
*/
- if (!pll_enabled)
+ if (!pll_enabled) {
+ release_cl_override = IS_CHERRYVIEW(dev) &&
+ !chv_phy_powergate_ch(dev_priv, phy, ch, true);
+
vlv_force_pll_on(dev, pipe, IS_CHERRYVIEW(dev) ?
&chv_dpll[0].dpll : &vlv_dpll[0].dpll);
+ }
/*
* Similar magic as in intel_dp_enable_port().
I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN);
POSTING_READ(intel_dp->output_reg);
- if (!pll_enabled)
+ if (!pll_enabled) {
vlv_force_pll_off(dev, pipe);
+
+ if (release_cl_override)
+ chv_phy_powergate_ch(dev_priv, phy, ch, false);
+ }
}
static enum pipe
edp_notifier);
struct drm_device *dev = intel_dp_to_dev(intel_dp);
struct drm_i915_private *dev_priv = dev->dev_private;
- u32 pp_div;
- u32 pp_ctrl_reg, pp_div_reg;
if (!is_edp(intel_dp) || code != SYS_RESTART)
return 0;
if (IS_VALLEYVIEW(dev)) {
enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
+ u32 pp_ctrl_reg, pp_div_reg;
+ u32 pp_div;
pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe);
pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe);
switch (msg->request & ~DP_AUX_I2C_MOT) {
case DP_AUX_NATIVE_WRITE:
case DP_AUX_I2C_WRITE:
+ case DP_AUX_I2C_WRITE_STATUS_UPDATE:
txsize = msg->size ? HEADER_SIZE + msg->size : BARE_ADDRESS_SIZE;
rxsize = 2; /* 0 or 1 data bytes */
return rate_to_index(rate, intel_dp->sink_rates);
}
+static void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
+ uint8_t *link_bw, uint8_t *rate_select)
+{
+ if (intel_dp->num_sink_rates) {
+ *link_bw = 0;
+ *rate_select =
+ intel_dp_rate_select(intel_dp, port_clock);
+ } else {
+ *link_bw = drm_dp_link_rate_to_bw_code(port_clock);
+ *rate_select = 0;
+ }
+}
+
bool
intel_dp_compute_config(struct intel_encoder *encoder,
struct intel_crtc_state *pipe_config)
int link_avail, link_clock;
int common_rates[DP_MAX_SUPPORTED_RATES] = {};
int common_len;
+ uint8_t link_bw, rate_select;
common_len = intel_dp_common_rates(intel_dp, common_rates);
* CEA-861-E - 5.1 Default Encoding Parameters
* VESA DisplayPort Ver.1.2a - 5.1.1.1 Video Colorimetry
*/
- if (bpp != 18 && drm_match_cea_mode(adjusted_mode) > 1)
- intel_dp->color_range = DP_COLOR_RANGE_16_235;
- else
- intel_dp->color_range = 0;
- }
-
- if (intel_dp->color_range)
- pipe_config->limited_color_range = true;
-
- intel_dp->lane_count = lane_count;
-
- if (intel_dp->num_sink_rates) {
- intel_dp->link_bw = 0;
- intel_dp->rate_select =
- intel_dp_rate_select(intel_dp, common_rates[clock]);
+ pipe_config->limited_color_range =
+ bpp != 18 && drm_match_cea_mode(adjusted_mode) > 1;
} else {
- intel_dp->link_bw =
- drm_dp_link_rate_to_bw_code(common_rates[clock]);
- intel_dp->rate_select = 0;
+ pipe_config->limited_color_range =
+ intel_dp->limited_color_range;
}
+ pipe_config->lane_count = lane_count;
+
pipe_config->pipe_bpp = bpp;
pipe_config->port_clock = common_rates[clock];
- DRM_DEBUG_KMS("DP link bw %02x lane count %d clock %d bpp %d\n",
- intel_dp->link_bw, intel_dp->lane_count,
+ intel_dp_compute_rate(intel_dp, pipe_config->port_clock,
+ &link_bw, &rate_select);
+
+ DRM_DEBUG_KMS("DP link bw %02x rate select %02x lane count %d clock %d bpp %d\n",
+ link_bw, rate_select, pipe_config->lane_count,
pipe_config->port_clock, bpp);
DRM_DEBUG_KMS("DP link bw required %i available %i\n",
mode_rate, link_avail);
udelay(500);
}
+void intel_dp_set_link_params(struct intel_dp *intel_dp,
+ const struct intel_crtc_state *pipe_config)
+{
+ intel_dp->link_rate = pipe_config->port_clock;
+ intel_dp->lane_count = pipe_config->lane_count;
+}
+
static void intel_dp_prepare(struct intel_encoder *encoder)
{
struct drm_device *dev = encoder->base.dev;
struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
enum port port = dp_to_dig_port(intel_dp)->port;
struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
- struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+
+ intel_dp_set_link_params(intel_dp, crtc->config);
/*
* There are four kinds of DP registers:
/* Handle DP bits in common between all three register formats */
intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
- intel_dp->DP |= DP_PORT_WIDTH(intel_dp->lane_count);
+ intel_dp->DP |= DP_PORT_WIDTH(crtc->config->lane_count);
if (crtc->config->has_audio)
intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE;
trans_dp &= ~TRANS_DP_ENH_FRAMING;
I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp);
} else {
- if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev))
- intel_dp->DP |= intel_dp->color_range;
+ if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
+ crtc->config->limited_color_range)
+ intel_dp->DP |= DP_COLOR_RANGE_16_235;
if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
intel_dp->DP |= DP_SYNC_HS_HIGH;
pipe_config->has_audio = tmp & DP_AUDIO_OUTPUT_ENABLE && port != PORT_A;
if (HAS_PCH_CPT(dev) && port != PORT_A) {
- tmp = I915_READ(TRANS_DP_CTL(crtc->pipe));
- if (tmp & TRANS_DP_HSYNC_ACTIVE_HIGH)
+ u32 trans_dp = I915_READ(TRANS_DP_CTL(crtc->pipe));
+
+ if (trans_dp & TRANS_DP_HSYNC_ACTIVE_HIGH)
flags |= DRM_MODE_FLAG_PHSYNC;
else
flags |= DRM_MODE_FLAG_NHSYNC;
- if (tmp & TRANS_DP_VSYNC_ACTIVE_HIGH)
+ if (trans_dp & TRANS_DP_VSYNC_ACTIVE_HIGH)
flags |= DRM_MODE_FLAG_PVSYNC;
else
flags |= DRM_MODE_FLAG_NVSYNC;
pipe_config->has_dp_encoder = true;
+ pipe_config->lane_count =
+ ((tmp & DP_PORT_WIDTH_MASK) >> DP_PORT_WIDTH_SHIFT) + 1;
+
intel_dp_get_m_n(crtc, pipe_config);
if (port == PORT_A) {
intel_dp_link_down(intel_dp);
}
-static void chv_post_disable_dp(struct intel_encoder *encoder)
+static void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+ bool reset)
{
- struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
- struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
- struct drm_device *dev = encoder->base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_crtc *intel_crtc =
- to_intel_crtc(encoder->base.crtc);
- enum dpio_channel ch = vlv_dport_to_channel(dport);
- enum pipe pipe = intel_crtc->pipe;
- u32 val;
+ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+ enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
+ struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+ enum pipe pipe = crtc->pipe;
+ uint32_t val;
- intel_dp_link_down(intel_dp);
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
+ if (reset)
+ val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
+ else
+ val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
- mutex_lock(&dev_priv->sb_lock);
+ if (crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
+ if (reset)
+ val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
+ else
+ val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
+ }
- /* Propagate soft reset to data lane reset */
val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
val |= CHV_PCS_REQ_SOFTRESET_EN;
+ if (reset)
+ val &= ~DPIO_PCS_CLK_SOFT_RESET;
+ else
+ val |= DPIO_PCS_CLK_SOFT_RESET;
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
- val |= CHV_PCS_REQ_SOFTRESET_EN;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
+ if (crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
+ val |= CHV_PCS_REQ_SOFTRESET_EN;
+ if (reset)
+ val &= ~DPIO_PCS_CLK_SOFT_RESET;
+ else
+ val |= DPIO_PCS_CLK_SOFT_RESET;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
+ }
+}
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
- val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
+static void chv_post_disable_dp(struct intel_encoder *encoder)
+{
+ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+ struct drm_device *dev = encoder->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ intel_dp_link_down(intel_dp);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
- val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
+ mutex_lock(&dev_priv->sb_lock);
+
+ /* Assert data lane reset */
+ chv_data_lane_soft_reset(encoder, true);
mutex_unlock(&dev_priv->sb_lock);
}
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
uint32_t dp_reg = I915_READ(intel_dp->output_reg);
- unsigned int lane_mask = 0x0;
if (WARN_ON(dp_reg & DP_PORT_EN))
return;
pps_unlock(intel_dp);
- if (IS_VALLEYVIEW(dev))
+ if (IS_VALLEYVIEW(dev)) {
+ unsigned int lane_mask = 0x0;
+
+ if (IS_CHERRYVIEW(dev))
+ lane_mask = intel_dp_unused_lane_mask(crtc->config->lane_count);
+
vlv_wait_port_ready(dev_priv, dp_to_dig_port(intel_dp),
lane_mask);
+ }
intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
intel_dp_start_link_train(intel_dp);
- intel_dp_complete_link_train(intel_dp);
intel_dp_stop_link_train(intel_dp);
if (crtc->config->has_audio) {
val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
- val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
-
- /* Deassert soft data lane reset*/
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
- val |= CHV_PCS_REQ_SOFTRESET_EN;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
- val |= CHV_PCS_REQ_SOFTRESET_EN;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
- val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
- val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
+ if (intel_crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
+ val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
+ }
/* Program Tx lane latency optimal setting*/
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < intel_crtc->config->lane_count; i++) {
/* Set the upar bit */
- data = (i == 1) ? 0x0 : 0x1;
+ if (intel_crtc->config->lane_count == 1)
+ data = 0x0;
+ else
+ data = (i == 1) ? 0x0 : 0x1;
vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i),
data << DPIO_UPAR_SHIFT);
}
val |= DPIO_TX2_STAGGER_MASK(0x1f);
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
- val |= DPIO_TX2_STAGGER_MASK(0x1f);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
+ if (intel_crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
+ val |= DPIO_TX2_STAGGER_MASK(0x1f);
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
+ }
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch),
DPIO_LANESTAGGER_STRAP(stagger) |
DPIO_TX1_STAGGER_MULT(6) |
DPIO_TX2_STAGGER_MULT(0));
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
- DPIO_LANESTAGGER_STRAP(stagger) |
- DPIO_LANESTAGGER_STRAP_OVRD |
- DPIO_TX1_STAGGER_MASK(0x1f) |
- DPIO_TX1_STAGGER_MULT(7) |
- DPIO_TX2_STAGGER_MULT(5));
+ if (intel_crtc->config->lane_count > 2) {
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
+ DPIO_LANESTAGGER_STRAP(stagger) |
+ DPIO_LANESTAGGER_STRAP_OVRD |
+ DPIO_TX1_STAGGER_MASK(0x1f) |
+ DPIO_TX1_STAGGER_MULT(7) |
+ DPIO_TX2_STAGGER_MULT(5));
+ }
+
+ /* Deassert data lane reset */
+ chv_data_lane_soft_reset(encoder, false);
mutex_unlock(&dev_priv->sb_lock);
intel_enable_dp(encoder);
+
+ /* Second common lane will stay alive on its own now */
+ if (dport->release_cl2_override) {
+ chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false);
+ dport->release_cl2_override = false;
+ }
}
static void chv_dp_pre_pll_enable(struct intel_encoder *encoder)
to_intel_crtc(encoder->base.crtc);
enum dpio_channel ch = vlv_dport_to_channel(dport);
enum pipe pipe = intel_crtc->pipe;
+ unsigned int lane_mask =
+ intel_dp_unused_lane_mask(intel_crtc->config->lane_count);
u32 val;
intel_dp_prepare(encoder);
+ /*
+ * Must trick the second common lane into life.
+ * Otherwise we can't even access the PLL.
+ */
+ if (ch == DPIO_CH0 && pipe == PIPE_B)
+ dport->release_cl2_override =
+ !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true);
+
+ chv_phy_powergate_lanes(encoder, true, lane_mask);
+
mutex_lock(&dev_priv->sb_lock);
+ /* Assert data lane reset */
+ chv_data_lane_soft_reset(encoder, true);
+
/* program left/right clock distribution */
if (pipe != PIPE_B) {
val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
val |= CHV_PCS_USEDCLKCHANNEL;
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
- val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
- if (pipe != PIPE_B)
- val &= ~CHV_PCS_USEDCLKCHANNEL;
- else
- val |= CHV_PCS_USEDCLKCHANNEL;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val);
+ if (intel_crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
+ val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
+ if (pipe != PIPE_B)
+ val &= ~CHV_PCS_USEDCLKCHANNEL;
+ else
+ val |= CHV_PCS_USEDCLKCHANNEL;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val);
+ }
/*
* This a a bit weird since generally CL
mutex_unlock(&dev_priv->sb_lock);
}
+static void chv_dp_post_pll_disable(struct intel_encoder *encoder)
+{
+ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+ enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
+ u32 val;
+
+ mutex_lock(&dev_priv->sb_lock);
+
+ /* disable left/right clock distribution */
+ if (pipe != PIPE_B) {
+ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
+ val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
+ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
+ } else {
+ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
+ val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
+ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
+ }
+
+ mutex_unlock(&dev_priv->sb_lock);
+
+ /*
+ * Leave the power down bit cleared for at least one
+ * lane so that chv_powergate_phy_ch() will power
+ * on something when the channel is otherwise unused.
+ * When the port is off and the override is removed
+ * the lanes power down anyway, so otherwise it doesn't
+ * really matter what the state of power down bits is
+ * after this.
+ */
+ chv_phy_powergate_lanes(encoder, false, 0x0);
+}
+
/*
* Native read with retry for link status and receiver capability reads for
* cases where the sink may still be asleep.
return 0;
}
+static bool chv_need_uniq_trans_scale(uint8_t train_set)
+{
+ return (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) == DP_TRAIN_PRE_EMPH_LEVEL_0 &&
+ (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) == DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
+}
+
static uint32_t chv_signal_levels(struct intel_dp *intel_dp)
{
struct drm_device *dev = intel_dp_to_dev(intel_dp);
val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
- val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
- val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
- val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
+ if (intel_crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
+ val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
+ val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
+ val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
+ }
val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch));
val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch));
- val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
- val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val);
+ if (intel_crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch));
+ val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
+ val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val);
+ }
/* Program swing deemph */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < intel_crtc->config->lane_count; i++) {
val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i));
val &= ~DPIO_SWING_DEEMPH9P5_MASK;
val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT;
}
/* Program swing margin */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < intel_crtc->config->lane_count; i++) {
val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
+
val &= ~DPIO_SWING_MARGIN000_MASK;
val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT;
+
+ /*
+ * Supposedly this value shouldn't matter when unique transition
+ * scale is disabled, but in fact it does matter. Let's just
+ * always program the same value and hope it's OK.
+ */
+ val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
+ val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT;
+
vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
}
- /* Disable unique transition scale */
- for (i = 0; i < 4; i++) {
+ /*
+ * The document said it needs to set bit 27 for ch0 and bit 26
+ * for ch1. Might be a typo in the doc.
+ * For now, for this unique transition scale selection, set bit
+ * 27 for ch0 and ch1.
+ */
+ for (i = 0; i < intel_crtc->config->lane_count; i++) {
val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i));
- val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
- vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
- }
-
- if (((train_set & DP_TRAIN_PRE_EMPHASIS_MASK)
- == DP_TRAIN_PRE_EMPH_LEVEL_0) &&
- ((train_set & DP_TRAIN_VOLTAGE_SWING_MASK)
- == DP_TRAIN_VOLTAGE_SWING_LEVEL_3)) {
-
- /*
- * The document said it needs to set bit 27 for ch0 and bit 26
- * for ch1. Might be a typo in the doc.
- * For now, for this unique transition scale selection, set bit
- * 27 for ch0 and ch1.
- */
- for (i = 0; i < 4; i++) {
- val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i));
+ if (chv_need_uniq_trans_scale(train_set))
val |= DPIO_TX_UNIQ_TRANS_SCALE_EN;
- vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
- }
-
- for (i = 0; i < 4; i++) {
- val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
- val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
- val |= (0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT);
- vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
- }
+ else
+ val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
+ vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
}
/* Start swing calculation */
val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
- val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
-
- /* LRC Bypass */
- val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
- val |= DPIO_LRC_BYPASS;
- vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, val);
+ if (intel_crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
+ val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
+ }
mutex_unlock(&dev_priv->sb_lock);
uint8_t dp_train_pat)
{
struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
- struct drm_device *dev = intel_dig_port->base.base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_private *dev_priv =
+ to_i915(intel_dig_port->base.base.dev);
uint8_t buf[sizeof(intel_dp->train_set) + 1];
int ret, len;
const uint8_t link_status[DP_LINK_STATUS_SIZE])
{
struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
- struct drm_device *dev = intel_dig_port->base.base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_private *dev_priv =
+ to_i915(intel_dig_port->base.base.dev);
int ret;
intel_get_adjust_train(intel_dp, link_status);
}
/* Enable corresponding port and start training pattern 1 */
-void
-intel_dp_start_link_train(struct intel_dp *intel_dp)
+static void
+intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
{
struct drm_encoder *encoder = &dp_to_dig_port(intel_dp)->base.base;
struct drm_device *dev = encoder->dev;
int voltage_tries, loop_tries;
uint32_t DP = intel_dp->DP;
uint8_t link_config[2];
+ uint8_t link_bw, rate_select;
if (HAS_DDI(dev))
intel_ddi_prepare_link_retrain(encoder);
+ intel_dp_compute_rate(intel_dp, intel_dp->link_rate,
+ &link_bw, &rate_select);
+
/* Write the link configuration data */
- link_config[0] = intel_dp->link_bw;
+ link_config[0] = link_bw;
link_config[1] = intel_dp->lane_count;
if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2);
if (intel_dp->num_sink_rates)
drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET,
- &intel_dp->rate_select, 1);
+ &rate_select, 1);
link_config[0] = 0;
link_config[1] = DP_SET_ANSI_8B10B;
intel_dp->DP = DP;
}
-void
-intel_dp_complete_link_train(struct intel_dp *intel_dp)
+static void
+intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
{
+ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+ struct drm_device *dev = dig_port->base.base.dev;
bool channel_eq = false;
int tries, cr_tries;
uint32_t DP = intel_dp->DP;
uint32_t training_pattern = DP_TRAINING_PATTERN_2;
- /* Training Pattern 3 for HBR2 ot 1.2 devices that support it*/
- if (intel_dp->link_bw == DP_LINK_BW_5_4 || intel_dp->use_tps3)
+ /*
+ * Training Pattern 3 for HBR2 or 1.2 devices that support it.
+ *
+ * Intel platforms that support HBR2 also support TPS3. TPS3 support is
+ * also mandatory for downstream devices that support HBR2.
+ *
+ * Due to WaDisableHBR2 SKL < B0 is the only exception where TPS3 is
+ * supported but still not enabled.
+ */
+ if (intel_dp_source_supports_hbr2(dev) &&
+ drm_dp_tps3_supported(intel_dp->dpcd))
training_pattern = DP_TRAINING_PATTERN_3;
+ else if (intel_dp->link_rate == 540000)
+ DRM_ERROR("5.4 Gbps link rate without HBR2/TPS3 support\n");
/* channel equalization */
if (!intel_dp_set_link_train(intel_dp, &DP,
}
/* Make sure clock is still ok */
- if (!drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) {
+ if (!drm_dp_clock_recovery_ok(link_status,
+ intel_dp->lane_count)) {
intel_dp->train_set_valid = false;
- intel_dp_start_link_train(intel_dp);
+ intel_dp_link_training_clock_recovery(intel_dp);
intel_dp_set_link_train(intel_dp, &DP,
training_pattern |
DP_LINK_SCRAMBLING_DISABLE);
continue;
}
- if (drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) {
+ if (drm_dp_channel_eq_ok(link_status,
+ intel_dp->lane_count)) {
channel_eq = true;
break;
}
/* Try 5 times, then try clock recovery if that fails */
if (tries > 5) {
intel_dp->train_set_valid = false;
- intel_dp_start_link_train(intel_dp);
+ intel_dp_link_training_clock_recovery(intel_dp);
intel_dp_set_link_train(intel_dp, &DP,
training_pattern |
DP_LINK_SCRAMBLING_DISABLE);
DP_TRAINING_PATTERN_DISABLE);
}
+void
+intel_dp_start_link_train(struct intel_dp *intel_dp)
+{
+ intel_dp_link_training_clock_recovery(intel_dp);
+ intel_dp_link_training_channel_equalization(intel_dp);
+}
+
static void
intel_dp_link_down(struct intel_dp *intel_dp)
{
}
}
- /* Training Pattern 3 support, Intel platforms that support HBR2 alone
- * have support for TP3 hence that check is used along with dpcd check
- * to ensure TP3 can be enabled.
- * SKL < B0: due it's WaDisableHBR2 is the only exception where TP3 is
- * supported but still not enabled.
- */
- if (intel_dp->dpcd[DP_DPCD_REV] >= 0x12 &&
- intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED &&
- intel_dp_source_supports_hbr2(dev)) {
- intel_dp->use_tps3 = true;
- DRM_DEBUG_KMS("Displayport TPS3 supported\n");
- } else
- intel_dp->use_tps3 = false;
+ DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n",
+ yesno(intel_dp_source_supports_hbr2(dev)),
+ yesno(drm_dp_tps3_supported(intel_dp->dpcd)));
/* Intermediate frequency support */
if (is_edp(intel_dp) &&
return intel_dp->is_mst;
}
-static void intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
+static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
{
struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
u8 buf;
+ int ret = 0;
if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0) {
DRM_DEBUG_KMS("Sink CRC couldn't be stopped properly\n");
- return;
+ ret = -EIO;
+ goto out;
}
if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
- buf & ~DP_TEST_SINK_START) < 0)
+ buf & ~DP_TEST_SINK_START) < 0) {
DRM_DEBUG_KMS("Sink CRC couldn't be stopped properly\n");
+ ret = -EIO;
+ goto out;
+ }
+ intel_dp->sink_crc.started = false;
+ out:
hsw_enable_ips(intel_crtc);
+ return ret;
}
static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
u8 buf;
+ int ret;
+
+ if (intel_dp->sink_crc.started) {
+ ret = intel_dp_sink_crc_stop(intel_dp);
+ if (ret)
+ return ret;
+ }
if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0)
return -EIO;
if (!(buf & DP_TEST_CRC_SUPPORTED))
return -ENOTTY;
+ intel_dp->sink_crc.last_count = buf & DP_TEST_COUNT_MASK;
+
if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0)
return -EIO;
return -EIO;
}
+ intel_dp->sink_crc.started = true;
return 0;
}
struct drm_device *dev = dig_port->base.base.dev;
struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
u8 buf;
- int test_crc_count;
+ int count, ret;
int attempts = 6;
- int ret;
+ bool old_equal_new;
ret = intel_dp_sink_crc_start(intel_dp);
if (ret)
return ret;
- if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) {
- ret = -EIO;
- goto stop;
- }
-
- test_crc_count = buf & DP_TEST_COUNT_MASK;
-
do {
+ intel_wait_for_vblank(dev, intel_crtc->pipe);
+
if (drm_dp_dpcd_readb(&intel_dp->aux,
DP_TEST_SINK_MISC, &buf) < 0) {
ret = -EIO;
goto stop;
}
- intel_wait_for_vblank(dev, intel_crtc->pipe);
- } while (--attempts && (buf & DP_TEST_COUNT_MASK) == test_crc_count);
+ count = buf & DP_TEST_COUNT_MASK;
+
+ /*
+ * Count might be reset during the loop. In this case
+ * last known count needs to be reset as well.
+ */
+ if (count == 0)
+ intel_dp->sink_crc.last_count = 0;
+
+ if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0) {
+ ret = -EIO;
+ goto stop;
+ }
+
+ old_equal_new = (count == intel_dp->sink_crc.last_count &&
+ !memcmp(intel_dp->sink_crc.last_crc, crc,
+ 6 * sizeof(u8)));
+
+ } while (--attempts && (count == 0 || old_equal_new));
+
+ intel_dp->sink_crc.last_count = buf & DP_TEST_COUNT_MASK;
+ memcpy(intel_dp->sink_crc.last_crc, crc, 6 * sizeof(u8));
if (attempts == 0) {
- DRM_DEBUG_KMS("Panel is unable to calculate CRC after 6 vblanks\n");
- ret = -ETIMEDOUT;
- goto stop;
+ if (old_equal_new) {
+ DRM_DEBUG_KMS("Unreliable Sink CRC counter: Current returned CRC is identical to the previous one\n");
+ } else {
+ DRM_ERROR("Panel is unable to calculate any CRC after 6 vblanks\n");
+ ret = -ETIMEDOUT;
+ goto stop;
+ }
}
- if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0)
- ret = -EIO;
stop:
intel_dp_sink_crc_stop(intel_dp);
return ret;
if (bret == true) {
/* check link status - esi[10] = 0x200c */
- if (intel_dp->active_mst_links && !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) {
+ if (intel_dp->active_mst_links &&
+ !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) {
DRM_DEBUG_KMS("channel EQ not ok, retraining\n");
intel_dp_start_link_train(intel_dp);
- intel_dp_complete_link_train(intel_dp);
intel_dp_stop_link_train(intel_dp);
}
DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n",
intel_encoder->base.name);
intel_dp_start_link_train(intel_dp);
- intel_dp_complete_link_train(intel_dp);
intel_dp_stop_link_train(intel_dp);
}
}
return status;
}
-static enum drm_connector_status
-ironlake_dp_detect(struct intel_dp *intel_dp)
+static bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
+ struct intel_digital_port *port)
{
- struct drm_device *dev = intel_dp_to_dev(intel_dp);
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+ u32 bit;
- if (!ibx_digital_port_connected(dev_priv, intel_dig_port))
- return connector_status_disconnected;
+ switch (port->port) {
+ case PORT_A:
+ return true;
+ case PORT_B:
+ bit = SDE_PORTB_HOTPLUG;
+ break;
+ case PORT_C:
+ bit = SDE_PORTC_HOTPLUG;
+ break;
+ case PORT_D:
+ bit = SDE_PORTD_HOTPLUG;
+ break;
+ default:
+ MISSING_CASE(port->port);
+ return false;
+ }
- return intel_dp_detect_dpcd(intel_dp);
+ return I915_READ(SDEISR) & bit;
+}
+
+static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv,
+ struct intel_digital_port *port)
+{
+ u32 bit;
+
+ switch (port->port) {
+ case PORT_A:
+ return true;
+ case PORT_B:
+ bit = SDE_PORTB_HOTPLUG_CPT;
+ break;
+ case PORT_C:
+ bit = SDE_PORTC_HOTPLUG_CPT;
+ break;
+ case PORT_D:
+ bit = SDE_PORTD_HOTPLUG_CPT;
+ break;
+ case PORT_E:
+ bit = SDE_PORTE_HOTPLUG_SPT;
+ break;
+ default:
+ MISSING_CASE(port->port);
+ return false;
+ }
+
+ return I915_READ(SDEISR) & bit;
+}
+
+static bool g4x_digital_port_connected(struct drm_i915_private *dev_priv,
+ struct intel_digital_port *port)
+{
+ u32 bit;
+
+ switch (port->port) {
+ case PORT_B:
+ bit = PORTB_HOTPLUG_LIVE_STATUS_G4X;
+ break;
+ case PORT_C:
+ bit = PORTC_HOTPLUG_LIVE_STATUS_G4X;
+ break;
+ case PORT_D:
+ bit = PORTD_HOTPLUG_LIVE_STATUS_G4X;
+ break;
+ default:
+ MISSING_CASE(port->port);
+ return false;
+ }
+
+ return I915_READ(PORT_HOTPLUG_STAT) & bit;
}
-static int g4x_digital_port_connected(struct drm_device *dev,
+static bool vlv_digital_port_connected(struct drm_i915_private *dev_priv,
+ struct intel_digital_port *port)
+{
+ u32 bit;
+
+ switch (port->port) {
+ case PORT_B:
+ bit = PORTB_HOTPLUG_LIVE_STATUS_VLV;
+ break;
+ case PORT_C:
+ bit = PORTC_HOTPLUG_LIVE_STATUS_VLV;
+ break;
+ case PORT_D:
+ bit = PORTD_HOTPLUG_LIVE_STATUS_VLV;
+ break;
+ default:
+ MISSING_CASE(port->port);
+ return false;
+ }
+
+ return I915_READ(PORT_HOTPLUG_STAT) & bit;
+}
+
+static bool bxt_digital_port_connected(struct drm_i915_private *dev_priv,
struct intel_digital_port *intel_dig_port)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
- uint32_t bit;
+ struct intel_encoder *intel_encoder = &intel_dig_port->base;
+ enum port port;
+ u32 bit;
- if (IS_VALLEYVIEW(dev)) {
- switch (intel_dig_port->port) {
- case PORT_B:
- bit = PORTB_HOTPLUG_LIVE_STATUS_VLV;
- break;
- case PORT_C:
- bit = PORTC_HOTPLUG_LIVE_STATUS_VLV;
- break;
- case PORT_D:
- bit = PORTD_HOTPLUG_LIVE_STATUS_VLV;
- break;
- default:
- return -EINVAL;
- }
- } else {
- switch (intel_dig_port->port) {
- case PORT_B:
- bit = PORTB_HOTPLUG_LIVE_STATUS_G4X;
- break;
- case PORT_C:
- bit = PORTC_HOTPLUG_LIVE_STATUS_G4X;
- break;
- case PORT_D:
- bit = PORTD_HOTPLUG_LIVE_STATUS_G4X;
- break;
- default:
- return -EINVAL;
- }
+ intel_hpd_pin_to_port(intel_encoder->hpd_pin, &port);
+ switch (port) {
+ case PORT_A:
+ bit = BXT_DE_PORT_HP_DDIA;
+ break;
+ case PORT_B:
+ bit = BXT_DE_PORT_HP_DDIB;
+ break;
+ case PORT_C:
+ bit = BXT_DE_PORT_HP_DDIC;
+ break;
+ default:
+ MISSING_CASE(port);
+ return false;
}
- if ((I915_READ(PORT_HOTPLUG_STAT) & bit) == 0)
- return 0;
- return 1;
+ return I915_READ(GEN8_DE_PORT_ISR) & bit;
+}
+
+/*
+ * intel_digital_port_connected - is the specified port connected?
+ * @dev_priv: i915 private structure
+ * @port: the port to test
+ *
+ * Return %true if @port is connected, %false otherwise.
+ */
+bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
+ struct intel_digital_port *port)
+{
+ if (HAS_PCH_IBX(dev_priv))
+ return ibx_digital_port_connected(dev_priv, port);
+ if (HAS_PCH_SPLIT(dev_priv))
+ return cpt_digital_port_connected(dev_priv, port);
+ else if (IS_BROXTON(dev_priv))
+ return bxt_digital_port_connected(dev_priv, port);
+ else if (IS_VALLEYVIEW(dev_priv))
+ return vlv_digital_port_connected(dev_priv, port);
+ else
+ return g4x_digital_port_connected(dev_priv, port);
+}
+
+static enum drm_connector_status
+ironlake_dp_detect(struct intel_dp *intel_dp)
+{
+ struct drm_device *dev = intel_dp_to_dev(intel_dp);
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+
+ if (!intel_digital_port_connected(dev_priv, intel_dig_port))
+ return connector_status_disconnected;
+
+ return intel_dp_detect_dpcd(intel_dp);
}
static enum drm_connector_status
{
struct drm_device *dev = intel_dp_to_dev(intel_dp);
struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
- int ret;
/* Can't disconnect eDP, but you can close the lid... */
if (is_edp(intel_dp)) {
return status;
}
- ret = g4x_digital_port_connected(dev, intel_dig_port);
- if (ret == -EINVAL)
- return connector_status_unknown;
- else if (ret == 0)
+ if (!intel_digital_port_connected(dev->dev_private, intel_dig_port))
return connector_status_disconnected;
return intel_dp_detect_dpcd(intel_dp);
if (property == dev_priv->broadcast_rgb_property) {
bool old_auto = intel_dp->color_range_auto;
- uint32_t old_range = intel_dp->color_range;
+ bool old_range = intel_dp->limited_color_range;
switch (val) {
case INTEL_BROADCAST_RGB_AUTO:
break;
case INTEL_BROADCAST_RGB_FULL:
intel_dp->color_range_auto = false;
- intel_dp->color_range = 0;
+ intel_dp->limited_color_range = false;
break;
case INTEL_BROADCAST_RGB_LIMITED:
intel_dp->color_range_auto = false;
- intel_dp->color_range = DP_COLOR_RANGE_16_235;
+ intel_dp->limited_color_range = true;
break;
default:
return -EINVAL;
}
if (old_auto == intel_dp->color_range_auto &&
- old_range == intel_dp->color_range)
+ old_range == intel_dp->limited_color_range)
return 0;
goto done;
/* indicate that we need to restart link training */
intel_dp->train_set_valid = false;
- if (HAS_PCH_SPLIT(dev)) {
- if (!ibx_digital_port_connected(dev_priv, intel_dig_port))
- goto mst_fail;
- } else {
- if (g4x_digital_port_connected(dev, intel_dig_port) != 1)
- goto mst_fail;
- }
+ if (!intel_digital_port_connected(dev_priv, intel_dig_port))
+ goto mst_fail;
if (!intel_dp_get_dpcd(intel_dp)) {
goto mst_fail;
[PORT_E] = DVO_PORT_DPE,
};
+ /*
+ * eDP not supported on g4x. so bail out early just
+ * for a bit extra safety in case the VBT is bonkers.
+ */
+ if (INTEL_INFO(dev)->gen < 5)
+ return false;
+
if (port == PORT_A)
return true;
struct intel_dp *intel_dp = dev_priv->drrs.dp;
struct intel_crtc_state *config = NULL;
struct intel_crtc *intel_crtc = NULL;
- u32 reg, val;
enum drrs_refresh_rate_type index = DRRS_HIGH_RR;
if (refresh_rate <= 0) {
DRM_ERROR("Unsupported refreshrate type\n");
}
} else if (INTEL_INFO(dev)->gen > 6) {
- reg = PIPECONF(intel_crtc->config->cpu_transcoder);
- val = I915_READ(reg);
+ u32 reg = PIPECONF(intel_crtc->config->cpu_transcoder);
+ u32 val;
+ val = I915_READ(reg);
if (index > DRRS_HIGH_RR) {
if (IS_VALLEYVIEW(dev))
val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV;
}
intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode);
- intel_connector->panel.backlight_power = intel_edp_backlight_power;
+ intel_connector->panel.backlight.power = intel_edp_backlight_power;
intel_panel_setup_backlight(connector, pipe);
return true;
break;
case PORT_B:
intel_encoder->hpd_pin = HPD_PORT_B;
+ if (IS_BROXTON(dev_priv) && (INTEL_REVID(dev) < BXT_REVID_B0))
+ intel_encoder->hpd_pin = HPD_PORT_A;
break;
case PORT_C:
intel_encoder->hpd_pin = HPD_PORT_C;
return;
intel_connector = intel_connector_alloc();
- if (!intel_connector) {
- kfree(intel_dig_port);
- return;
- }
+ if (!intel_connector)
+ goto err_connector_alloc;
intel_encoder = &intel_dig_port->base;
encoder = &intel_encoder->base;
intel_encoder->pre_enable = chv_pre_enable_dp;
intel_encoder->enable = vlv_enable_dp;
intel_encoder->post_disable = chv_post_disable_dp;
+ intel_encoder->post_pll_disable = chv_dp_post_pll_disable;
} else if (IS_VALLEYVIEW(dev)) {
intel_encoder->pre_pll_enable = vlv_dp_pre_pll_enable;
intel_encoder->pre_enable = vlv_pre_enable_dp;
intel_dig_port->hpd_pulse = intel_dp_hpd_pulse;
dev_priv->hotplug.irq_port[port] = intel_dig_port;
- if (!intel_dp_init_connector(intel_dig_port, intel_connector)) {
- drm_encoder_cleanup(encoder);
- kfree(intel_dig_port);
- kfree(intel_connector);
- }
+ if (!intel_dp_init_connector(intel_dig_port, intel_connector))
+ goto err_init_connector;
+
+ return;
+
+err_init_connector:
+ drm_encoder_cleanup(encoder);
+ kfree(intel_connector);
+err_connector_alloc:
+ kfree(intel_dig_port);
+
+ return;
}
void intel_dp_mst_suspend(struct drm_device *dev)
struct intel_dp *intel_dp = &intel_dig_port->dp;
struct drm_atomic_state *state;
int bpp, i;
- int lane_count, slots, rate;
- struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+ int lane_count, slots;
+ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
struct drm_connector *drm_connector;
struct intel_connector *connector, *found = NULL;
struct drm_connector_state *connector_state;
*/
lane_count = drm_dp_max_lane_count(intel_dp->dpcd);
- rate = intel_dp_max_link_rate(intel_dp);
- if (intel_dp->num_sink_rates) {
- intel_dp->link_bw = 0;
- intel_dp->rate_select = intel_dp_rate_select(intel_dp, rate);
- } else {
- intel_dp->link_bw = drm_dp_link_rate_to_bw_code(rate);
- intel_dp->rate_select = 0;
- }
-
- intel_dp->lane_count = lane_count;
+ pipe_config->lane_count = lane_count;
pipe_config->pipe_bpp = 24;
- pipe_config->port_clock = rate;
+ pipe_config->port_clock = intel_dp_max_link_rate(intel_dp);
state = pipe_config->base.state;
return false;
}
- mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->clock, bpp);
+ mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);
pipe_config->pbn = mst_pbn;
slots = drm_dp_find_vcpi_slots(&intel_dp->mst_mgr, mst_pbn);
if (intel_dp->active_mst_links == 0) {
enum port port = intel_ddi_get_encoder_port(encoder);
+ intel_dp_set_link_params(intel_dp, intel_crtc->config);
+
/* FIXME: add support for SKL */
if (INTEL_INFO(dev)->gen < 9)
I915_WRITE(PORT_CLK_SEL(port),
intel_dp_start_link_train(intel_dp);
- intel_dp_complete_link_train(intel_dp);
intel_dp_stop_link_train(intel_dp);
}
break;
}
pipe_config->base.adjusted_mode.flags |= flags;
+
+ pipe_config->lane_count =
+ ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
+
intel_dp_get_m_n(crtc, pipe_config);
intel_ddi_clock_get(&intel_dig_port->base, pipe_config);
drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0);
drm_mode_connector_set_path_property(connector, pathprop);
+ return connector;
+}
+
+static void intel_dp_register_mst_connector(struct drm_connector *connector)
+{
+ struct intel_connector *intel_connector = to_intel_connector(connector);
+ struct drm_device *dev = connector->dev;
drm_modeset_lock_all(dev);
intel_connector_add_to_fbdev(intel_connector);
drm_modeset_unlock_all(dev);
drm_connector_register(&intel_connector->base);
- return connector;
}
static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
static struct drm_dp_mst_topology_cbs mst_cbs = {
.add_connector = intel_dp_add_mst_connector,
+ .register_connector = intel_dp_register_mst_connector,
.destroy_connector = intel_dp_destroy_mst_connector,
.hotplug = intel_dp_mst_hotplug,
};
void (*mode_set)(struct intel_encoder *intel_encoder);
void (*disable)(struct intel_encoder *);
void (*post_disable)(struct intel_encoder *);
+ void (*post_pll_disable)(struct intel_encoder *);
/* Read out the current hw state of this connector, returning true if
* the encoder is active. If the encoder is enabled it also set the pipe
* it is connected to in the pipe parameter. */
bool active_low_pwm;
/* PWM chip */
+ bool util_pin_active_low; /* bxt+ */
+ u8 controller; /* bxt+ only */
struct pwm_device *pwm;
struct backlight_device *device;
- } backlight;
- void (*backlight_power)(struct intel_connector *, bool enable);
+ /* Connector and platform specific backlight functions */
+ int (*setup)(struct intel_connector *connector, enum pipe pipe);
+ uint32_t (*get)(struct intel_connector *connector);
+ void (*set)(struct intel_connector *connector, uint32_t level);
+ void (*disable)(struct intel_connector *connector);
+ void (*enable)(struct intel_connector *connector);
+ uint32_t (*hz_to_pwm)(struct intel_connector *connector,
+ uint32_t hz);
+ void (*power)(struct intel_connector *, bool enable);
+ } backlight;
};
struct intel_connector {
#define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS (1<<0) /* unreliable sync mode.flags */
unsigned long quirks;
+ bool update_pipe;
+
/* Pipe source size (ie. panel fitter input size)
* All planes will be positioned inside this space,
* and get clipped at the edges. */
/* Used by SDVO (and if we ever fix it, HDMI). */
unsigned pixel_multiplier;
+ uint8_t lane_count;
+
/* Panel fitter controls for gen2-gen4 + VLV */
struct {
u32 control;
* gen4+ this only adjusts up to a tile, offsets within a tile are
* handled in the hw itself (with the TILEOFF register). */
unsigned long dspaddr_offset;
+ int adjusted_x;
+ int adjusted_y;
struct drm_i915_gem_object *cursor_bo;
uint32_t cursor_addr;
int scanline_offset;
- unsigned start_vbl_count;
+ struct {
+ unsigned start_vbl_count;
+ ktime_t start_vbl_time;
+ int min_vbl, max_vbl;
+ int scanline_start;
+ } debug;
+
struct intel_crtc_atomic_commit atomic;
/* scalers available on this crtc */
struct intel_hdmi {
u32 hdmi_reg;
int ddc_bus;
- uint32_t color_range;
+ bool limited_color_range;
bool color_range_auto;
bool has_hdmi_sink;
bool has_audio;
enum hdmi_force_audio force_audio;
bool rgb_quant_range_selectable;
enum hdmi_picture_aspect aspect_ratio;
+ struct intel_connector *attached_connector;
void (*write_infoframe)(struct drm_encoder *encoder,
enum hdmi_infoframe_type type,
const void *frame, ssize_t len);
void (*set_infoframes)(struct drm_encoder *encoder,
bool enable,
- struct drm_display_mode *adjusted_mode);
+ const struct drm_display_mode *adjusted_mode);
bool (*infoframe_enabled)(struct drm_encoder *encoder);
};
M2_N2
};
+struct sink_crc {
+ bool started;
+ u8 last_crc[6];
+ int last_count;
+};
+
struct intel_dp {
uint32_t output_reg;
uint32_t aux_ch_ctl_reg;
uint32_t DP;
+ int link_rate;
+ uint8_t lane_count;
bool has_audio;
enum hdmi_force_audio force_audio;
- uint32_t color_range;
+ bool limited_color_range;
bool color_range_auto;
- uint8_t link_bw;
- uint8_t rate_select;
- uint8_t lane_count;
uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
/* sink rates as reported by DP_SUPPORTED_LINK_RATES */
uint8_t num_sink_rates;
int sink_rates[DP_MAX_SUPPORTED_RATES];
+ struct sink_crc sink_crc;
struct drm_dp_aux aux;
uint8_t train_set[4];
int panel_power_up_delay;
enum pipe pps_pipe;
struct edp_power_seq pps_delays;
- bool use_tps3;
bool can_mst; /* this port supports mst */
bool is_mst;
int active_mst_links;
struct intel_dp dp;
struct intel_hdmi hdmi;
enum irqreturn (*hpd_pulse)(struct intel_digital_port *, bool);
+ bool release_cl2_override;
};
struct intel_dp_mst_encoder {
void *port; /* store this opaque as its illegal to dereference it */
};
-static inline int
+static inline enum dpio_channel
vlv_dport_to_channel(struct intel_digital_port *dport)
{
switch (dport->port) {
}
}
-static inline int
+static inline enum dpio_phy
+vlv_dport_to_phy(struct intel_digital_port *dport)
+{
+ switch (dport->port) {
+ case PORT_B:
+ case PORT_C:
+ return DPIO_PHY0;
+ case PORT_D:
+ return DPIO_PHY1;
+ default:
+ BUG();
+ }
+}
+
+static inline enum dpio_channel
vlv_pipe_to_channel(enum pipe pipe)
{
switch (pipe) {
u32 flip_count;
u32 gtt_offset;
struct drm_i915_gem_request *flip_queued_req;
- int flip_queued_vblank;
- int flip_ready_vblank;
+ u32 flip_queued_vblank;
+ u32 flip_ready_vblank;
bool enable_stall_check;
};
extern const struct drm_plane_funcs intel_plane_funcs;
bool intel_has_pending_fb_unpin(struct drm_device *dev);
int intel_pch_rawclk(struct drm_device *dev);
+int intel_hrawclk(struct drm_device *dev);
void intel_mark_busy(struct drm_device *dev);
void intel_mark_idle(struct drm_device *dev);
void intel_crtc_restore_mode(struct drm_crtc *crtc);
int intel_connector_init(struct intel_connector *);
struct intel_connector *intel_connector_alloc(void);
bool intel_connector_get_hw_state(struct intel_connector *connector);
-bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
- struct intel_digital_port *port);
void intel_connector_attach_encoder(struct intel_connector *connector,
struct intel_encoder *encoder);
struct drm_encoder *intel_best_encoder(struct drm_connector *connector);
void intel_finish_page_flip_plane(struct drm_device *dev, int plane);
void intel_check_page_flip(struct drm_device *dev, int pipe);
int intel_prepare_plane_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state);
void intel_cleanup_plane_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *old_state);
int intel_plane_atomic_get_property(struct drm_plane *plane,
const struct drm_plane_state *state,
unsigned int
intel_tile_height(struct drm_device *dev, uint32_t pixel_format,
- uint64_t fb_format_modifier);
+ uint64_t fb_format_modifier, unsigned int plane);
static inline bool
intel_rotation_90_or_270(unsigned int rotation)
int skl_max_scale(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state);
unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane,
- struct drm_i915_gem_object *obj);
+ struct drm_i915_gem_object *obj,
+ unsigned int plane);
+
u32 skl_plane_ctl_format(uint32_t pixel_format);
u32 skl_plane_ctl_tiling(uint64_t fb_modifier);
u32 skl_plane_ctl_rotation(unsigned int rotation);
void intel_dp_init(struct drm_device *dev, int output_reg, enum port port);
bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
struct intel_connector *intel_connector);
+void intel_dp_set_link_params(struct intel_dp *intel_dp,
+ const struct intel_crtc_state *pipe_config);
void intel_dp_start_link_train(struct intel_dp *intel_dp);
-void intel_dp_complete_link_train(struct intel_dp *intel_dp);
void intel_dp_stop_link_train(struct intel_dp *intel_dp);
void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
void intel_dp_encoder_destroy(struct drm_encoder *encoder);
void intel_edp_drrs_invalidate(struct drm_device *dev,
unsigned frontbuffer_bits);
void intel_edp_drrs_flush(struct drm_device *dev, unsigned frontbuffer_bits);
+bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
+ struct intel_digital_port *port);
void hsw_dp_set_ddi_pll_sel(struct intel_crtc_state *pipe_config);
/* intel_dp_mst.c */
int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter);
void intel_attach_force_audio_property(struct drm_connector *connector);
void intel_attach_broadcast_rgb_property(struct drm_connector *connector);
+void intel_attach_aspect_ratio_property(struct drm_connector *connector);
/* intel_overlay.c */
void intel_panel_enable_backlight(struct intel_connector *connector);
void intel_panel_disable_backlight(struct intel_connector *connector);
void intel_panel_destroy_backlight(struct drm_connector *connector);
-void intel_panel_init_backlight_funcs(struct drm_device *dev);
enum drm_connector_status intel_panel_detect(struct drm_device *dev);
extern struct drm_display_mode *intel_find_panel_downclock(
struct drm_device *dev,
void intel_display_set_init_power(struct drm_i915_private *dev, bool enable);
+void chv_phy_powergate_lanes(struct intel_encoder *encoder,
+ bool override, unsigned int mask);
+bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
+ enum dpio_channel ch, bool override);
+
+
/* intel_pm.c */
void intel_init_clock_gating(struct drm_device *dev);
void intel_suspend_hw(struct drm_device *dev);
int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane);
int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
struct drm_file *file_priv);
-void intel_pipe_update_start(struct intel_crtc *crtc,
- uint32_t *start_vbl_count);
-void intel_pipe_update_end(struct intel_crtc *crtc, u32 start_vbl_count);
+void intel_pipe_update_start(struct intel_crtc *crtc);
+void intel_pipe_update_end(struct intel_crtc *crtc);
/* intel_tv.c */
void intel_tv_init(struct drm_device *dev);
return true;
}
-static void intel_dsi_port_enable(struct intel_encoder *encoder)
+static void bxt_dsi_device_ready(struct intel_encoder *encoder)
{
- struct drm_device *dev = encoder->base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+ struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
enum port port;
- u32 temp;
+ u32 val;
- if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) {
- temp = I915_READ(VLV_CHICKEN_3);
- temp &= ~PIXEL_OVERLAP_CNT_MASK |
- intel_dsi->pixel_overlap <<
- PIXEL_OVERLAP_CNT_SHIFT;
- I915_WRITE(VLV_CHICKEN_3, temp);
- }
+ DRM_DEBUG_KMS("\n");
+ /* Exit Low power state in 4 steps*/
for_each_dsi_port(port, intel_dsi->ports) {
- temp = I915_READ(MIPI_PORT_CTRL(port));
- temp &= ~LANE_CONFIGURATION_MASK;
- temp &= ~DUAL_LINK_MODE_MASK;
- if (intel_dsi->ports == ((1 << PORT_A) | (1 << PORT_C))) {
- temp |= (intel_dsi->dual_link - 1)
- << DUAL_LINK_MODE_SHIFT;
- temp |= intel_crtc->pipe ?
- LANE_CONFIGURATION_DUAL_LINK_B :
- LANE_CONFIGURATION_DUAL_LINK_A;
- }
- /* assert ip_tg_enable signal */
- I915_WRITE(MIPI_PORT_CTRL(port), temp | DPI_ENABLE);
- POSTING_READ(MIPI_PORT_CTRL(port));
- }
-}
+ /* 1. Enable MIPI PHY transparent latch */
+ val = I915_READ(BXT_MIPI_PORT_CTRL(port));
+ I915_WRITE(BXT_MIPI_PORT_CTRL(port), val | LP_OUTPUT_HOLD);
+ usleep_range(2000, 2500);
-static void intel_dsi_port_disable(struct intel_encoder *encoder)
-{
- struct drm_device *dev = encoder->base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
- enum port port;
- u32 temp;
+ /* 2. Enter ULPS */
+ val = I915_READ(MIPI_DEVICE_READY(port));
+ val &= ~ULPS_STATE_MASK;
+ val |= (ULPS_STATE_ENTER | DEVICE_READY);
+ I915_WRITE(MIPI_DEVICE_READY(port), val);
+ usleep_range(2, 3);
+
+ /* 3. Exit ULPS */
+ val = I915_READ(MIPI_DEVICE_READY(port));
+ val &= ~ULPS_STATE_MASK;
+ val |= (ULPS_STATE_EXIT | DEVICE_READY);
+ I915_WRITE(MIPI_DEVICE_READY(port), val);
+ usleep_range(1000, 1500);
- for_each_dsi_port(port, intel_dsi->ports) {
- /* de-assert ip_tg_enable signal */
- temp = I915_READ(MIPI_PORT_CTRL(port));
- I915_WRITE(MIPI_PORT_CTRL(port), temp & ~DPI_ENABLE);
- POSTING_READ(MIPI_PORT_CTRL(port));
+ /* Clear ULPS and set device ready */
+ val = I915_READ(MIPI_DEVICE_READY(port));
+ val &= ~ULPS_STATE_MASK;
+ val |= DEVICE_READY;
+ I915_WRITE(MIPI_DEVICE_READY(port), val);
}
}
-static void intel_dsi_device_ready(struct intel_encoder *encoder)
+static void vlv_dsi_device_ready(struct intel_encoder *encoder)
{
struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
}
}
+static void intel_dsi_device_ready(struct intel_encoder *encoder)
+{
+ struct drm_device *dev = encoder->base.dev;
+
+ if (IS_VALLEYVIEW(dev))
+ vlv_dsi_device_ready(encoder);
+ else if (IS_BROXTON(dev))
+ bxt_dsi_device_ready(encoder);
+}
+
+static void intel_dsi_port_enable(struct intel_encoder *encoder)
+{
+ struct drm_device *dev = encoder->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+ enum port port;
+ u32 temp;
+ u32 port_ctrl;
+
+ if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) {
+ temp = I915_READ(VLV_CHICKEN_3);
+ temp &= ~PIXEL_OVERLAP_CNT_MASK |
+ intel_dsi->pixel_overlap <<
+ PIXEL_OVERLAP_CNT_SHIFT;
+ I915_WRITE(VLV_CHICKEN_3, temp);
+ }
+
+ for_each_dsi_port(port, intel_dsi->ports) {
+ port_ctrl = IS_BROXTON(dev) ? BXT_MIPI_PORT_CTRL(port) :
+ MIPI_PORT_CTRL(port);
+
+ temp = I915_READ(port_ctrl);
+
+ temp &= ~LANE_CONFIGURATION_MASK;
+ temp &= ~DUAL_LINK_MODE_MASK;
+
+ if (intel_dsi->ports == ((1 << PORT_A) | (1 << PORT_C))) {
+ temp |= (intel_dsi->dual_link - 1)
+ << DUAL_LINK_MODE_SHIFT;
+ temp |= intel_crtc->pipe ?
+ LANE_CONFIGURATION_DUAL_LINK_B :
+ LANE_CONFIGURATION_DUAL_LINK_A;
+ }
+ /* assert ip_tg_enable signal */
+ I915_WRITE(port_ctrl, temp | DPI_ENABLE);
+ POSTING_READ(port_ctrl);
+ }
+}
+
+static void intel_dsi_port_disable(struct intel_encoder *encoder)
+{
+ struct drm_device *dev = encoder->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+ enum port port;
+ u32 temp;
+ u32 port_ctrl;
+
+ for_each_dsi_port(port, intel_dsi->ports) {
+ /* de-assert ip_tg_enable signal */
+ port_ctrl = IS_BROXTON(dev) ? BXT_MIPI_PORT_CTRL(port) :
+ MIPI_PORT_CTRL(port);
+ temp = I915_READ(port_ctrl);
+ I915_WRITE(port_ctrl, temp & ~DPI_ENABLE);
+ POSTING_READ(port_ctrl);
+ }
+}
+
static void intel_dsi_enable(struct intel_encoder *encoder)
{
struct drm_device *dev = encoder->base.dev;
msleep(intel_dsi->panel_on_delay);
- /* Disable DPOunit clock gating, can stall pipe
- * and we need DPLL REFA always enabled */
- tmp = I915_READ(DPLL(pipe));
- tmp |= DPLL_REF_CLK_ENABLE_VLV;
- I915_WRITE(DPLL(pipe), tmp);
-
- /* update the hw state for DPLL */
- intel_crtc->config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
- DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
-
- tmp = I915_READ(DSPCLK_GATE_D);
- tmp |= DPOUNIT_CLOCK_GATE_DISABLE;
- I915_WRITE(DSPCLK_GATE_D, tmp);
+ if (IS_VALLEYVIEW(dev)) {
+ /*
+ * Disable DPOunit clock gating, can stall pipe
+ * and we need DPLL REFA always enabled
+ */
+ tmp = I915_READ(DPLL(pipe));
+ tmp |= DPLL_REF_CLK_ENABLE_VLV;
+ I915_WRITE(DPLL(pipe), tmp);
+
+ /* update the hw state for DPLL */
+ intel_crtc->config->dpll_hw_state.dpll =
+ DPLL_INTEGRATED_REF_CLK_VLV |
+ DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
+
+ tmp = I915_READ(DSPCLK_GATE_D);
+ tmp |= DPOUNIT_CLOCK_GATE_DISABLE;
+ I915_WRITE(DSPCLK_GATE_D, tmp);
+ }
/* put device in ready state */
intel_dsi_device_ready(encoder);
/* Panel commands can be sent when clock is in LP11 */
I915_WRITE(MIPI_DEVICE_READY(port), 0x0);
- temp = I915_READ(MIPI_CTRL(port));
- temp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
- I915_WRITE(MIPI_CTRL(port), temp |
- intel_dsi->escape_clk_div <<
- ESCAPE_CLOCK_DIVIDER_SHIFT);
-
+ intel_dsi_reset_clocks(encoder, port);
I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
temp = I915_READ(MIPI_DSI_FUNC_PRG(port));
static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
{
+ struct drm_device *dev = encoder->base.dev;
struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
enum port port;
u32 val;
+ u32 port_ctrl = 0;
DRM_DEBUG_KMS("\n");
for_each_dsi_port(port, intel_dsi->ports) {
ULPS_STATE_ENTER);
usleep_range(2000, 2500);
+ if (IS_BROXTON(dev))
+ port_ctrl = BXT_MIPI_PORT_CTRL(port);
+ else if (IS_VALLEYVIEW(dev))
+ /* Common bit for both MIPI Port A & MIPI Port C */
+ port_ctrl = MIPI_PORT_CTRL(PORT_A);
+
/* Wait till Clock lanes are in LP-00 state for MIPI Port A
* only. MIPI Port C has no similar bit for checking
*/
- if (wait_for(((I915_READ(MIPI_PORT_CTRL(PORT_A)) & AFE_LATCHOUT)
- == 0x00000), 30))
+ if (wait_for(((I915_READ(port_ctrl) & AFE_LATCHOUT)
+ == 0x00000), 30))
DRM_ERROR("DSI LP not going Low\n");
- /* Disable MIPI PHY transparent latch
- * Common bit for both MIPI Port A & MIPI Port C
- */
- val = I915_READ(MIPI_PORT_CTRL(PORT_A));
- I915_WRITE(MIPI_PORT_CTRL(PORT_A), val & ~LP_OUTPUT_HOLD);
+ /* Disable MIPI PHY transparent latch */
+ val = I915_READ(port_ctrl);
+ I915_WRITE(port_ctrl, val & ~LP_OUTPUT_HOLD);
usleep_range(1000, 1500);
I915_WRITE(MIPI_DEVICE_READY(port), 0x00);
usleep_range(2000, 2500);
}
- vlv_disable_dsi_pll(encoder);
+ intel_disable_dsi_pll(encoder);
}
static void intel_dsi_post_disable(struct intel_encoder *encoder)
struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
struct drm_device *dev = encoder->base.dev;
enum intel_display_power_domain power_domain;
- u32 dpi_enabled, func;
+ u32 dpi_enabled, func, ctrl_reg;
enum port port;
DRM_DEBUG_KMS("\n");
/* XXX: this only works for one DSI output */
for_each_dsi_port(port, intel_dsi->ports) {
func = I915_READ(MIPI_DSI_FUNC_PRG(port));
- dpi_enabled = I915_READ(MIPI_PORT_CTRL(port)) &
- DPI_ENABLE;
+ ctrl_reg = IS_BROXTON(dev) ? BXT_MIPI_PORT_CTRL(port) :
+ MIPI_PORT_CTRL(port);
+ dpi_enabled = I915_READ(ctrl_reg) & DPI_ENABLE;
/* Due to some hardware limitations on BYT, MIPI Port C DPI
* Enable bit does not get set. To check whether DSI Port C
static void intel_dsi_get_config(struct intel_encoder *encoder,
struct intel_crtc_state *pipe_config)
{
- u32 pclk;
+ u32 pclk = 0;
DRM_DEBUG_KMS("\n");
/*
*/
pipe_config->dpll_hw_state.dpll_md = 0;
- pclk = vlv_get_dsi_pclk(encoder, pipe_config->pipe_bpp);
+ if (IS_BROXTON(encoder->base.dev))
+ pclk = bxt_get_dsi_pclk(encoder, pipe_config->pipe_bpp);
+ else if (IS_VALLEYVIEW(encoder->base.dev))
+ pclk = vlv_get_dsi_pclk(encoder, pipe_config->pipe_bpp);
+
if (!pclk)
return;
{
struct intel_connector *intel_connector = to_intel_connector(connector);
struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode;
+ int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
DRM_DEBUG_KMS("\n");
return MODE_PANEL;
if (mode->vdisplay > fixed_mode->vdisplay)
return MODE_PANEL;
+ if (fixed_mode->clock > max_dotclk)
+ return MODE_CLOCK_HIGH;
}
return MODE_OK;
}
static void set_dsi_timings(struct drm_encoder *encoder,
- const struct drm_display_mode *mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_device *dev = encoder->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
u16 hactive, hfp, hsync, hbp, vfp, vsync, vbp;
- hactive = mode->hdisplay;
- hfp = mode->hsync_start - mode->hdisplay;
- hsync = mode->hsync_end - mode->hsync_start;
- hbp = mode->htotal - mode->hsync_end;
+ hactive = adjusted_mode->crtc_hdisplay;
+ hfp = adjusted_mode->crtc_hsync_start - adjusted_mode->crtc_hdisplay;
+ hsync = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start;
+ hbp = adjusted_mode->crtc_htotal - adjusted_mode->crtc_hsync_end;
if (intel_dsi->dual_link) {
hactive /= 2;
hbp /= 2;
}
- vfp = mode->vsync_start - mode->vdisplay;
- vsync = mode->vsync_end - mode->vsync_start;
- vbp = mode->vtotal - mode->vsync_end;
+ vfp = adjusted_mode->crtc_vsync_start - adjusted_mode->crtc_vdisplay;
+ vsync = adjusted_mode->crtc_vsync_end - adjusted_mode->crtc_vsync_start;
+ vbp = adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vsync_end;
/* horizontal values are in terms of high speed byte clock */
hactive = txbyteclkhs(hactive, bpp, lane_count,
hbp = txbyteclkhs(hbp, bpp, lane_count, intel_dsi->burst_mode_ratio);
for_each_dsi_port(port, intel_dsi->ports) {
+ if (IS_BROXTON(dev)) {
+ /*
+ * Program hdisplay and vdisplay on MIPI transcoder.
+ * This is different from calculated hactive and
+ * vactive, as they are calculated per channel basis,
+ * whereas these values should be based on resolution.
+ */
+ I915_WRITE(BXT_MIPI_TRANS_HACTIVE(port),
+ adjusted_mode->crtc_hdisplay);
+ I915_WRITE(BXT_MIPI_TRANS_VACTIVE(port),
+ adjusted_mode->crtc_vdisplay);
+ I915_WRITE(BXT_MIPI_TRANS_VTOTAL(port),
+ adjusted_mode->crtc_vtotal);
+ }
+
I915_WRITE(MIPI_HACTIVE_AREA_COUNT(port), hactive);
I915_WRITE(MIPI_HFP_COUNT(port), hfp);
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
- struct drm_display_mode *adjusted_mode =
- &intel_crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
enum port port;
unsigned int bpp = intel_crtc->config->pipe_bpp;
u32 val, tmp;
DRM_DEBUG_KMS("pipe %c\n", pipe_name(intel_crtc->pipe));
- mode_hdisplay = adjusted_mode->hdisplay;
+ mode_hdisplay = adjusted_mode->crtc_hdisplay;
if (intel_dsi->dual_link) {
mode_hdisplay /= 2;
}
for_each_dsi_port(port, intel_dsi->ports) {
- /* escape clock divider, 20MHz, shared for A and C.
- * device ready must be off when doing this! txclkesc? */
- tmp = I915_READ(MIPI_CTRL(PORT_A));
- tmp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
- I915_WRITE(MIPI_CTRL(PORT_A), tmp | ESCAPE_CLOCK_DIVIDER_1);
-
- /* read request priority is per pipe */
- tmp = I915_READ(MIPI_CTRL(port));
- tmp &= ~READ_REQUEST_PRIORITY_MASK;
- I915_WRITE(MIPI_CTRL(port), tmp | READ_REQUEST_PRIORITY_HIGH);
+ if (IS_VALLEYVIEW(dev)) {
+ /*
+ * escape clock divider, 20MHz, shared for A and C.
+ * device ready must be off when doing this! txclkesc?
+ */
+ tmp = I915_READ(MIPI_CTRL(PORT_A));
+ tmp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
+ I915_WRITE(MIPI_CTRL(PORT_A), tmp |
+ ESCAPE_CLOCK_DIVIDER_1);
+
+ /* read request priority is per pipe */
+ tmp = I915_READ(MIPI_CTRL(port));
+ tmp &= ~READ_REQUEST_PRIORITY_MASK;
+ I915_WRITE(MIPI_CTRL(port), tmp |
+ READ_REQUEST_PRIORITY_HIGH);
+ } else if (IS_BROXTON(dev)) {
+ /*
+ * FIXME:
+ * BXT can connect any PIPE to any MIPI port.
+ * Select the pipe based on the MIPI port read from
+ * VBT for now. Pick PIPE A for MIPI port A and C
+ * for port C.
+ */
+ tmp = I915_READ(MIPI_CTRL(port));
+ tmp &= ~BXT_PIPE_SELECT_MASK;
+
+ if (port == PORT_A)
+ tmp |= BXT_PIPE_SELECT_A;
+ else if (port == PORT_C)
+ tmp |= BXT_PIPE_SELECT_C;
+
+ I915_WRITE(MIPI_CTRL(port), tmp);
+ }
/* XXX: why here, why like this? handling in irq handler?! */
I915_WRITE(MIPI_INTR_STAT(port), 0xffffffff);
I915_WRITE(MIPI_DPHY_PARAM(port), intel_dsi->dphy_reg);
I915_WRITE(MIPI_DPI_RESOLUTION(port),
- adjusted_mode->vdisplay << VERTICAL_ADDRESS_SHIFT |
+ adjusted_mode->crtc_vdisplay << VERTICAL_ADDRESS_SHIFT |
mode_hdisplay << HORIZONTAL_ADDRESS_SHIFT);
}
if (is_vid_mode(intel_dsi) &&
intel_dsi->video_mode_format == VIDEO_MODE_BURST) {
I915_WRITE(MIPI_HS_TX_TIMEOUT(port),
- txbyteclkhs(adjusted_mode->htotal, bpp,
- intel_dsi->lane_count,
- intel_dsi->burst_mode_ratio) + 1);
+ txbyteclkhs(adjusted_mode->crtc_htotal, bpp,
+ intel_dsi->lane_count,
+ intel_dsi->burst_mode_ratio) + 1);
} else {
I915_WRITE(MIPI_HS_TX_TIMEOUT(port),
- txbyteclkhs(adjusted_mode->vtotal *
- adjusted_mode->htotal,
- bpp, intel_dsi->lane_count,
- intel_dsi->burst_mode_ratio) + 1);
+ txbyteclkhs(adjusted_mode->crtc_vtotal *
+ adjusted_mode->crtc_htotal,
+ bpp, intel_dsi->lane_count,
+ intel_dsi->burst_mode_ratio) + 1);
}
I915_WRITE(MIPI_LP_RX_TIMEOUT(port), intel_dsi->lp_rx_timeout);
I915_WRITE(MIPI_TURN_AROUND_TIMEOUT(port),
I915_WRITE(MIPI_INIT_COUNT(port),
txclkesc(intel_dsi->escape_clk_div, 100));
+ if (IS_BROXTON(dev) && (!intel_dsi->dual_link)) {
+ /*
+ * BXT spec says write MIPI_INIT_COUNT for
+ * both the ports, even if only one is
+ * getting used. So write the other port
+ * if not in dual link mode.
+ */
+ I915_WRITE(MIPI_INIT_COUNT(port ==
+ PORT_A ? PORT_C : PORT_A),
+ intel_dsi->init_count);
+ }
/* recovery disables */
I915_WRITE(MIPI_EOT_DISABLE(port), tmp);
DRM_DEBUG_KMS("\n");
intel_dsi_prepare(encoder);
+ intel_enable_dsi_pll(encoder);
- vlv_enable_dsi_pll(encoder);
}
static enum drm_connector_status
return container_of(encoder, struct intel_dsi, base.base);
}
-extern void vlv_enable_dsi_pll(struct intel_encoder *encoder);
-extern void vlv_disable_dsi_pll(struct intel_encoder *encoder);
+extern void intel_enable_dsi_pll(struct intel_encoder *encoder);
+extern void intel_disable_dsi_pll(struct intel_encoder *encoder);
extern u32 vlv_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp);
+extern u32 bxt_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp);
+extern void intel_dsi_reset_clocks(struct intel_encoder *encoder,
+ enum port port);
struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id);
vlv_cck_write(dev_priv, CCK_REG_DSI_PLL_CONTROL, dsi_mnp.dsi_pll_ctrl);
}
-void vlv_enable_dsi_pll(struct intel_encoder *encoder)
+static void vlv_enable_dsi_pll(struct intel_encoder *encoder)
{
struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
u32 tmp;
DRM_DEBUG_KMS("DSI PLL locked\n");
}
-void vlv_disable_dsi_pll(struct intel_encoder *encoder)
+static void vlv_disable_dsi_pll(struct intel_encoder *encoder)
{
struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
u32 tmp;
mutex_unlock(&dev_priv->sb_lock);
}
+static void bxt_disable_dsi_pll(struct intel_encoder *encoder)
+{
+ struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+ u32 val;
+
+ DRM_DEBUG_KMS("\n");
+
+ val = I915_READ(BXT_DSI_PLL_ENABLE);
+ val &= ~BXT_DSI_PLL_DO_ENABLE;
+ I915_WRITE(BXT_DSI_PLL_ENABLE, val);
+
+ /*
+ * PLL lock should deassert within 200us.
+ * Wait up to 1ms before timing out.
+ */
+ if (wait_for((I915_READ(BXT_DSI_PLL_ENABLE)
+ & BXT_DSI_PLL_LOCKED) == 0, 1))
+ DRM_ERROR("Timeout waiting for PLL lock deassertion\n");
+}
+
static void assert_bpp_mismatch(int pixel_format, int pipe_bpp)
{
int bpp = dsi_pixel_format_bpp(pixel_format);
return pclk;
}
+
+u32 bxt_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp)
+{
+ u32 pclk;
+ u32 dsi_clk;
+ u32 dsi_ratio;
+ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+ struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+
+ /* Divide by zero */
+ if (!pipe_bpp) {
+ DRM_ERROR("Invalid BPP(0)\n");
+ return 0;
+ }
+
+ dsi_ratio = I915_READ(BXT_DSI_PLL_CTL) &
+ BXT_DSI_PLL_RATIO_MASK;
+
+ /* Invalid DSI ratio ? */
+ if (dsi_ratio < BXT_DSI_PLL_RATIO_MIN ||
+ dsi_ratio > BXT_DSI_PLL_RATIO_MAX) {
+ DRM_ERROR("Invalid DSI pll ratio(%u) programmed\n", dsi_ratio);
+ return 0;
+ }
+
+ dsi_clk = (dsi_ratio * BXT_REF_CLOCK_KHZ) / 2;
+
+ /* pixel_format and pipe_bpp should agree */
+ assert_bpp_mismatch(intel_dsi->pixel_format, pipe_bpp);
+
+ pclk = DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, pipe_bpp);
+
+ DRM_DEBUG_DRIVER("Calculated pclk=%u\n", pclk);
+ return pclk;
+}
+
+static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
+{
+ u32 temp;
+ struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+
+ temp = I915_READ(MIPI_CTRL(port));
+ temp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
+ I915_WRITE(MIPI_CTRL(port), temp |
+ intel_dsi->escape_clk_div <<
+ ESCAPE_CLOCK_DIVIDER_SHIFT);
+}
+
+/* Program BXT Mipi clocks and dividers */
+static void bxt_dsi_program_clocks(struct drm_device *dev, enum port port)
+{
+ u32 tmp;
+ u32 divider;
+ u32 dsi_rate;
+ u32 pll_ratio;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ /* Clear old configurations */
+ tmp = I915_READ(BXT_MIPI_CLOCK_CTL);
+ tmp &= ~(BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port));
+ tmp &= ~(BXT_MIPI_RX_ESCLK_FIXDIV_MASK(port));
+ tmp &= ~(BXT_MIPI_ESCLK_VAR_DIV_MASK(port));
+ tmp &= ~(BXT_MIPI_DPHY_DIVIDER_MASK(port));
+
+ /* Get the current DSI rate(actual) */
+ pll_ratio = I915_READ(BXT_DSI_PLL_CTL) &
+ BXT_DSI_PLL_RATIO_MASK;
+ dsi_rate = (BXT_REF_CLOCK_KHZ * pll_ratio) / 2;
+
+ /* Max possible output of clock is 39.5 MHz, program value -1 */
+ divider = (dsi_rate / BXT_MAX_VAR_OUTPUT_KHZ) - 1;
+ tmp |= BXT_MIPI_ESCLK_VAR_DIV(port, divider);
+
+ /*
+ * Tx escape clock must be as close to 20MHz possible, but should
+ * not exceed it. Hence select divide by 2
+ */
+ tmp |= BXT_MIPI_TX_ESCLK_8XDIV_BY2(port);
+
+ tmp |= BXT_MIPI_RX_ESCLK_8X_BY3(port);
+
+ I915_WRITE(BXT_MIPI_CLOCK_CTL, tmp);
+}
+
+static bool bxt_configure_dsi_pll(struct intel_encoder *encoder)
+{
+ struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+ u8 dsi_ratio;
+ u32 dsi_clk;
+ u32 val;
+
+ dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format,
+ intel_dsi->lane_count);
+
+ /*
+ * From clock diagram, to get PLL ratio divider, divide double of DSI
+ * link rate (i.e., 2*8x=16x frequency value) by ref clock. Make sure to
+ * round 'up' the result
+ */
+ dsi_ratio = DIV_ROUND_UP(dsi_clk * 2, BXT_REF_CLOCK_KHZ);
+ if (dsi_ratio < BXT_DSI_PLL_RATIO_MIN ||
+ dsi_ratio > BXT_DSI_PLL_RATIO_MAX) {
+ DRM_ERROR("Cant get a suitable ratio from DSI PLL ratios\n");
+ return false;
+ }
+
+ /*
+ * Program DSI ratio and Select MIPIC and MIPIA PLL output as 8x
+ * Spec says both have to be programmed, even if one is not getting
+ * used. Configure MIPI_CLOCK_CTL dividers in modeset
+ */
+ val = I915_READ(BXT_DSI_PLL_CTL);
+ val &= ~BXT_DSI_PLL_PVD_RATIO_MASK;
+ val &= ~BXT_DSI_FREQ_SEL_MASK;
+ val &= ~BXT_DSI_PLL_RATIO_MASK;
+ val |= (dsi_ratio | BXT_DSIA_16X_BY2 | BXT_DSIC_16X_BY2);
+
+ /* As per recommendation from hardware team,
+ * Prog PVD ratio =1 if dsi ratio <= 50
+ */
+ if (dsi_ratio <= 50) {
+ val &= ~BXT_DSI_PLL_PVD_RATIO_MASK;
+ val |= BXT_DSI_PLL_PVD_RATIO_1;
+ }
+
+ I915_WRITE(BXT_DSI_PLL_CTL, val);
+ POSTING_READ(BXT_DSI_PLL_CTL);
+
+ return true;
+}
+
+static void bxt_enable_dsi_pll(struct intel_encoder *encoder)
+{
+ struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+ enum port port;
+ u32 val;
+
+ DRM_DEBUG_KMS("\n");
+
+ val = I915_READ(BXT_DSI_PLL_ENABLE);
+
+ if (val & BXT_DSI_PLL_DO_ENABLE) {
+ WARN(1, "DSI PLL already enabled. Disabling it.\n");
+ val &= ~BXT_DSI_PLL_DO_ENABLE;
+ I915_WRITE(BXT_DSI_PLL_ENABLE, val);
+ }
+
+ /* Configure PLL vales */
+ if (!bxt_configure_dsi_pll(encoder)) {
+ DRM_ERROR("Configure DSI PLL failed, abort PLL enable\n");
+ return;
+ }
+
+ /* Program TX, RX, Dphy clocks */
+ for_each_dsi_port(port, intel_dsi->ports)
+ bxt_dsi_program_clocks(encoder->base.dev, port);
+
+ /* Enable DSI PLL */
+ val = I915_READ(BXT_DSI_PLL_ENABLE);
+ val |= BXT_DSI_PLL_DO_ENABLE;
+ I915_WRITE(BXT_DSI_PLL_ENABLE, val);
+
+ /* Timeout and fail if PLL not locked */
+ if (wait_for(I915_READ(BXT_DSI_PLL_ENABLE) & BXT_DSI_PLL_LOCKED, 1)) {
+ DRM_ERROR("Timed out waiting for DSI PLL to lock\n");
+ return;
+ }
+
+ DRM_DEBUG_KMS("DSI PLL locked\n");
+}
+
+void intel_enable_dsi_pll(struct intel_encoder *encoder)
+{
+ struct drm_device *dev = encoder->base.dev;
+
+ if (IS_VALLEYVIEW(dev))
+ vlv_enable_dsi_pll(encoder);
+ else if (IS_BROXTON(dev))
+ bxt_enable_dsi_pll(encoder);
+}
+
+void intel_disable_dsi_pll(struct intel_encoder *encoder)
+{
+ struct drm_device *dev = encoder->base.dev;
+
+ if (IS_VALLEYVIEW(dev))
+ vlv_disable_dsi_pll(encoder);
+ else if (IS_BROXTON(dev))
+ bxt_disable_dsi_pll(encoder);
+}
+
+static void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
+{
+ u32 tmp;
+ struct drm_device *dev = encoder->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ /* Clear old configurations */
+ tmp = I915_READ(BXT_MIPI_CLOCK_CTL);
+ tmp &= ~(BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port));
+ tmp &= ~(BXT_MIPI_RX_ESCLK_FIXDIV_MASK(port));
+ tmp &= ~(BXT_MIPI_ESCLK_VAR_DIV_MASK(port));
+ tmp &= ~(BXT_MIPI_DPHY_DIVIDER_MASK(port));
+ I915_WRITE(BXT_MIPI_CLOCK_CTL, tmp);
+ I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
+}
+
+void intel_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
+{
+ struct drm_device *dev = encoder->base.dev;
+
+ if (IS_BROXTON(dev))
+ bxt_dsi_reset_clocks(encoder, port);
+ else if (IS_VALLEYVIEW(dev))
+ vlv_dsi_reset_clocks(encoder, port);
+}
struct intel_dvo_device dev;
- struct drm_display_mode *panel_fixed_mode;
+ struct intel_connector *attached_connector;
+
bool panel_wants_dither;
};
struct drm_display_mode *mode)
{
struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
+ const struct drm_display_mode *fixed_mode =
+ to_intel_connector(connector)->panel.fixed_mode;
+ int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
+ int target_clock = mode->clock;
if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
return MODE_NO_DBLESCAN;
/* XXX: Validate clock range */
- if (intel_dvo->panel_fixed_mode) {
- if (mode->hdisplay > intel_dvo->panel_fixed_mode->hdisplay)
+ if (fixed_mode) {
+ if (mode->hdisplay > fixed_mode->hdisplay)
return MODE_PANEL;
- if (mode->vdisplay > intel_dvo->panel_fixed_mode->vdisplay)
+ if (mode->vdisplay > fixed_mode->vdisplay)
return MODE_PANEL;
+
+ target_clock = fixed_mode->clock;
}
+ if (target_clock > max_dotclk)
+ return MODE_CLOCK_HIGH;
+
return intel_dvo->dev.dev_ops->mode_valid(&intel_dvo->dev, mode);
}
struct intel_crtc_state *pipe_config)
{
struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
+ const struct drm_display_mode *fixed_mode =
+ intel_dvo->attached_connector->panel.fixed_mode;
struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
/* If we have timings from the BIOS for the panel, put them in
* with the panel scaling set up to source from the H/VDisplay
* of the original mode.
*/
- if (intel_dvo->panel_fixed_mode != NULL) {
-#define C(x) adjusted_mode->x = intel_dvo->panel_fixed_mode->x
- C(hdisplay);
- C(hsync_start);
- C(hsync_end);
- C(htotal);
- C(vdisplay);
- C(vsync_start);
- C(vsync_end);
- C(vtotal);
- C(clock);
-#undef C
-
- drm_mode_set_crtcinfo(adjusted_mode, 0);
- }
+ if (fixed_mode)
+ intel_fixed_panel_mode(fixed_mode, adjusted_mode);
return true;
}
struct drm_device *dev = encoder->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
- struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
int pipe = crtc->pipe;
u32 dvo_val;
dvo_val |= DVO_VSYNC_ACTIVE_HIGH;
/*I915_WRITE(DVOB_SRCDIM,
- (adjusted_mode->hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) |
- (adjusted_mode->VDisplay << DVO_SRCDIM_VERTICAL_SHIFT));*/
+ (adjusted_mode->crtc_hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) |
+ (adjusted_mode->crtc_vdisplay << DVO_SRCDIM_VERTICAL_SHIFT));*/
I915_WRITE(dvo_srcdim_reg,
- (adjusted_mode->hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) |
- (adjusted_mode->vdisplay << DVO_SRCDIM_VERTICAL_SHIFT));
+ (adjusted_mode->crtc_hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) |
+ (adjusted_mode->crtc_vdisplay << DVO_SRCDIM_VERTICAL_SHIFT));
/*I915_WRITE(DVOB, dvo_val);*/
I915_WRITE(dvo_reg, dvo_val);
}
static int intel_dvo_get_modes(struct drm_connector *connector)
{
- struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
struct drm_i915_private *dev_priv = connector->dev->dev_private;
+ const struct drm_display_mode *fixed_mode =
+ to_intel_connector(connector)->panel.fixed_mode;
/* We should probably have an i2c driver get_modes function for those
* devices which will have a fixed set of modes determined by the chip
if (!list_empty(&connector->probed_modes))
return 1;
- if (intel_dvo->panel_fixed_mode != NULL) {
+ if (fixed_mode) {
struct drm_display_mode *mode;
- mode = drm_mode_duplicate(connector->dev, intel_dvo->panel_fixed_mode);
+ mode = drm_mode_duplicate(connector->dev, fixed_mode);
if (mode) {
drm_mode_probed_add(connector, mode);
return 1;
static void intel_dvo_destroy(struct drm_connector *connector)
{
drm_connector_cleanup(connector);
+ intel_panel_fini(&to_intel_connector(connector)->panel);
kfree(connector);
}
if (intel_dvo->dev.dev_ops->destroy)
intel_dvo->dev.dev_ops->destroy(&intel_dvo->dev);
- kfree(intel_dvo->panel_fixed_mode);
-
intel_encoder_destroy(encoder);
}
return;
}
+ intel_dvo->attached_connector = intel_connector;
+
intel_encoder = &intel_dvo->base;
drm_encoder_init(dev, &intel_encoder->base,
&intel_dvo_enc_funcs, encoder_type);
* headers, likely), so for now, just get the current
* mode being output through DVO.
*/
- intel_dvo->panel_fixed_mode =
- intel_dvo_get_current_mode(connector);
+ intel_panel_init(&intel_connector->panel,
+ intel_dvo_get_current_mode(connector),
+ NULL);
intel_dvo->panel_wants_dither = true;
}
#include "intel_drv.h"
#include "i915_drv.h"
+static inline bool fbc_supported(struct drm_i915_private *dev_priv)
+{
+ return dev_priv->fbc.enable_fbc != NULL;
+}
+
+/*
+ * In some platforms where the CRTC's x:0/y:0 coordinates doesn't match the
+ * frontbuffer's x:0/y:0 coordinates we lie to the hardware about the plane's
+ * origin so the x and y offsets can actually fit the registers. As a
+ * consequence, the fence doesn't really start exactly at the display plane
+ * address we program because it starts at the real start of the buffer, so we
+ * have to take this into consideration here.
+ */
+static unsigned int get_crtc_fence_y_offset(struct intel_crtc *crtc)
+{
+ return crtc->base.y - crtc->adjusted_y;
+}
+
static void i8xx_fbc_disable(struct drm_i915_private *dev_priv)
{
u32 fbc_ctl;
/* Clear old tags */
for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
- I915_WRITE(FBC_TAG + (i * 4), 0);
+ I915_WRITE(FBC_TAG(i), 0);
if (IS_GEN4(dev_priv)) {
u32 fbc_ctl2;
fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
fbc_ctl2 |= FBC_CTL_PLANE(crtc->plane);
I915_WRITE(FBC_CONTROL2, fbc_ctl2);
- I915_WRITE(FBC_FENCE_OFF, crtc->base.y);
+ I915_WRITE(FBC_FENCE_OFF, get_crtc_fence_y_offset(crtc));
}
/* enable it... */
dpfc_ctl |= DPFC_CTL_LIMIT_1X;
dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
- I915_WRITE(DPFC_FENCE_YOFF, crtc->base.y);
+ I915_WRITE(DPFC_FENCE_YOFF, get_crtc_fence_y_offset(crtc));
/* enable it... */
I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
struct drm_i915_gem_object *obj = intel_fb_obj(fb);
u32 dpfc_ctl;
int threshold = dev_priv->fbc.threshold;
+ unsigned int y_offset;
dev_priv->fbc.enabled = true;
if (IS_GEN5(dev_priv))
dpfc_ctl |= obj->fence_reg;
- I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->base.y);
+ y_offset = get_crtc_fence_y_offset(crtc);
+ I915_WRITE(ILK_DPFC_FENCE_YOFF, y_offset);
I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID);
/* enable it... */
I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
if (IS_GEN6(dev_priv)) {
I915_WRITE(SNB_DPFC_CTL_SA,
SNB_CPU_FENCE_ENABLE | obj->fence_reg);
- I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->base.y);
+ I915_WRITE(DPFC_CPU_FENCE_OFFSET, y_offset);
}
intel_fbc_nuke(dev_priv);
if (dev_priv->fbc.false_color)
dpfc_ctl |= FBC_CTL_FALSE_COLOR;
- I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
-
if (IS_IVYBRIDGE(dev_priv)) {
/* WaFbcAsynchFlipDisableFbcQueue:ivb */
I915_WRITE(ILK_DISPLAY_CHICKEN1,
I915_READ(ILK_DISPLAY_CHICKEN1) |
ILK_FBCQ_DIS);
- } else {
+ } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
I915_WRITE(CHICKEN_PIPESL_1(crtc->pipe),
I915_READ(CHICKEN_PIPESL_1(crtc->pipe)) |
HSW_FBCQ_DIS);
}
+ I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
+
I915_WRITE(SNB_DPFC_CTL_SA,
SNB_CPU_FENCE_ENABLE | obj->fence_reg);
- I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->base.y);
+ I915_WRITE(DPFC_CPU_FENCE_OFFSET, get_crtc_fence_y_offset(crtc));
intel_fbc_nuke(dev_priv);
return dev_priv->fbc.enabled;
}
+static void intel_fbc_enable(struct intel_crtc *crtc,
+ const struct drm_framebuffer *fb)
+{
+ struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+
+ dev_priv->fbc.enable_fbc(crtc);
+
+ dev_priv->fbc.crtc = crtc;
+ dev_priv->fbc.fb_id = fb->base.id;
+ dev_priv->fbc.y = crtc->base.y;
+}
+
static void intel_fbc_work_fn(struct work_struct *__work)
{
struct intel_fbc_work *work =
/* Double check that we haven't switched fb without cancelling
* the prior work.
*/
- if (crtc_fb == work->fb) {
- dev_priv->fbc.enable_fbc(work->crtc);
-
- dev_priv->fbc.crtc = work->crtc;
- dev_priv->fbc.fb_id = crtc_fb->base.id;
- dev_priv->fbc.y = work->crtc->base.y;
- }
+ if (crtc_fb == work->fb)
+ intel_fbc_enable(work->crtc, work->fb);
dev_priv->fbc.fbc_work = NULL;
}
dev_priv->fbc.fbc_work = NULL;
}
-static void intel_fbc_enable(struct intel_crtc *crtc)
+static void intel_fbc_schedule_enable(struct intel_crtc *crtc)
{
struct intel_fbc_work *work;
struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
work = kzalloc(sizeof(*work), GFP_KERNEL);
if (work == NULL) {
DRM_ERROR("Failed to allocate FBC work structure\n");
- dev_priv->fbc.enable_fbc(crtc);
+ intel_fbc_enable(crtc, crtc->base.primary->fb);
return;
}
*/
void intel_fbc_disable(struct drm_i915_private *dev_priv)
{
- if (!dev_priv->fbc.enable_fbc)
+ if (!fbc_supported(dev_priv))
return;
mutex_lock(&dev_priv->fbc.lock);
{
struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
- if (!dev_priv->fbc.enable_fbc)
+ if (!fbc_supported(dev_priv))
return;
mutex_lock(&dev_priv->fbc.lock);
return "rotation unsupported";
case FBC_IN_DBG_MASTER:
return "Kernel debugger is active";
+ case FBC_BAD_STRIDE:
+ return "framebuffer stride not supported";
+ case FBC_PIXEL_RATE:
+ return "pixel rate is too big";
+ case FBC_PIXEL_FORMAT:
+ return "pixel format is invalid";
default:
MISSING_CASE(reason);
return "unknown reason";
{
int compression_threshold = 1;
int ret;
+ u64 end;
+
+ /* The FBC hardware for BDW/SKL doesn't have access to the stolen
+ * reserved range size, so it always assumes the maximum (8mb) is used.
+ * If we enable FBC using a CFB on that memory range we'll get FIFO
+ * underruns, even if that range is not reserved by the BIOS. */
+ if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))
+ end = dev_priv->gtt.stolen_size - 8 * 1024 * 1024;
+ else
+ end = dev_priv->gtt.stolen_usable_size;
/* HACK: This code depends on what we will do in *_enable_fbc. If that
* code changes, this code needs to change as well.
*/
/* Try to over-allocate to reduce reallocations and fragmentation. */
- ret = i915_gem_stolen_insert_node(dev_priv, node, size <<= 1, 4096);
+ ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size <<= 1,
+ 4096, 0, end);
if (ret == 0)
return compression_threshold;
(fb_cpp == 2 && compression_threshold == 2))
return 0;
- ret = i915_gem_stolen_insert_node(dev_priv, node, size >>= 1, 4096);
+ ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1,
+ 4096, 0, end);
if (ret && INTEL_INFO(dev_priv)->gen <= 4) {
return 0;
} else if (ret) {
dev_priv->fbc.uncompressed_size = size;
- DRM_DEBUG_KMS("reserved %d bytes of contiguous stolen space for FBC\n",
- size);
+ DRM_DEBUG_KMS("reserved %llu bytes of contiguous stolen space for FBC, threshold: %d\n",
+ dev_priv->fbc.compressed_fb.size,
+ dev_priv->fbc.threshold);
return 0;
void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
{
- if (!dev_priv->fbc.enable_fbc)
+ if (!fbc_supported(dev_priv))
return;
mutex_lock(&dev_priv->fbc.lock);
mutex_unlock(&dev_priv->fbc.lock);
}
-static int intel_fbc_setup_cfb(struct drm_i915_private *dev_priv, int size,
- int fb_cpp)
+/*
+ * For SKL+, the plane source size used by the hardware is based on the value we
+ * write to the PLANE_SIZE register. For BDW-, the hardware looks at the value
+ * we wrote to PIPESRC.
+ */
+static void intel_fbc_get_plane_source_size(struct intel_crtc *crtc,
+ int *width, int *height)
{
+ struct intel_plane_state *plane_state =
+ to_intel_plane_state(crtc->base.primary->state);
+ int w, h;
+
+ if (intel_rotation_90_or_270(plane_state->base.rotation)) {
+ w = drm_rect_height(&plane_state->src) >> 16;
+ h = drm_rect_width(&plane_state->src) >> 16;
+ } else {
+ w = drm_rect_width(&plane_state->src) >> 16;
+ h = drm_rect_height(&plane_state->src) >> 16;
+ }
+
+ if (width)
+ *width = w;
+ if (height)
+ *height = h;
+}
+
+static int intel_fbc_calculate_cfb_size(struct intel_crtc *crtc)
+{
+ struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+ struct drm_framebuffer *fb = crtc->base.primary->fb;
+ int lines;
+
+ intel_fbc_get_plane_source_size(crtc, NULL, &lines);
+ if (INTEL_INFO(dev_priv)->gen >= 7)
+ lines = min(lines, 2048);
+
+ return lines * fb->pitches[0];
+}
+
+static int intel_fbc_setup_cfb(struct intel_crtc *crtc)
+{
+ struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+ struct drm_framebuffer *fb = crtc->base.primary->fb;
+ int size, cpp;
+
+ size = intel_fbc_calculate_cfb_size(crtc);
+ cpp = drm_format_plane_cpp(fb->pixel_format, 0);
+
if (size <= dev_priv->fbc.uncompressed_size)
return 0;
/* Release any current block */
__intel_fbc_cleanup_cfb(dev_priv);
- return intel_fbc_alloc_cfb(dev_priv, size, fb_cpp);
+ return intel_fbc_alloc_cfb(dev_priv, size, cpp);
+}
+
+static bool stride_is_valid(struct drm_i915_private *dev_priv,
+ unsigned int stride)
+{
+ /* These should have been caught earlier. */
+ WARN_ON(stride < 512);
+ WARN_ON((stride & (64 - 1)) != 0);
+
+ /* Below are the additional FBC restrictions. */
+
+ if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv))
+ return stride == 4096 || stride == 8192;
+
+ if (IS_GEN4(dev_priv) && !IS_G4X(dev_priv) && stride < 2048)
+ return false;
+
+ if (stride > 16384)
+ return false;
+
+ return true;
+}
+
+static bool pixel_format_is_valid(struct drm_framebuffer *fb)
+{
+ struct drm_device *dev = fb->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ switch (fb->pixel_format) {
+ case DRM_FORMAT_XRGB8888:
+ case DRM_FORMAT_XBGR8888:
+ return true;
+ case DRM_FORMAT_XRGB1555:
+ case DRM_FORMAT_RGB565:
+ /* 16bpp not supported on gen2 */
+ if (IS_GEN2(dev))
+ return false;
+ /* WaFbcOnly1to1Ratio:ctg */
+ if (IS_G4X(dev_priv))
+ return false;
+ return true;
+ default:
+ return false;
+ }
+}
+
+/*
+ * For some reason, the hardware tracking starts looking at whatever we
+ * programmed as the display plane base address register. It does not look at
+ * the X and Y offset registers. That's why we look at the crtc->adjusted{x,y}
+ * variables instead of just looking at the pipe/plane size.
+ */
+static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
+{
+ struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+ unsigned int effective_w, effective_h, max_w, max_h;
+
+ if (INTEL_INFO(dev_priv)->gen >= 8 || IS_HASWELL(dev_priv)) {
+ max_w = 4096;
+ max_h = 4096;
+ } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
+ max_w = 4096;
+ max_h = 2048;
+ } else {
+ max_w = 2048;
+ max_h = 1536;
+ }
+
+ intel_fbc_get_plane_source_size(crtc, &effective_w, &effective_h);
+ effective_w += crtc->adjusted_x;
+ effective_h += crtc->adjusted_y;
+
+ return effective_w <= max_w && effective_h <= max_h;
}
/**
struct drm_framebuffer *fb;
struct drm_i915_gem_object *obj;
const struct drm_display_mode *adjusted_mode;
- unsigned int max_width, max_height;
WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));
goto out_disable;
}
- if (INTEL_INFO(dev_priv)->gen >= 8 || IS_HASWELL(dev_priv)) {
- max_width = 4096;
- max_height = 4096;
- } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
- max_width = 4096;
- max_height = 2048;
- } else {
- max_width = 2048;
- max_height = 1536;
- }
- if (intel_crtc->config->pipe_src_w > max_width ||
- intel_crtc->config->pipe_src_h > max_height) {
+ if (!intel_fbc_hw_tracking_covers_screen(intel_crtc)) {
set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE);
goto out_disable;
}
+
if ((INTEL_INFO(dev_priv)->gen < 4 || HAS_DDI(dev_priv)) &&
intel_crtc->plane != PLANE_A) {
set_no_fbc_reason(dev_priv, FBC_BAD_PLANE);
goto out_disable;
}
+ if (!stride_is_valid(dev_priv, fb->pitches[0])) {
+ set_no_fbc_reason(dev_priv, FBC_BAD_STRIDE);
+ goto out_disable;
+ }
+
+ if (!pixel_format_is_valid(fb)) {
+ set_no_fbc_reason(dev_priv, FBC_PIXEL_FORMAT);
+ goto out_disable;
+ }
+
/* If the kernel debugger is active, always disable compression */
if (in_dbg_master()) {
set_no_fbc_reason(dev_priv, FBC_IN_DBG_MASTER);
goto out_disable;
}
- if (intel_fbc_setup_cfb(dev_priv, obj->base.size,
- drm_format_plane_cpp(fb->pixel_format, 0))) {
+ /* WaFbcExceedCdClockThreshold:hsw,bdw */
+ if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) &&
+ ilk_pipe_pixel_rate(intel_crtc->config) >=
+ dev_priv->cdclk_freq * 95 / 100) {
+ set_no_fbc_reason(dev_priv, FBC_PIXEL_RATE);
+ goto out_disable;
+ }
+
+ if (intel_fbc_setup_cfb(intel_crtc)) {
set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL);
goto out_disable;
}
__intel_fbc_disable(dev_priv);
}
- intel_fbc_enable(intel_crtc);
+ intel_fbc_schedule_enable(intel_crtc);
dev_priv->fbc.no_fbc_reason = FBC_OK;
return;
*/
void intel_fbc_update(struct drm_i915_private *dev_priv)
{
- if (!dev_priv->fbc.enable_fbc)
+ if (!fbc_supported(dev_priv))
return;
mutex_lock(&dev_priv->fbc.lock);
{
unsigned int fbc_bits;
- if (!dev_priv->fbc.enable_fbc)
+ if (!fbc_supported(dev_priv))
return;
if (origin == ORIGIN_GTT)
void intel_fbc_flush(struct drm_i915_private *dev_priv,
unsigned int frontbuffer_bits, enum fb_op_origin origin)
{
- if (!dev_priv->fbc.enable_fbc)
+ if (!fbc_supported(dev_priv))
return;
if (origin == ORIGIN_GTT)
container_of(helper, struct intel_fbdev, helper);
struct drm_framebuffer *fb;
struct drm_device *dev = helper->dev;
+ struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_mode_fb_cmd2 mode_cmd = {};
- struct drm_i915_gem_object *obj;
+ struct drm_i915_gem_object *obj = NULL;
int size, ret;
/* we don't do packed 24bpp */
size = mode_cmd.pitches[0] * mode_cmd.height;
size = PAGE_ALIGN(size);
- obj = i915_gem_object_create_stolen(dev, size);
+
+ /* If the FB is too big, just don't use it since fbdev is not very
+ * important and we should probably use that space with FBC or other
+ * features. */
+ if (size * 2 < dev_priv->gtt.stolen_usable_size)
+ obj = i915_gem_object_create_stolen(dev, size);
if (obj == NULL)
obj = i915_gem_alloc_object(dev, size);
if (!obj) {
/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
- DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08lx, bo %p\n",
+ DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08llx, bo %p\n",
fb->width, fb->height,
i915_gem_obj_ggtt_offset(obj), obj);
struct intel_crtc *intel_crtc;
unsigned int max_size = 0;
- if (!i915.fastboot)
- return false;
-
/* Find the largest fb */
for_each_crtc(dev, crtc) {
struct drm_i915_gem_object *obj =
intel_fb_obj(crtc->primary->state->fb);
intel_crtc = to_intel_crtc(crtc);
- if (!intel_crtc->active || !obj) {
+ if (!crtc->state->active || !obj) {
DRM_DEBUG_KMS("pipe %c not active or no fb, skipping\n",
pipe_name(intel_crtc->pipe));
continue;
intel_crtc = to_intel_crtc(crtc);
- if (!intel_crtc->active) {
+ if (!crtc->state->active) {
DRM_DEBUG_KMS("pipe %c not active, skipping\n",
pipe_name(intel_crtc->pipe));
continue;
for_each_crtc(dev, crtc) {
intel_crtc = to_intel_crtc(crtc);
- if (!intel_crtc->active)
+ if (!crtc->state->active)
continue;
WARN(!crtc->primary->fb,
return ret;
}
+ ifbdev->helper.atomic = true;
+
dev_priv->fbdev = ifbdev;
INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker);
--- /dev/null
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+#ifndef _INTEL_GUC_H_
+#define _INTEL_GUC_H_
+
+#include "intel_guc_fwif.h"
+#include "i915_guc_reg.h"
+
+struct i915_guc_client {
+ struct drm_i915_gem_object *client_obj;
+ struct intel_context *owner;
+ struct intel_guc *guc;
+ uint32_t priority;
+ uint32_t ctx_index;
+
+ uint32_t proc_desc_offset;
+ uint32_t doorbell_offset;
+ uint32_t cookie;
+ uint16_t doorbell_id;
+ uint16_t padding; /* Maintain alignment */
+
+ uint32_t wq_offset;
+ uint32_t wq_size;
+
+ spinlock_t wq_lock; /* Protects all data below */
+ uint32_t wq_tail;
+
+ /* GuC submission statistics & status */
+ uint64_t submissions[I915_NUM_RINGS];
+ uint32_t q_fail;
+ uint32_t b_fail;
+ int retcode;
+};
+
+enum intel_guc_fw_status {
+ GUC_FIRMWARE_FAIL = -1,
+ GUC_FIRMWARE_NONE = 0,
+ GUC_FIRMWARE_PENDING,
+ GUC_FIRMWARE_SUCCESS
+};
+
+/*
+ * This structure encapsulates all the data needed during the process
+ * of fetching, caching, and loading the firmware image into the GuC.
+ */
+struct intel_guc_fw {
+ struct drm_device * guc_dev;
+ const char * guc_fw_path;
+ size_t guc_fw_size;
+ struct drm_i915_gem_object * guc_fw_obj;
+ enum intel_guc_fw_status guc_fw_fetch_status;
+ enum intel_guc_fw_status guc_fw_load_status;
+
+ uint16_t guc_fw_major_wanted;
+ uint16_t guc_fw_minor_wanted;
+ uint16_t guc_fw_major_found;
+ uint16_t guc_fw_minor_found;
+};
+
+struct intel_guc {
+ struct intel_guc_fw guc_fw;
+
+ uint32_t log_flags;
+ struct drm_i915_gem_object *log_obj;
+
+ struct drm_i915_gem_object *ctx_pool_obj;
+ struct ida ctx_ids;
+
+ struct i915_guc_client *execbuf_client;
+
+ spinlock_t host2guc_lock; /* Protects all data below */
+
+ DECLARE_BITMAP(doorbell_bitmap, GUC_MAX_DOORBELLS);
+ uint32_t db_cacheline; /* Cyclic counter mod pagesize */
+
+ /* Action status & statistics */
+ uint64_t action_count; /* Total commands issued */
+ uint32_t action_cmd; /* Last command word */
+ uint32_t action_status; /* Last return status */
+ uint32_t action_fail; /* Total number of failures */
+ int32_t action_err; /* Last error code */
+
+ uint64_t submissions[I915_NUM_RINGS];
+ uint32_t last_seqno[I915_NUM_RINGS];
+};
+
+/* intel_guc_loader.c */
+extern void intel_guc_ucode_init(struct drm_device *dev);
+extern int intel_guc_ucode_load(struct drm_device *dev);
+extern void intel_guc_ucode_fini(struct drm_device *dev);
+extern const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status);
+extern int intel_guc_suspend(struct drm_device *dev);
+extern int intel_guc_resume(struct drm_device *dev);
+
+/* i915_guc_submission.c */
+int i915_guc_submission_init(struct drm_device *dev);
+int i915_guc_submission_enable(struct drm_device *dev);
+int i915_guc_submit(struct i915_guc_client *client,
+ struct drm_i915_gem_request *rq);
+void i915_guc_submission_disable(struct drm_device *dev);
+void i915_guc_submission_fini(struct drm_device *dev);
+
+#endif
* EDITING THIS FILE IS THEREFORE NOT RECOMMENDED - YOUR CHANGES MAY BE LOST.
*/
-#define GFXCORE_FAMILY_GEN8 11
#define GFXCORE_FAMILY_GEN9 12
-#define GFXCORE_FAMILY_FORCE_ULONG 0x7fffffff
+#define GFXCORE_FAMILY_UNKNOWN 0x7fffffff
-#define GUC_CTX_PRIORITY_CRITICAL 0
+#define GUC_CTX_PRIORITY_KMD_HIGH 0
#define GUC_CTX_PRIORITY_HIGH 1
-#define GUC_CTX_PRIORITY_NORMAL 2
-#define GUC_CTX_PRIORITY_LOW 3
+#define GUC_CTX_PRIORITY_KMD_NORMAL 2
+#define GUC_CTX_PRIORITY_NORMAL 3
#define GUC_MAX_GPU_CONTEXTS 1024
-#define GUC_INVALID_CTX_ID (GUC_MAX_GPU_CONTEXTS + 1)
+#define GUC_INVALID_CTX_ID GUC_MAX_GPU_CONTEXTS
/* Work queue item header definitions */
#define WQ_STATUS_ACTIVE 1
#define GUC_CTX_DESC_ATTR_RESET (1 << 4)
#define GUC_CTX_DESC_ATTR_WQLOCKED (1 << 5)
#define GUC_CTX_DESC_ATTR_PCH (1 << 6)
+#define GUC_CTX_DESC_ATTR_TERMINATED (1 << 7)
/* The guc control data is 10 DWORDs */
#define GUC_CTL_CTXINFO 0
#define GUC_CTL_DISABLE_SCHEDULER (1 << 4)
#define GUC_CTL_PREEMPTION_LOG (1 << 5)
#define GUC_CTL_ENABLE_SLPC (1 << 7)
+#define GUC_CTL_RESET_ON_PREMPT_FAILURE (1 << 8)
#define GUC_CTL_DEBUG 8
#define GUC_LOG_VERBOSITY_SHIFT 0
#define GUC_LOG_VERBOSITY_LOW (0 << GUC_LOG_VERBOSITY_SHIFT)
/* Verbosity range-check limits, without the shift */
#define GUC_LOG_VERBOSITY_MIN 0
#define GUC_LOG_VERBOSITY_MAX 3
+#define GUC_CTL_RSRVD 9
-#define GUC_CTL_MAX_DWORDS (GUC_CTL_DEBUG + 1)
+#define GUC_CTL_MAX_DWORDS (GUC_CTL_RSRVD + 1)
struct guc_doorbell_info {
u32 db_status;
u32 engine_presence;
- u32 reserved0[1];
+ u8 engine_suspended;
+
+ u8 reserved0[3];
u64 reserved1[1];
u64 desc_private;
} __packed;
+#define GUC_FORCEWAKE_RENDER (1 << 0)
+#define GUC_FORCEWAKE_MEDIA (1 << 1)
+
+#define GUC_POWER_UNSPECIFIED 0
+#define GUC_POWER_D0 1
+#define GUC_POWER_D1 2
+#define GUC_POWER_D2 3
+#define GUC_POWER_D3 4
+
/* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */
enum host2guc_action {
HOST2GUC_ACTION_DEFAULT = 0x0,
HOST2GUC_ACTION_SAMPLE_FORCEWAKE = 0x6,
HOST2GUC_ACTION_ALLOCATE_DOORBELL = 0x10,
HOST2GUC_ACTION_DEALLOCATE_DOORBELL = 0x20,
+ HOST2GUC_ACTION_ENTER_S_STATE = 0x501,
+ HOST2GUC_ACTION_EXIT_S_STATE = 0x502,
HOST2GUC_ACTION_SLPC_REQUEST = 0x3003,
HOST2GUC_ACTION_LIMIT
};
--- /dev/null
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ * Vinit Azad <vinit.azad@intel.com>
+ * Ben Widawsky <ben@bwidawsk.net>
+ * Dave Gordon <david.s.gordon@intel.com>
+ * Alex Dai <yu.dai@intel.com>
+ */
+#include <linux/firmware.h>
+#include "i915_drv.h"
+#include "intel_guc.h"
+
+/**
+ * DOC: GuC
+ *
+ * intel_guc:
+ * Top level structure of guc. It handles firmware loading and manages client
+ * pool and doorbells. intel_guc owns a i915_guc_client to replace the legacy
+ * ExecList submission.
+ *
+ * Firmware versioning:
+ * The firmware build process will generate a version header file with major and
+ * minor version defined. The versions are built into CSS header of firmware.
+ * i915 kernel driver set the minimal firmware version required per platform.
+ * The firmware installation package will install (symbolic link) proper version
+ * of firmware.
+ *
+ * GuC address space:
+ * GuC does not allow any gfx GGTT address that falls into range [0, WOPCM_TOP),
+ * which is reserved for Boot ROM, SRAM and WOPCM. Currently this top address is
+ * 512K. In order to exclude 0-512K address space from GGTT, all gfx objects
+ * used by GuC is pinned with PIN_OFFSET_BIAS along with size of WOPCM.
+ *
+ * Firmware log:
+ * Firmware log is enabled by setting i915.guc_log_level to non-negative level.
+ * Log data is printed out via reading debugfs i915_guc_log_dump. Reading from
+ * i915_guc_load_status will print out firmware loading status and scratch
+ * registers value.
+ *
+ */
+
+#define I915_SKL_GUC_UCODE "i915/skl_guc_ver4.bin"
+MODULE_FIRMWARE(I915_SKL_GUC_UCODE);
+
+/* User-friendly representation of an enum */
+const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status)
+{
+ switch (status) {
+ case GUC_FIRMWARE_FAIL:
+ return "FAIL";
+ case GUC_FIRMWARE_NONE:
+ return "NONE";
+ case GUC_FIRMWARE_PENDING:
+ return "PENDING";
+ case GUC_FIRMWARE_SUCCESS:
+ return "SUCCESS";
+ default:
+ return "UNKNOWN!";
+ }
+};
+
+static void direct_interrupts_to_host(struct drm_i915_private *dev_priv)
+{
+ struct intel_engine_cs *ring;
+ int i, irqs;
+
+ /* tell all command streamers NOT to forward interrupts and vblank to GuC */
+ irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_NEVER);
+ irqs |= _MASKED_BIT_DISABLE(GFX_INTERRUPT_STEERING);
+ for_each_ring(ring, dev_priv, i)
+ I915_WRITE(RING_MODE_GEN7(ring), irqs);
+
+ /* route all GT interrupts to the host */
+ I915_WRITE(GUC_BCS_RCS_IER, 0);
+ I915_WRITE(GUC_VCS2_VCS1_IER, 0);
+ I915_WRITE(GUC_WD_VECS_IER, 0);
+}
+
+static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv)
+{
+ struct intel_engine_cs *ring;
+ int i, irqs;
+
+ /* tell all command streamers to forward interrupts and vblank to GuC */
+ irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_ALWAYS);
+ irqs |= _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
+ for_each_ring(ring, dev_priv, i)
+ I915_WRITE(RING_MODE_GEN7(ring), irqs);
+
+ /* route USER_INTERRUPT to Host, all others are sent to GuC. */
+ irqs = GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
+ GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
+ /* These three registers have the same bit definitions */
+ I915_WRITE(GUC_BCS_RCS_IER, ~irqs);
+ I915_WRITE(GUC_VCS2_VCS1_IER, ~irqs);
+ I915_WRITE(GUC_WD_VECS_IER, ~irqs);
+}
+
+static u32 get_gttype(struct drm_i915_private *dev_priv)
+{
+ /* XXX: GT type based on PCI device ID? field seems unused by fw */
+ return 0;
+}
+
+static u32 get_core_family(struct drm_i915_private *dev_priv)
+{
+ switch (INTEL_INFO(dev_priv)->gen) {
+ case 9:
+ return GFXCORE_FAMILY_GEN9;
+
+ default:
+ DRM_ERROR("GUC: unsupported core family\n");
+ return GFXCORE_FAMILY_UNKNOWN;
+ }
+}
+
+static void set_guc_init_params(struct drm_i915_private *dev_priv)
+{
+ struct intel_guc *guc = &dev_priv->guc;
+ u32 params[GUC_CTL_MAX_DWORDS];
+ int i;
+
+ memset(¶ms, 0, sizeof(params));
+
+ params[GUC_CTL_DEVICE_INFO] |=
+ (get_gttype(dev_priv) << GUC_CTL_GTTYPE_SHIFT) |
+ (get_core_family(dev_priv) << GUC_CTL_COREFAMILY_SHIFT);
+
+ /*
+ * GuC ARAT increment is 10 ns. GuC default scheduler quantum is one
+ * second. This ARAR is calculated by:
+ * Scheduler-Quantum-in-ns / ARAT-increment-in-ns = 1000000000 / 10
+ */
+ params[GUC_CTL_ARAT_HIGH] = 0;
+ params[GUC_CTL_ARAT_LOW] = 100000000;
+
+ params[GUC_CTL_WA] |= GUC_CTL_WA_UK_BY_DRIVER;
+
+ params[GUC_CTL_FEATURE] |= GUC_CTL_DISABLE_SCHEDULER |
+ GUC_CTL_VCS2_ENABLED;
+
+ if (i915.guc_log_level >= 0) {
+ params[GUC_CTL_LOG_PARAMS] = guc->log_flags;
+ params[GUC_CTL_DEBUG] =
+ i915.guc_log_level << GUC_LOG_VERBOSITY_SHIFT;
+ }
+
+ /* If GuC submission is enabled, set up additional parameters here */
+ if (i915.enable_guc_submission) {
+ u32 pgs = i915_gem_obj_ggtt_offset(dev_priv->guc.ctx_pool_obj);
+ u32 ctx_in_16 = GUC_MAX_GPU_CONTEXTS / 16;
+
+ pgs >>= PAGE_SHIFT;
+ params[GUC_CTL_CTXINFO] = (pgs << GUC_CTL_BASE_ADDR_SHIFT) |
+ (ctx_in_16 << GUC_CTL_CTXNUM_IN16_SHIFT);
+
+ params[GUC_CTL_FEATURE] |= GUC_CTL_KERNEL_SUBMISSIONS;
+
+ /* Unmask this bit to enable the GuC's internal scheduler */
+ params[GUC_CTL_FEATURE] &= ~GUC_CTL_DISABLE_SCHEDULER;
+ }
+
+ I915_WRITE(SOFT_SCRATCH(0), 0);
+
+ for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
+ I915_WRITE(SOFT_SCRATCH(1 + i), params[i]);
+}
+
+/*
+ * Read the GuC status register (GUC_STATUS) and store it in the
+ * specified location; then return a boolean indicating whether
+ * the value matches either of two values representing completion
+ * of the GuC boot process.
+ *
+ * This is used for polling the GuC status in a wait_for_atomic()
+ * loop below.
+ */
+static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
+ u32 *status)
+{
+ u32 val = I915_READ(GUC_STATUS);
+ u32 uk_val = val & GS_UKERNEL_MASK;
+ *status = val;
+ return (uk_val == GS_UKERNEL_READY ||
+ ((val & GS_MIA_CORE_STATE) && uk_val == GS_UKERNEL_LAPIC_DONE));
+}
+
+/*
+ * Transfer the firmware image to RAM for execution by the microcontroller.
+ *
+ * GuC Firmware layout:
+ * +-------------------------------+ ----
+ * | CSS header | 128B
+ * | contains major/minor version |
+ * +-------------------------------+ ----
+ * | uCode |
+ * +-------------------------------+ ----
+ * | RSA signature | 256B
+ * +-------------------------------+ ----
+ *
+ * Architecturally, the DMA engine is bidirectional, and can potentially even
+ * transfer between GTT locations. This functionality is left out of the API
+ * for now as there is no need for it.
+ *
+ * Note that GuC needs the CSS header plus uKernel code to be copied by the
+ * DMA engine in one operation, whereas the RSA signature is loaded via MMIO.
+ */
+
+#define UOS_CSS_HEADER_OFFSET 0
+#define UOS_VER_MINOR_OFFSET 0x44
+#define UOS_VER_MAJOR_OFFSET 0x46
+#define UOS_CSS_HEADER_SIZE 0x80
+#define UOS_RSA_SIG_SIZE 0x100
+
+static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv)
+{
+ struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
+ struct drm_i915_gem_object *fw_obj = guc_fw->guc_fw_obj;
+ unsigned long offset;
+ struct sg_table *sg = fw_obj->pages;
+ u32 status, ucode_size, rsa[UOS_RSA_SIG_SIZE / sizeof(u32)];
+ int i, ret = 0;
+
+ /* uCode size, also is where RSA signature starts */
+ offset = ucode_size = guc_fw->guc_fw_size - UOS_RSA_SIG_SIZE;
+ I915_WRITE(DMA_COPY_SIZE, ucode_size);
+
+ /* Copy RSA signature from the fw image to HW for verification */
+ sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, UOS_RSA_SIG_SIZE, offset);
+ for (i = 0; i < UOS_RSA_SIG_SIZE / sizeof(u32); i++)
+ I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]);
+
+ /* Set the source address for the new blob */
+ offset = i915_gem_obj_ggtt_offset(fw_obj);
+ I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
+ I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
+
+ /*
+ * Set the DMA destination. Current uCode expects the code to be
+ * loaded at 8k; locations below this are used for the stack.
+ */
+ I915_WRITE(DMA_ADDR_1_LOW, 0x2000);
+ I915_WRITE(DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM);
+
+ /* Finally start the DMA */
+ I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(UOS_MOVE | START_DMA));
+
+ /*
+ * Spin-wait for the DMA to complete & the GuC to start up.
+ * NB: Docs recommend not using the interrupt for completion.
+ * Measurements indicate this should take no more than 20ms, so a
+ * timeout here indicates that the GuC has failed and is unusable.
+ * (Higher levels of the driver will attempt to fall back to
+ * execlist mode if this happens.)
+ */
+ ret = wait_for_atomic(guc_ucode_response(dev_priv, &status), 100);
+
+ DRM_DEBUG_DRIVER("DMA status 0x%x, GuC status 0x%x\n",
+ I915_READ(DMA_CTRL), status);
+
+ if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) {
+ DRM_ERROR("GuC firmware signature verification failed\n");
+ ret = -ENOEXEC;
+ }
+
+ DRM_DEBUG_DRIVER("returning %d\n", ret);
+
+ return ret;
+}
+
+/*
+ * Load the GuC firmware blob into the MinuteIA.
+ */
+static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
+{
+ struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
+ struct drm_device *dev = dev_priv->dev;
+ int ret;
+
+ ret = i915_gem_object_set_to_gtt_domain(guc_fw->guc_fw_obj, false);
+ if (ret) {
+ DRM_DEBUG_DRIVER("set-domain failed %d\n", ret);
+ return ret;
+ }
+
+ ret = i915_gem_obj_ggtt_pin(guc_fw->guc_fw_obj, 0, 0);
+ if (ret) {
+ DRM_DEBUG_DRIVER("pin failed %d\n", ret);
+ return ret;
+ }
+
+ /* Invalidate GuC TLB to let GuC take the latest updates to GTT. */
+ I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+
+ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+
+ /* init WOPCM */
+ I915_WRITE(GUC_WOPCM_SIZE, GUC_WOPCM_SIZE_VALUE);
+ I915_WRITE(DMA_GUC_WOPCM_OFFSET, GUC_WOPCM_OFFSET_VALUE);
+
+ /* Enable MIA caching. GuC clock gating is disabled. */
+ I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE);
+
+ /* WaDisableMinuteIaClockGating:skl,bxt */
+ if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) ||
+ (IS_BROXTON(dev) && INTEL_REVID(dev) == BXT_REVID_A0)) {
+ I915_WRITE(GUC_SHIM_CONTROL, (I915_READ(GUC_SHIM_CONTROL) &
+ ~GUC_ENABLE_MIA_CLOCK_GATING));
+ }
+
+ /* WaC6DisallowByGfxPause*/
+ I915_WRITE(GEN6_GFXPAUSE, 0x30FFF);
+
+ if (IS_BROXTON(dev))
+ I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
+ else
+ I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
+
+ if (IS_GEN9(dev)) {
+ /* DOP Clock Gating Enable for GuC clocks */
+ I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
+ I915_READ(GEN7_MISCCPCTL)));
+
+ /* allows for 5us before GT can go to RC6 */
+ I915_WRITE(GUC_ARAT_C6DIS, 0x1FF);
+ }
+
+ set_guc_init_params(dev_priv);
+
+ ret = guc_ucode_xfer_dma(dev_priv);
+
+ intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+
+ /*
+ * We keep the object pages for reuse during resume. But we can unpin it
+ * now that DMA has completed, so it doesn't continue to take up space.
+ */
+ i915_gem_object_ggtt_unpin(guc_fw->guc_fw_obj);
+
+ return ret;
+}
+
+/**
+ * intel_guc_ucode_load() - load GuC uCode into the device
+ * @dev: drm device
+ *
+ * Called from gem_init_hw() during driver loading and also after a GPU reset.
+ *
+ * The firmware image should have already been fetched into memory by the
+ * earlier call to intel_guc_ucode_init(), so here we need only check that
+ * is succeeded, and then transfer the image to the h/w.
+ *
+ * Return: non-zero code on error
+ */
+int intel_guc_ucode_load(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
+ int err = 0;
+
+ DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n",
+ intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status),
+ intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
+
+ direct_interrupts_to_host(dev_priv);
+
+ if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_NONE)
+ return 0;
+
+ if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_SUCCESS &&
+ guc_fw->guc_fw_load_status == GUC_FIRMWARE_FAIL)
+ return -ENOEXEC;
+
+ guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING;
+
+ DRM_DEBUG_DRIVER("GuC fw fetch status %s\n",
+ intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status));
+
+ switch (guc_fw->guc_fw_fetch_status) {
+ case GUC_FIRMWARE_FAIL:
+ /* something went wrong :( */
+ err = -EIO;
+ goto fail;
+
+ case GUC_FIRMWARE_NONE:
+ case GUC_FIRMWARE_PENDING:
+ default:
+ /* "can't happen" */
+ WARN_ONCE(1, "GuC fw %s invalid guc_fw_fetch_status %s [%d]\n",
+ guc_fw->guc_fw_path,
+ intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status),
+ guc_fw->guc_fw_fetch_status);
+ err = -ENXIO;
+ goto fail;
+
+ case GUC_FIRMWARE_SUCCESS:
+ break;
+ }
+
+ err = i915_guc_submission_init(dev);
+ if (err)
+ goto fail;
+
+ err = guc_ucode_xfer(dev_priv);
+ if (err)
+ goto fail;
+
+ guc_fw->guc_fw_load_status = GUC_FIRMWARE_SUCCESS;
+
+ DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n",
+ intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status),
+ intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
+
+ if (i915.enable_guc_submission) {
+ /* The execbuf_client will be recreated. Release it first. */
+ i915_guc_submission_disable(dev);
+
+ err = i915_guc_submission_enable(dev);
+ if (err)
+ goto fail;
+ direct_interrupts_to_guc(dev_priv);
+ }
+
+ return 0;
+
+fail:
+ if (guc_fw->guc_fw_load_status == GUC_FIRMWARE_PENDING)
+ guc_fw->guc_fw_load_status = GUC_FIRMWARE_FAIL;
+
+ direct_interrupts_to_host(dev_priv);
+ i915_guc_submission_disable(dev);
+
+ return err;
+}
+
+static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
+{
+ struct drm_i915_gem_object *obj;
+ const struct firmware *fw;
+ const u8 *css_header;
+ const size_t minsize = UOS_CSS_HEADER_SIZE + UOS_RSA_SIG_SIZE;
+ const size_t maxsize = GUC_WOPCM_SIZE_VALUE + UOS_RSA_SIG_SIZE
+ - 0x8000; /* 32k reserved (8K stack + 24k context) */
+ int err;
+
+ DRM_DEBUG_DRIVER("before requesting firmware: GuC fw fetch status %s\n",
+ intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status));
+
+ err = request_firmware(&fw, guc_fw->guc_fw_path, &dev->pdev->dev);
+ if (err)
+ goto fail;
+ if (!fw)
+ goto fail;
+
+ DRM_DEBUG_DRIVER("fetch GuC fw from %s succeeded, fw %p\n",
+ guc_fw->guc_fw_path, fw);
+ DRM_DEBUG_DRIVER("firmware file size %zu (minimum %zu, maximum %zu)\n",
+ fw->size, minsize, maxsize);
+
+ /* Check the size of the blob befoe examining buffer contents */
+ if (fw->size < minsize || fw->size > maxsize)
+ goto fail;
+
+ /*
+ * The GuC firmware image has the version number embedded at a well-known
+ * offset within the firmware blob; note that major / minor version are
+ * TWO bytes each (i.e. u16), although all pointers and offsets are defined
+ * in terms of bytes (u8).
+ */
+ css_header = fw->data + UOS_CSS_HEADER_OFFSET;
+ guc_fw->guc_fw_major_found = *(u16 *)(css_header + UOS_VER_MAJOR_OFFSET);
+ guc_fw->guc_fw_minor_found = *(u16 *)(css_header + UOS_VER_MINOR_OFFSET);
+
+ if (guc_fw->guc_fw_major_found != guc_fw->guc_fw_major_wanted ||
+ guc_fw->guc_fw_minor_found < guc_fw->guc_fw_minor_wanted) {
+ DRM_ERROR("GuC firmware version %d.%d, required %d.%d\n",
+ guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found,
+ guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted);
+ err = -ENOEXEC;
+ goto fail;
+ }
+
+ DRM_DEBUG_DRIVER("firmware version %d.%d OK (minimum %d.%d)\n",
+ guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found,
+ guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted);
+
+ mutex_lock(&dev->struct_mutex);
+ obj = i915_gem_object_create_from_data(dev, fw->data, fw->size);
+ mutex_unlock(&dev->struct_mutex);
+ if (IS_ERR_OR_NULL(obj)) {
+ err = obj ? PTR_ERR(obj) : -ENOMEM;
+ goto fail;
+ }
+
+ guc_fw->guc_fw_obj = obj;
+ guc_fw->guc_fw_size = fw->size;
+
+ DRM_DEBUG_DRIVER("GuC fw fetch status SUCCESS, obj %p\n",
+ guc_fw->guc_fw_obj);
+
+ release_firmware(fw);
+ guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_SUCCESS;
+ return;
+
+fail:
+ DRM_DEBUG_DRIVER("GuC fw fetch status FAIL; err %d, fw %p, obj %p\n",
+ err, fw, guc_fw->guc_fw_obj);
+ DRM_ERROR("Failed to fetch GuC firmware from %s (error %d)\n",
+ guc_fw->guc_fw_path, err);
+
+ obj = guc_fw->guc_fw_obj;
+ if (obj)
+ drm_gem_object_unreference(&obj->base);
+ guc_fw->guc_fw_obj = NULL;
+
+ release_firmware(fw); /* OK even if fw is NULL */
+ guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_FAIL;
+}
+
+/**
+ * intel_guc_ucode_init() - define parameters and fetch firmware
+ * @dev: drm device
+ *
+ * Called early during driver load, but after GEM is initialised.
+ *
+ * The firmware will be transferred to the GuC's memory later,
+ * when intel_guc_ucode_load() is called.
+ */
+void intel_guc_ucode_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
+ const char *fw_path;
+
+ if (!HAS_GUC_SCHED(dev))
+ i915.enable_guc_submission = false;
+
+ if (!HAS_GUC_UCODE(dev)) {
+ fw_path = NULL;
+ } else if (IS_SKYLAKE(dev)) {
+ fw_path = I915_SKL_GUC_UCODE;
+ guc_fw->guc_fw_major_wanted = 4;
+ guc_fw->guc_fw_minor_wanted = 3;
+ } else {
+ i915.enable_guc_submission = false;
+ fw_path = ""; /* unknown device */
+ }
+
+ guc_fw->guc_dev = dev;
+ guc_fw->guc_fw_path = fw_path;
+ guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_NONE;
+ guc_fw->guc_fw_load_status = GUC_FIRMWARE_NONE;
+
+ if (fw_path == NULL)
+ return;
+
+ if (*fw_path == '\0') {
+ DRM_ERROR("No GuC firmware known for this platform\n");
+ guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_FAIL;
+ return;
+ }
+
+ guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_PENDING;
+ DRM_DEBUG_DRIVER("GuC firmware pending, path %s\n", fw_path);
+ guc_fw_fetch(dev, guc_fw);
+ /* status must now be FAIL or SUCCESS */
+}
+
+/**
+ * intel_guc_ucode_fini() - clean up all allocated resources
+ * @dev: drm device
+ */
+void intel_guc_ucode_fini(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
+
+ direct_interrupts_to_host(dev_priv);
+ i915_guc_submission_fini(dev);
+
+ mutex_lock(&dev->struct_mutex);
+ if (guc_fw->guc_fw_obj)
+ drm_gem_object_unreference(&guc_fw->guc_fw_obj->base);
+ guc_fw->guc_fw_obj = NULL;
+ mutex_unlock(&dev->struct_mutex);
+
+ guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_NONE;
+}
}
}
-static u32 hsw_infoframe_data_reg(enum hdmi_infoframe_type type,
- enum transcoder cpu_transcoder,
- struct drm_i915_private *dev_priv)
+static u32 hsw_dip_data_reg(struct drm_i915_private *dev_priv,
+ enum transcoder cpu_transcoder,
+ enum hdmi_infoframe_type type,
+ int i)
{
switch (type) {
case HDMI_INFOFRAME_TYPE_AVI:
- return HSW_TVIDEO_DIP_AVI_DATA(cpu_transcoder);
+ return HSW_TVIDEO_DIP_AVI_DATA(cpu_transcoder, i);
case HDMI_INFOFRAME_TYPE_SPD:
- return HSW_TVIDEO_DIP_SPD_DATA(cpu_transcoder);
+ return HSW_TVIDEO_DIP_SPD_DATA(cpu_transcoder, i);
case HDMI_INFOFRAME_TYPE_VENDOR:
- return HSW_TVIDEO_DIP_VS_DATA(cpu_transcoder);
+ return HSW_TVIDEO_DIP_VS_DATA(cpu_transcoder, i);
default:
DRM_DEBUG_DRIVER("unknown info frame type %d\n", type);
return 0;
struct drm_device *dev = encoder->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
- u32 ctl_reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder);
+ enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
+ u32 ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
u32 data_reg;
int i;
u32 val = I915_READ(ctl_reg);
- data_reg = hsw_infoframe_data_reg(type,
- intel_crtc->config->cpu_transcoder,
- dev_priv);
+ data_reg = hsw_dip_data_reg(dev_priv, cpu_transcoder, type, 0);
if (data_reg == 0)
return;
mmiowb();
for (i = 0; i < len; i += 4) {
- I915_WRITE(data_reg + i, *data);
+ I915_WRITE(hsw_dip_data_reg(dev_priv, cpu_transcoder,
+ type, i >> 2), *data);
data++;
}
/* Write every possible data byte to force correct ECC calculation. */
for (; i < VIDEO_DIP_DATA_SIZE; i += 4)
- I915_WRITE(data_reg + i, 0);
+ I915_WRITE(hsw_dip_data_reg(dev_priv, cpu_transcoder,
+ type, i >> 2), 0);
mmiowb();
val |= hsw_infoframe_enable(type);
}
static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
union hdmi_infoframe frame;
int ret;
- /* Set user selected PAR to incoming mode's member */
- adjusted_mode->picture_aspect_ratio = intel_hdmi->aspect_ratio;
-
ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
adjusted_mode);
if (ret < 0) {
static void
intel_hdmi_set_hdmi_infoframe(struct drm_encoder *encoder,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *adjusted_mode)
{
union hdmi_infoframe frame;
int ret;
static void g4x_set_infoframes(struct drm_encoder *encoder,
bool enable,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = encoder->dev->dev_private;
struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
static void ibx_set_infoframes(struct drm_encoder *encoder,
bool enable,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = encoder->dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
static void cpt_set_infoframes(struct drm_encoder *encoder,
bool enable,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = encoder->dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
static void vlv_set_infoframes(struct drm_encoder *encoder,
bool enable,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = encoder->dev->dev_private;
struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
static void hsw_set_infoframes(struct drm_encoder *encoder,
bool enable,
- struct drm_display_mode *adjusted_mode)
+ const struct drm_display_mode *adjusted_mode)
{
struct drm_i915_private *dev_priv = encoder->dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
- struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
u32 hdmi_val;
hdmi_val = SDVO_ENCODING_HDMI;
- if (!HAS_PCH_SPLIT(dev))
- hdmi_val |= intel_hdmi->color_range;
+ if (!HAS_PCH_SPLIT(dev) && crtc->config->limited_color_range)
+ hdmi_val |= HDMI_COLOR_RANGE_16_235;
if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
hdmi_val |= SDVO_VSYNC_ACTIVE_HIGH;
if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
if (intel_hdmi->color_range_auto) {
/* See CEA-861-E - 5.1 Default Encoding Parameters */
- if (pipe_config->has_hdmi_sink &&
- drm_match_cea_mode(adjusted_mode) > 1)
- intel_hdmi->color_range = HDMI_COLOR_RANGE_16_235;
- else
- intel_hdmi->color_range = 0;
+ pipe_config->limited_color_range =
+ pipe_config->has_hdmi_sink &&
+ drm_match_cea_mode(adjusted_mode) > 1;
+ } else {
+ pipe_config->limited_color_range =
+ intel_hdmi->limited_color_range;
}
if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) {
clock_12bpc *= 2;
}
- if (intel_hdmi->color_range)
- pipe_config->limited_color_range = true;
-
if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev))
pipe_config->has_pch_encoder = true;
return false;
}
+ /* Set user selected PAR to incoming mode's member */
+ adjusted_mode->picture_aspect_ratio = intel_hdmi->aspect_ratio;
+
return true;
}
}
static bool
-intel_hdmi_set_edid(struct drm_connector *connector)
+intel_hdmi_set_edid(struct drm_connector *connector, bool force)
{
struct drm_i915_private *dev_priv = to_i915(connector->dev);
struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
struct intel_encoder *intel_encoder =
&hdmi_to_dig_port(intel_hdmi)->base;
enum intel_display_power_domain power_domain;
- struct edid *edid;
+ struct edid *edid = NULL;
bool connected = false;
power_domain = intel_display_port_power_domain(intel_encoder);
intel_display_power_get(dev_priv, power_domain);
- edid = drm_get_edid(connector,
- intel_gmbus_get_adapter(dev_priv,
- intel_hdmi->ddc_bus));
+ if (force)
+ edid = drm_get_edid(connector,
+ intel_gmbus_get_adapter(dev_priv,
+ intel_hdmi->ddc_bus));
intel_display_power_put(dev_priv, power_domain);
intel_hdmi_detect(struct drm_connector *connector, bool force)
{
enum drm_connector_status status;
+ struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+ struct drm_i915_private *dev_priv = to_i915(connector->dev);
+ bool live_status = false;
+ unsigned int retry = 3;
DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
connector->base.id, connector->name);
+ while (!live_status && --retry) {
+ live_status = intel_digital_port_connected(dev_priv,
+ hdmi_to_dig_port(intel_hdmi));
+ mdelay(10);
+ }
+
+ if (!live_status)
+ DRM_DEBUG_KMS("Live status not up!");
+
intel_hdmi_unset_edid(connector);
- if (intel_hdmi_set_edid(connector)) {
+ if (intel_hdmi_set_edid(connector, live_status)) {
struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
if (connector->status != connector_status_connected)
return;
- intel_hdmi_set_edid(connector);
+ intel_hdmi_set_edid(connector, true);
hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
}
if (property == dev_priv->broadcast_rgb_property) {
bool old_auto = intel_hdmi->color_range_auto;
- uint32_t old_range = intel_hdmi->color_range;
+ bool old_range = intel_hdmi->limited_color_range;
switch (val) {
case INTEL_BROADCAST_RGB_AUTO:
break;
case INTEL_BROADCAST_RGB_FULL:
intel_hdmi->color_range_auto = false;
- intel_hdmi->color_range = 0;
+ intel_hdmi->limited_color_range = false;
break;
case INTEL_BROADCAST_RGB_LIMITED:
intel_hdmi->color_range_auto = false;
- intel_hdmi->color_range = HDMI_COLOR_RANGE_16_235;
+ intel_hdmi->limited_color_range = true;
break;
default:
return -EINVAL;
}
if (old_auto == intel_hdmi->color_range_auto &&
- old_range == intel_hdmi->color_range)
+ old_range == intel_hdmi->limited_color_range)
return 0;
goto done;
{
struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
- struct drm_display_mode *adjusted_mode =
- &intel_crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
intel_hdmi_prepare(encoder);
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc =
to_intel_crtc(encoder->base.crtc);
- struct drm_display_mode *adjusted_mode =
- &intel_crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
enum dpio_channel port = vlv_dport_to_channel(dport);
int pipe = intel_crtc->pipe;
u32 val;
mutex_unlock(&dev_priv->sb_lock);
}
+static void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+ bool reset)
+{
+ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+ enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
+ struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+ enum pipe pipe = crtc->pipe;
+ uint32_t val;
+
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
+ if (reset)
+ val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
+ else
+ val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
+
+ if (crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
+ if (reset)
+ val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
+ else
+ val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
+ }
+
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
+ val |= CHV_PCS_REQ_SOFTRESET_EN;
+ if (reset)
+ val &= ~DPIO_PCS_CLK_SOFT_RESET;
+ else
+ val |= DPIO_PCS_CLK_SOFT_RESET;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
+
+ if (crtc->config->lane_count > 2) {
+ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
+ val |= CHV_PCS_REQ_SOFTRESET_EN;
+ if (reset)
+ val &= ~DPIO_PCS_CLK_SOFT_RESET;
+ else
+ val |= DPIO_PCS_CLK_SOFT_RESET;
+ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
+ }
+}
+
static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder)
{
struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
intel_hdmi_prepare(encoder);
+ /*
+ * Must trick the second common lane into life.
+ * Otherwise we can't even access the PLL.
+ */
+ if (ch == DPIO_CH0 && pipe == PIPE_B)
+ dport->release_cl2_override =
+ !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true);
+
+ chv_phy_powergate_lanes(encoder, true, 0x0);
+
mutex_lock(&dev_priv->sb_lock);
+ /* Assert data lane reset */
+ chv_data_lane_soft_reset(encoder, true);
+
/* program left/right clock distribution */
if (pipe != PIPE_B) {
val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
mutex_unlock(&dev_priv->sb_lock);
}
+static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder)
+{
+ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+ enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
+ u32 val;
+
+ mutex_lock(&dev_priv->sb_lock);
+
+ /* disable left/right clock distribution */
+ if (pipe != PIPE_B) {
+ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
+ val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
+ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
+ } else {
+ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
+ val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
+ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
+ }
+
+ mutex_unlock(&dev_priv->sb_lock);
+
+ /*
+ * Leave the power down bit cleared for at least one
+ * lane so that chv_powergate_phy_ch() will power
+ * on something when the channel is otherwise unused.
+ * When the port is off and the override is removed
+ * the lanes power down anyway, so otherwise it doesn't
+ * really matter what the state of power down bits is
+ * after this.
+ */
+ chv_phy_powergate_lanes(encoder, false, 0x0);
+}
+
static void vlv_hdmi_post_disable(struct intel_encoder *encoder)
{
struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
static void chv_hdmi_post_disable(struct intel_encoder *encoder)
{
- struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
struct drm_device *dev = encoder->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_crtc *intel_crtc =
- to_intel_crtc(encoder->base.crtc);
- enum dpio_channel ch = vlv_dport_to_channel(dport);
- enum pipe pipe = intel_crtc->pipe;
- u32 val;
mutex_lock(&dev_priv->sb_lock);
- /* Propagate soft reset to data lane reset */
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
- val |= CHV_PCS_REQ_SOFTRESET_EN;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
- val |= CHV_PCS_REQ_SOFTRESET_EN;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
- val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
- val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
+ /* Assert data lane reset */
+ chv_data_lane_soft_reset(encoder, true);
mutex_unlock(&dev_priv->sb_lock);
}
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc =
to_intel_crtc(encoder->base.crtc);
- struct drm_display_mode *adjusted_mode =
- &intel_crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
enum dpio_channel ch = vlv_dport_to_channel(dport);
int pipe = intel_crtc->pipe;
int data, i, stagger;
val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
- /* Deassert soft data lane reset*/
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
- val |= CHV_PCS_REQ_SOFTRESET_EN;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
- val |= CHV_PCS_REQ_SOFTRESET_EN;
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
- val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
-
- val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
- val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
- vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
-
/* Program Tx latency optimal setting */
for (i = 0; i < 4; i++) {
/* Set the upar bit */
DPIO_TX1_STAGGER_MULT(7) |
DPIO_TX2_STAGGER_MULT(5));
+ /* Deassert data lane reset */
+ chv_data_lane_soft_reset(encoder, false);
+
/* Clear calc init */
val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
for (i = 0; i < 4; i++) {
val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
+
val &= ~DPIO_SWING_MARGIN000_MASK;
val |= 102 << DPIO_SWING_MARGIN000_SHIFT;
+
+ /*
+ * Supposedly this value shouldn't matter when unique transition
+ * scale is disabled, but in fact it does matter. Let's just
+ * always program the same value and hope it's OK.
+ */
+ val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
+ val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT;
+
vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
}
- /* Disable unique transition scale */
+ /*
+ * The document said it needs to set bit 27 for ch0 and bit 26
+ * for ch1. Might be a typo in the doc.
+ * For now, for this unique transition scale selection, set bit
+ * 27 for ch0 and ch1.
+ */
for (i = 0; i < 4; i++) {
val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i));
val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
}
- /* Additional steps for 1200mV-0dB */
-#if 0
- val = vlv_dpio_read(dev_priv, pipe, VLV_TX_DW3(ch));
- if (ch)
- val |= DPIO_TX_UNIQ_TRANS_SCALE_CH1;
- else
- val |= DPIO_TX_UNIQ_TRANS_SCALE_CH0;
- vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(ch), val);
-
- vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(ch),
- vlv_dpio_read(dev_priv, pipe, VLV_TX_DW2(ch)) |
- (0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT));
-#endif
/* Start swing calculation */
val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
- /* LRC Bypass */
- val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
- val |= DPIO_LRC_BYPASS;
- vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, val);
-
mutex_unlock(&dev_priv->sb_lock);
intel_hdmi->set_infoframes(&encoder->base,
g4x_enable_hdmi(encoder);
vlv_wait_port_ready(dev_priv, dport, 0x0);
+
+ /* Second common lane will stay alive on its own now */
+ if (dport->release_cl2_override) {
+ chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false);
+ dport->release_cl2_override = false;
+ }
}
static void intel_hdmi_destroy(struct drm_connector *connector)
.destroy = intel_encoder_destroy,
};
-static void
-intel_attach_aspect_ratio_property(struct drm_connector *connector)
-{
- if (!drm_mode_create_aspect_ratio_property(connector->dev))
- drm_object_attach_property(&connector->base,
- connector->dev->mode_config.aspect_ratio_property,
- DRM_MODE_PICTURE_ASPECT_NONE);
-}
-
static void
intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *connector)
{
intel_hdmi->ddc_bus = GMBUS_PIN_1_BXT;
else
intel_hdmi->ddc_bus = GMBUS_PIN_DPB;
- intel_encoder->hpd_pin = HPD_PORT_B;
+ /*
+ * On BXT A0/A1, sw needs to activate DDIA HPD logic and
+ * interrupts to check the external panel connection.
+ */
+ if (IS_BROXTON(dev_priv) && (INTEL_REVID(dev) < BXT_REVID_B0))
+ intel_encoder->hpd_pin = HPD_PORT_A;
+ else
+ intel_encoder->hpd_pin = HPD_PORT_B;
break;
case PORT_C:
if (IS_BROXTON(dev_priv))
intel_connector_attach_encoder(intel_connector, intel_encoder);
drm_connector_register(connector);
+ intel_hdmi->attached_connector = intel_connector;
/* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
* 0xd. Failure to do so will result in spurious interrupts being
intel_encoder->pre_enable = chv_hdmi_pre_enable;
intel_encoder->enable = vlv_enable_hdmi;
intel_encoder->post_disable = chv_hdmi_post_disable;
+ intel_encoder->post_pll_disable = chv_hdmi_post_pll_disable;
} else if (IS_VALLEYVIEW(dev)) {
intel_encoder->pre_pll_enable = vlv_hdmi_pre_pll_enable;
intel_encoder->pre_enable = vlv_hdmi_pre_enable;
/* Enable polling and queue hotplug re-enabling. */
if (hpd_disabled) {
- drm_kms_helper_poll_enable(dev);
+ drm_kms_helper_poll_enable_locked(dev);
mod_delayed_work(system_wq, &dev_priv->hotplug.reenable_work,
msecs_to_jiffies(HPD_STORM_REENABLE_DELAY));
}
{
struct drm_i915_private *dev_priv = dev->dev_private;
- I915_WRITE(dev_priv->gpio_mmio_base + GMBUS0, 0);
- I915_WRITE(dev_priv->gpio_mmio_base + GMBUS4, 0);
+ I915_WRITE(GMBUS0, 0);
+ I915_WRITE(GMBUS4, 0);
}
static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable)
u32 gmbus4_irq_en)
{
int i;
- int reg_offset = dev_priv->gpio_mmio_base;
u32 gmbus2 = 0;
DEFINE_WAIT(wait);
/* Important: The hw handles only the first bit, so set only one! Since
* we also need to check for NAKs besides the hw ready/idle signal, we
* need to wake up periodically and check that ourselves. */
- I915_WRITE(GMBUS4 + reg_offset, gmbus4_irq_en);
+ I915_WRITE(GMBUS4, gmbus4_irq_en);
for (i = 0; i < msecs_to_jiffies_timeout(50); i++) {
prepare_to_wait(&dev_priv->gmbus_wait_queue, &wait,
TASK_UNINTERRUPTIBLE);
- gmbus2 = I915_READ_NOTRACE(GMBUS2 + reg_offset);
+ gmbus2 = I915_READ_NOTRACE(GMBUS2);
if (gmbus2 & (GMBUS_SATOER | gmbus2_status))
break;
}
finish_wait(&dev_priv->gmbus_wait_queue, &wait);
- I915_WRITE(GMBUS4 + reg_offset, 0);
+ I915_WRITE(GMBUS4, 0);
if (gmbus2 & GMBUS_SATOER)
return -ENXIO;
gmbus_wait_idle(struct drm_i915_private *dev_priv)
{
int ret;
- int reg_offset = dev_priv->gpio_mmio_base;
-#define C ((I915_READ_NOTRACE(GMBUS2 + reg_offset) & GMBUS_ACTIVE) == 0)
+#define C ((I915_READ_NOTRACE(GMBUS2) & GMBUS_ACTIVE) == 0)
if (!HAS_GMBUS_IRQ(dev_priv->dev))
return wait_for(C, 10);
/* Important: The hw handles only the first bit, so set only one! */
- I915_WRITE(GMBUS4 + reg_offset, GMBUS_IDLE_EN);
+ I915_WRITE(GMBUS4, GMBUS_IDLE_EN);
ret = wait_event_timeout(dev_priv->gmbus_wait_queue, C,
msecs_to_jiffies_timeout(10));
- I915_WRITE(GMBUS4 + reg_offset, 0);
+ I915_WRITE(GMBUS4, 0);
if (ret)
return 0;
unsigned short addr, u8 *buf, unsigned int len,
u32 gmbus1_index)
{
- int reg_offset = dev_priv->gpio_mmio_base;
-
- I915_WRITE(GMBUS1 + reg_offset,
+ I915_WRITE(GMBUS1,
gmbus1_index |
GMBUS_CYCLE_WAIT |
(len << GMBUS_BYTE_COUNT_SHIFT) |
if (ret)
return ret;
- val = I915_READ(GMBUS3 + reg_offset);
+ val = I915_READ(GMBUS3);
do {
*buf++ = val & 0xff;
val >>= 8;
gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv,
unsigned short addr, u8 *buf, unsigned int len)
{
- int reg_offset = dev_priv->gpio_mmio_base;
unsigned int chunk_size = len;
u32 val, loop;
len -= 1;
}
- I915_WRITE(GMBUS3 + reg_offset, val);
- I915_WRITE(GMBUS1 + reg_offset,
+ I915_WRITE(GMBUS3, val);
+ I915_WRITE(GMBUS1,
GMBUS_CYCLE_WAIT |
(chunk_size << GMBUS_BYTE_COUNT_SHIFT) |
(addr << GMBUS_SLAVE_ADDR_SHIFT) |
val |= *buf++ << (8 * loop);
} while (--len && ++loop < 4);
- I915_WRITE(GMBUS3 + reg_offset, val);
+ I915_WRITE(GMBUS3, val);
ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_RDY,
GMBUS_HW_RDY_EN);
static int
gmbus_xfer_index_read(struct drm_i915_private *dev_priv, struct i2c_msg *msgs)
{
- int reg_offset = dev_priv->gpio_mmio_base;
u32 gmbus1_index = 0;
u32 gmbus5 = 0;
int ret;
/* GMBUS5 holds 16-bit index */
if (gmbus5)
- I915_WRITE(GMBUS5 + reg_offset, gmbus5);
+ I915_WRITE(GMBUS5, gmbus5);
ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus1_index);
/* Clear GMBUS5 after each index transfer */
if (gmbus5)
- I915_WRITE(GMBUS5 + reg_offset, 0);
+ I915_WRITE(GMBUS5, 0);
return ret;
}
struct intel_gmbus,
adapter);
struct drm_i915_private *dev_priv = bus->dev_priv;
- int i = 0, inc, try = 0, reg_offset;
+ int i = 0, inc, try = 0;
int ret = 0;
intel_aux_display_runtime_get(dev_priv);
goto out;
}
- reg_offset = dev_priv->gpio_mmio_base;
-
retry:
- I915_WRITE(GMBUS0 + reg_offset, bus->reg0);
+ I915_WRITE(GMBUS0, bus->reg0);
for (; i < num; i += inc) {
inc = 1;
* a STOP on the very first cycle. To simplify the code we
* unconditionally generate the STOP condition with an additional gmbus
* cycle. */
- I915_WRITE(GMBUS1 + reg_offset, GMBUS_CYCLE_STOP | GMBUS_SW_RDY);
+ I915_WRITE(GMBUS1, GMBUS_CYCLE_STOP | GMBUS_SW_RDY);
/* Mark the GMBUS interface as disabled after waiting for idle.
* We will re-enable it at the start of the next xfer,
adapter->name);
ret = -ETIMEDOUT;
}
- I915_WRITE(GMBUS0 + reg_offset, 0);
+ I915_WRITE(GMBUS0, 0);
ret = ret ?: i;
goto out;
* of resetting the GMBUS controller and so clearing the
* BUS_ERROR raised by the slave's NAK.
*/
- I915_WRITE(GMBUS1 + reg_offset, GMBUS_SW_CLR_INT);
- I915_WRITE(GMBUS1 + reg_offset, 0);
- I915_WRITE(GMBUS0 + reg_offset, 0);
+ I915_WRITE(GMBUS1, GMBUS_SW_CLR_INT);
+ I915_WRITE(GMBUS1, 0);
+ I915_WRITE(GMBUS0, 0);
DRM_DEBUG_KMS("GMBUS [%s] NAK for addr: %04x %c(%d)\n",
adapter->name, msgs[i].addr,
timeout:
DRM_INFO("GMBUS [%s] timed out, falling back to bit banging on pin %d\n",
bus->adapter.name, bus->reg0 & 0xff);
- I915_WRITE(GMBUS0 + reg_offset, 0);
+ I915_WRITE(GMBUS0, 0);
/* Hardware may not support GMBUS over these pins? Try GPIO bitbanging instead. */
bus->force_bit = 1;
reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \
}
+#define ASSIGN_CTX_PML4(ppgtt, reg_state) { \
+ reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \
+ reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \
+}
+
enum {
ADVANCED_CONTEXT = 0,
- LEGACY_CONTEXT,
+ LEGACY_32B_CONTEXT,
ADVANCED_AD_CONTEXT,
LEGACY_64B_CONTEXT
};
-#define GEN8_CTX_MODE_SHIFT 3
+#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3
+#define GEN8_CTX_ADDRESSING_MODE(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
+ LEGACY_64B_CONTEXT :\
+ LEGACY_32B_CONTEXT)
enum {
FAULT_AND_HANG = 0,
FAULT_AND_HALT, /* Debug only */
#define CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17
static int intel_lr_context_pin(struct drm_i915_gem_request *rq);
+static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring,
+ struct drm_i915_gem_object *default_ctx_obj);
+
/**
* intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
{
WARN_ON(i915.enable_ppgtt == -1);
+ /* On platforms with execlist available, vGPU will only
+ * support execlist mode, no ring buffer mode.
+ */
+ if (HAS_LOGICAL_RING_CONTEXTS(dev) && intel_vgpu_active(dev))
+ return 1;
+
if (INTEL_INFO(dev)->gen >= 9)
return 1;
*/
u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj)
{
- u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj);
+ u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj) +
+ LRC_PPHWSP_PN * PAGE_SIZE;
/* LRCA is required to be 4K aligned so the more significant 20 bits
* are globally unique */
return lrca >> 12;
}
-static uint64_t execlists_ctx_descriptor(struct drm_i915_gem_request *rq)
+static bool disable_lite_restore_wa(struct intel_engine_cs *ring)
{
- struct intel_engine_cs *ring = rq->ring;
struct drm_device *dev = ring->dev;
- struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
+
+ return ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) ||
+ (IS_BROXTON(dev) && INTEL_REVID(dev) == BXT_REVID_A0)) &&
+ (ring->id == VCS || ring->id == VCS2);
+}
+
+uint64_t intel_lr_context_descriptor(struct intel_context *ctx,
+ struct intel_engine_cs *ring)
+{
+ struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
uint64_t desc;
- uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj);
+ uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj) +
+ LRC_PPHWSP_PN * PAGE_SIZE;
WARN_ON(lrca & 0xFFFFFFFF00000FFFULL);
desc = GEN8_CTX_VALID;
- desc |= LEGACY_CONTEXT << GEN8_CTX_MODE_SHIFT;
+ desc |= GEN8_CTX_ADDRESSING_MODE(dev) << GEN8_CTX_ADDRESSING_MODE_SHIFT;
if (IS_GEN8(ctx_obj->base.dev))
desc |= GEN8_CTX_L3LLC_COHERENT;
desc |= GEN8_CTX_PRIVILEGE;
/* desc |= GEN8_CTX_FORCE_RESTORE; */
/* WaEnableForceRestoreInCtxtDescForVCS:skl */
- if (IS_GEN9(dev) &&
- INTEL_REVID(dev) <= SKL_REVID_B0 &&
- (ring->id == BCS || ring->id == VCS ||
- ring->id == VECS || ring->id == VCS2))
+ /* WaEnableForceRestoreInCtxtDescForVCS:bxt */
+ if (disable_lite_restore_wa(ring))
desc |= GEN8_CTX_FORCE_RESTORE;
return desc;
uint64_t desc[2];
if (rq1) {
- desc[1] = execlists_ctx_descriptor(rq1);
+ desc[1] = intel_lr_context_descriptor(rq1->ctx, rq1->ring);
rq1->elsp_submitted++;
} else {
desc[1] = 0;
}
- desc[0] = execlists_ctx_descriptor(rq0);
+ desc[0] = intel_lr_context_descriptor(rq0->ctx, rq0->ring);
rq0->elsp_submitted++;
/* You must always write both descriptors in the order below. */
I915_WRITE_FW(RING_ELSP(ring), lower_32_bits(desc[0]));
/* ELSP is a wo register, use another nearby reg for posting */
- POSTING_READ_FW(RING_EXECLIST_STATUS(ring));
+ POSTING_READ_FW(RING_EXECLIST_STATUS_LO(ring));
intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
spin_unlock(&dev_priv->uncore.lock);
}
WARN_ON(!i915_gem_obj_is_pinned(ctx_obj));
WARN_ON(!i915_gem_obj_is_pinned(rb_obj));
- page = i915_gem_object_get_page(ctx_obj, 1);
+ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
reg_state = kmap_atomic(page);
reg_state[CTX_RING_TAIL+1] = rq->tail;
reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(rb_obj);
- /* True PPGTT with dynamic page allocation: update PDP registers and
- * point the unallocated PDPs to the scratch page
- */
- if (ppgtt) {
+ if (ppgtt && !USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
+ /* True 32b PPGTT with dynamic page allocation: update PDP
+ * registers and point the unallocated PDPs to scratch page.
+ * PML4 is allocated during ppgtt init, so this is not needed
+ * in 48-bit mode.
+ */
ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
u32 status_pointer;
u8 read_pointer;
u8 write_pointer;
- u32 status;
+ u32 status = 0;
u32 status_id;
u32 submit_contexts = 0;
status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring));
read_pointer = ring->next_context_status_buffer;
- write_pointer = status_pointer & 0x07;
+ write_pointer = status_pointer & GEN8_CSB_PTR_MASK;
if (read_pointer > write_pointer)
- write_pointer += 6;
+ write_pointer += GEN8_CSB_ENTRIES;
spin_lock(&ring->execlist_lock);
while (read_pointer < write_pointer) {
read_pointer++;
- status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
- (read_pointer % 6) * 8);
- status_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
- (read_pointer % 6) * 8 + 4);
+ status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, read_pointer % GEN8_CSB_ENTRIES));
+ status_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(ring, read_pointer % GEN8_CSB_ENTRIES));
if (status & GEN8_CTX_STATUS_IDLE_ACTIVE)
continue;
}
}
- if (submit_contexts != 0)
+ if (disable_lite_restore_wa(ring)) {
+ /* Prevent a ctx to preempt itself */
+ if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) &&
+ (submit_contexts != 0))
+ execlists_context_unqueue(ring);
+ } else if (submit_contexts != 0) {
execlists_context_unqueue(ring);
+ }
spin_unlock(&ring->execlist_lock);
WARN(submit_contexts > 2, "More than two context complete events?\n");
- ring->next_context_status_buffer = write_pointer % 6;
+ ring->next_context_status_buffer = write_pointer % GEN8_CSB_ENTRIES;
I915_WRITE(RING_CONTEXT_STATUS_PTR(ring),
- _MASKED_FIELD(0x07 << 8, ((u32)ring->next_context_status_buffer & 0x07) << 8));
+ _MASKED_FIELD(GEN8_CSB_PTR_MASK << 8,
+ ((u32)ring->next_context_status_buffer &
+ GEN8_CSB_PTR_MASK) << 8));
}
static int execlists_context_queue(struct drm_i915_gem_request *request)
i915_gem_request_reference(request);
- request->tail = request->ringbuf->tail;
-
spin_lock_irq(&ring->execlist_lock);
list_for_each_entry(cursor, &ring->execlist_queue, execlist_link)
intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
{
struct intel_engine_cs *ring = request->ring;
+ struct drm_i915_private *dev_priv = request->i915;
intel_logical_ring_advance(request->ringbuf);
+ request->tail = request->ringbuf->tail;
+
if (intel_ring_stopped(ring))
return;
- execlists_context_queue(request);
+ if (dev_priv->guc.execbuf_client)
+ i915_guc_submit(dev_priv->guc.execbuf_client, request);
+ else
+ execlists_context_queue(request);
}
static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
/**
* intel_logical_ring_begin() - prepare the logical ringbuffer to accept some commands
*
- * @request: The request to start some new work for
- * @ctx: Logical ring context whose ringbuffer is being prepared.
+ * @req: The request to start some new work for
* @num_dwords: number of DWORDs that we plan to write to the ringbuffer.
*
* The ringbuffer might not be ready to accept the commands right away (maybe it needs to
return -EINVAL;
}
- if (args->num_cliprects != 0) {
- DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
- return -EINVAL;
- } else {
- if (args->DR4 == 0xffffffff) {
- DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
- args->DR4 = 0;
- }
-
- if (args->DR1 || args->DR4 || args->cliprects_ptr) {
- DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
- return -EINVAL;
- }
- }
-
if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
DRM_DEBUG("sol reset is gen7 only\n");
return -EINVAL;
return 0;
}
+static int intel_lr_context_do_pin(struct intel_engine_cs *ring,
+ struct drm_i915_gem_object *ctx_obj,
+ struct intel_ringbuffer *ringbuf)
+{
+ struct drm_device *dev = ring->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int ret = 0;
+
+ WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
+ ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN,
+ PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+ if (ret)
+ return ret;
+
+ ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);
+ if (ret)
+ goto unpin_ctx_obj;
+
+ ctx_obj->dirty = true;
+
+ /* Invalidate GuC TLB. */
+ if (i915.enable_guc_submission)
+ I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+
+ return ret;
+
+unpin_ctx_obj:
+ i915_gem_object_ggtt_unpin(ctx_obj);
+
+ return ret;
+}
+
static int intel_lr_context_pin(struct drm_i915_gem_request *rq)
{
+ int ret = 0;
struct intel_engine_cs *ring = rq->ring;
struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
struct intel_ringbuffer *ringbuf = rq->ringbuf;
- int ret = 0;
- WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
if (rq->ctx->engine[ring->id].pin_count++ == 0) {
- ret = i915_gem_obj_ggtt_pin(ctx_obj,
- GEN8_LR_CONTEXT_ALIGN, 0);
+ ret = intel_lr_context_do_pin(ring, ctx_obj, ringbuf);
if (ret)
goto reset_pin_count;
-
- ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);
- if (ret)
- goto unpin_ctx_obj;
-
- ctx_obj->dirty = true;
}
-
return ret;
-unpin_ctx_obj:
- i915_gem_object_ggtt_unpin(ctx_obj);
reset_pin_count:
rq->ctx->engine[ring->id].pin_count = 0;
-
return ret;
}
if (IS_SKYLAKE(ring->dev) && INTEL_REVID(ring->dev) <= SKL_REVID_E0)
l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS;
- wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8(1) |
+ wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 |
MI_SRM_LRM_GLOBAL_GTT));
wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256);
wa_ctx_emit(batch, index, 0);
wa_ctx_emit(batch, index, 0);
- wa_ctx_emit(batch, index, (MI_LOAD_REGISTER_MEM_GEN8(1) |
+ wa_ctx_emit(batch, index, (MI_LOAD_REGISTER_MEM_GEN8 |
MI_SRM_LRM_GLOBAL_GTT));
wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256);
/* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
if (IS_BROADWELL(ring->dev)) {
- index = gen8_emit_flush_coherentl3_wa(ring, batch, index);
- if (index < 0)
- return index;
+ int rc = gen8_emit_flush_coherentl3_wa(ring, batch, index);
+ if (rc < 0)
+ return rc;
+ index = rc;
}
/* WaClearSlmSpaceAtContextSwitch:bdw,chv */
{
struct drm_device *dev = ring->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
+ u8 next_context_status_buffer_hw;
+
+ lrc_setup_hardware_status_page(ring,
+ ring->default_context->engine[ring->id].state);
I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
_MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
_MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
POSTING_READ(RING_MODE_GEN7(ring));
- ring->next_context_status_buffer = 0;
+
+ /*
+ * Instead of resetting the Context Status Buffer (CSB) read pointer to
+ * zero, we need to read the write pointer from hardware and use its
+ * value because "this register is power context save restored".
+ * Effectively, these states have been observed:
+ *
+ * | Suspend-to-idle (freeze) | Suspend-to-RAM (mem) |
+ * BDW | CSB regs not reset | CSB regs reset |
+ * CHT | CSB regs not reset | CSB regs not reset |
+ */
+ next_context_status_buffer_hw = (I915_READ(RING_CONTEXT_STATUS_PTR(ring))
+ & GEN8_CSB_PTR_MASK);
+
+ /*
+ * When the CSB registers are reset (also after power-up / gpu reset),
+ * CSB write pointer is set to all 1's, which is not valid, use '5' in
+ * this special case, so the first element read is CSB[0].
+ */
+ if (next_context_status_buffer_hw == GEN8_CSB_PTR_MASK)
+ next_context_status_buffer_hw = (GEN8_CSB_ENTRIES - 1);
+
+ ring->next_context_status_buffer = next_context_status_buffer_hw;
DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
* Ideally, we should set Force PD Restore in ctx descriptor,
* but we can't. Force Restore would be a second option, but
* it is unsafe in case of lite-restore (because the ctx is
- * not idle). */
+ * not idle). PML4 is allocated during ppgtt init so this is
+ * not needed in 48-bit.*/
if (req->ctx->ppgtt &&
(intel_ring_flag(req->ring) & req->ctx->ppgtt->pd_dirty_rings)) {
- ret = intel_logical_ring_emit_pdps(req);
- if (ret)
- return ret;
+ if (!USES_FULL_48BIT_PPGTT(req->i915) &&
+ !intel_vgpu_active(req->i915->dev)) {
+ ret = intel_logical_ring_emit_pdps(req);
+ if (ret)
+ return ret;
+ }
req->ctx->ppgtt->pd_dirty_rings &= ~intel_ring_flag(req->ring);
}
intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
}
+static u32 bxt_a_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
+{
+
+ /*
+ * On BXT A steppings there is a HW coherency issue whereby the
+ * MI_STORE_DATA_IMM storing the completed request's seqno
+ * occasionally doesn't invalidate the CPU cache. Work around this by
+ * clflushing the corresponding cacheline whenever the caller wants
+ * the coherency to be guaranteed. Note that this cacheline is known
+ * to be clean at this point, since we only write it in
+ * bxt_a_set_seqno(), where we also do a clflush after the write. So
+ * this clflush in practice becomes an invalidate operation.
+ */
+
+ if (!lazy_coherency)
+ intel_flush_status_page(ring, I915_GEM_HWS_INDEX);
+
+ return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+}
+
+static void bxt_a_set_seqno(struct intel_engine_cs *ring, u32 seqno)
+{
+ intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
+
+ /* See bxt_a_get_seqno() explaining the reason for the clflush. */
+ intel_flush_status_page(ring, I915_GEM_HWS_INDEX);
+}
+
static int gen8_emit_request(struct drm_i915_gem_request *request)
{
struct intel_ringbuffer *ringbuf = request->ringbuf;
if (ret)
return ret;
- ret = intel_lr_context_deferred_create(ring->default_context, ring);
+ ret = intel_lr_context_deferred_alloc(ring->default_context, ring);
+ if (ret)
+ return ret;
+
+ /* As this is the default context, always pin it */
+ ret = intel_lr_context_do_pin(
+ ring,
+ ring->default_context->engine[ring->id].state,
+ ring->default_context->engine[ring->id].ringbuf);
+ if (ret) {
+ DRM_ERROR(
+ "Failed to pin and map ringbuffer %s: %d\n",
+ ring->name, ret);
+ return ret;
+ }
return ret;
}
ring->init_hw = gen8_init_render_ring;
ring->init_context = gen8_init_rcs_context;
ring->cleanup = intel_fini_pipe_control;
- ring->get_seqno = gen8_get_seqno;
- ring->set_seqno = gen8_set_seqno;
+ if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+ ring->get_seqno = bxt_a_get_seqno;
+ ring->set_seqno = bxt_a_set_seqno;
+ } else {
+ ring->get_seqno = gen8_get_seqno;
+ ring->set_seqno = gen8_set_seqno;
+ }
ring->emit_request = gen8_emit_request;
ring->emit_flush = gen8_emit_flush_render;
ring->irq_get = gen8_logical_ring_get_irq;
GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
ring->init_hw = gen8_init_common_ring;
- ring->get_seqno = gen8_get_seqno;
- ring->set_seqno = gen8_set_seqno;
+ if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+ ring->get_seqno = bxt_a_get_seqno;
+ ring->set_seqno = bxt_a_set_seqno;
+ } else {
+ ring->get_seqno = gen8_get_seqno;
+ ring->set_seqno = gen8_set_seqno;
+ }
ring->emit_request = gen8_emit_request;
ring->emit_flush = gen8_emit_flush;
ring->irq_get = gen8_logical_ring_get_irq;
GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
ring->init_hw = gen8_init_common_ring;
- ring->get_seqno = gen8_get_seqno;
- ring->set_seqno = gen8_set_seqno;
+ if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+ ring->get_seqno = bxt_a_get_seqno;
+ ring->set_seqno = bxt_a_set_seqno;
+ } else {
+ ring->get_seqno = gen8_get_seqno;
+ ring->set_seqno = gen8_set_seqno;
+ }
ring->emit_request = gen8_emit_request;
ring->emit_flush = gen8_emit_flush;
ring->irq_get = gen8_logical_ring_get_irq;
GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
ring->init_hw = gen8_init_common_ring;
- ring->get_seqno = gen8_get_seqno;
- ring->set_seqno = gen8_set_seqno;
+ if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+ ring->get_seqno = bxt_a_get_seqno;
+ ring->set_seqno = bxt_a_set_seqno;
+ } else {
+ ring->get_seqno = gen8_get_seqno;
+ ring->set_seqno = gen8_set_seqno;
+ }
ring->emit_request = gen8_emit_request;
ring->emit_flush = gen8_emit_flush;
ring->irq_get = gen8_logical_ring_get_irq;
goto cleanup_vebox_ring;
}
- ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
- if (ret)
- goto cleanup_bsd2_ring;
-
return 0;
-cleanup_bsd2_ring:
- intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
cleanup_vebox_ring:
intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
cleanup_blt_ring:
/* The second page of the context object contains some fields which must
* be set up prior to the first execution. */
- page = i915_gem_object_get_page(ctx_obj, 1);
+ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
reg_state = kmap_atomic(page);
/* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
- /* With dynamic page allocation, PDPs may not be allocated at this point,
- * Point the unallocated PDPs to the scratch page
- */
- ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
- ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
- ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
- ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
+ if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
+ /* 64b PPGTT (48bit canonical)
+ * PDP0_DESCRIPTOR contains the base address to PML4 and
+ * other PDP Descriptors are ignored.
+ */
+ ASSIGN_CTX_PML4(ppgtt, reg_state);
+ } else {
+ /* 32b PPGTT
+ * PDP*_DESCRIPTOR contains the base address of space supported.
+ * With dynamic page allocation, PDPs may not be allocated at
+ * this point. Point the unallocated PDPs to the scratch page
+ */
+ ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
+ ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
+ ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
+ ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
+ }
+
if (ring->id == RCS) {
reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
reg_state[CTX_R_PWR_CLK_STATE] = GEN8_R_PWR_CLK_STATE;
i915_gem_object_ggtt_unpin(ctx_obj);
}
WARN_ON(ctx->engine[ring->id].pin_count);
- intel_destroy_ringbuffer_obj(ringbuf);
- kfree(ringbuf);
+ intel_ringbuffer_free(ringbuf);
drm_gem_object_unreference(&ctx_obj->base);
}
}
struct drm_i915_gem_object *default_ctx_obj)
{
struct drm_i915_private *dev_priv = ring->dev->dev_private;
+ struct page *page;
- /* The status page is offset 0 from the default context object
- * in LRC mode. */
- ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj);
- ring->status_page.page_addr =
- kmap(sg_page(default_ctx_obj->pages->sgl));
+ /* The HWSP is part of the default context object in LRC mode. */
+ ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj)
+ + LRC_PPHWSP_PN * PAGE_SIZE;
+ page = i915_gem_object_get_page(default_ctx_obj, LRC_PPHWSP_PN);
+ ring->status_page.page_addr = kmap(page);
ring->status_page.obj = default_ctx_obj;
I915_WRITE(RING_HWS_PGA(ring->mmio_base),
}
/**
- * intel_lr_context_deferred_create() - create the LRC specific bits of a context
+ * intel_lr_context_deferred_alloc() - create the LRC specific bits of a context
* @ctx: LR context to create.
* @ring: engine to be used with the context.
*
*
* Return: non-zero on error.
*/
-int intel_lr_context_deferred_create(struct intel_context *ctx,
+
+int intel_lr_context_deferred_alloc(struct intel_context *ctx,
struct intel_engine_cs *ring)
{
- const bool is_global_default_ctx = (ctx == ring->default_context);
struct drm_device *dev = ring->dev;
struct drm_i915_gem_object *ctx_obj;
uint32_t context_size;
context_size = round_up(get_lr_context_size(ring), 4096);
+ /* One extra page as the sharing data between driver and GuC */
+ context_size += PAGE_SIZE * LRC_PPHWSP_PN;
+
ctx_obj = i915_gem_alloc_object(dev, context_size);
if (!ctx_obj) {
DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
return -ENOMEM;
}
- if (is_global_default_ctx) {
- ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0);
- if (ret) {
- DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n",
- ret);
- drm_gem_object_unreference(&ctx_obj->base);
- return ret;
- }
- }
-
- ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
- if (!ringbuf) {
- DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
- ring->name);
- ret = -ENOMEM;
- goto error_unpin_ctx;
- }
-
- ringbuf->ring = ring;
-
- ringbuf->size = 32 * PAGE_SIZE;
- ringbuf->effective_size = ringbuf->size;
- ringbuf->head = 0;
- ringbuf->tail = 0;
- ringbuf->last_retired_head = -1;
- intel_ring_update_space(ringbuf);
-
- if (ringbuf->obj == NULL) {
- ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
- if (ret) {
- DRM_DEBUG_DRIVER(
- "Failed to allocate ringbuffer obj %s: %d\n",
- ring->name, ret);
- goto error_free_rbuf;
- }
-
- if (is_global_default_ctx) {
- ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
- if (ret) {
- DRM_ERROR(
- "Failed to pin and map ringbuffer %s: %d\n",
- ring->name, ret);
- goto error_destroy_rbuf;
- }
- }
-
+ ringbuf = intel_engine_create_ringbuffer(ring, 4 * PAGE_SIZE);
+ if (IS_ERR(ringbuf)) {
+ ret = PTR_ERR(ringbuf);
+ goto error_deref_obj;
}
ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
if (ret) {
DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
- goto error;
+ goto error_ringbuf;
}
ctx->engine[ring->id].ringbuf = ringbuf;
ctx->engine[ring->id].state = ctx_obj;
- if (ctx == ring->default_context)
- lrc_setup_hardware_status_page(ring, ctx_obj);
- else if (ring->id == RCS && !ctx->rcs_initialized) {
- if (ring->init_context) {
- struct drm_i915_gem_request *req;
+ if (ctx != ring->default_context && ring->init_context) {
+ struct drm_i915_gem_request *req;
- ret = i915_gem_request_alloc(ring, ctx, &req);
- if (ret)
- return ret;
-
- ret = ring->init_context(req);
- if (ret) {
- DRM_ERROR("ring init context: %d\n", ret);
- i915_gem_request_cancel(req);
- ctx->engine[ring->id].ringbuf = NULL;
- ctx->engine[ring->id].state = NULL;
- goto error;
- }
-
- i915_add_request_no_flush(req);
+ ret = i915_gem_request_alloc(ring,
+ ctx, &req);
+ if (ret) {
+ DRM_ERROR("ring create req: %d\n",
+ ret);
+ goto error_ringbuf;
}
- ctx->rcs_initialized = true;
+ ret = ring->init_context(req);
+ if (ret) {
+ DRM_ERROR("ring init context: %d\n",
+ ret);
+ i915_gem_request_cancel(req);
+ goto error_ringbuf;
+ }
+ i915_add_request_no_flush(req);
}
-
return 0;
-error:
- if (is_global_default_ctx)
- intel_unpin_ringbuffer_obj(ringbuf);
-error_destroy_rbuf:
- intel_destroy_ringbuffer_obj(ringbuf);
-error_free_rbuf:
- kfree(ringbuf);
-error_unpin_ctx:
- if (is_global_default_ctx)
- i915_gem_object_ggtt_unpin(ctx_obj);
+error_ringbuf:
+ intel_ringbuffer_free(ringbuf);
+error_deref_obj:
drm_gem_object_unreference(&ctx_obj->base);
+ ctx->engine[ring->id].ringbuf = NULL;
+ ctx->engine[ring->id].state = NULL;
return ret;
}
WARN(1, "Failed get_pages for context obj\n");
continue;
}
- page = i915_gem_object_get_page(ctx_obj, 1);
+ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
reg_state = kmap_atomic(page);
reg_state[CTX_RING_HEAD+1] = 0;
#define _INTEL_LRC_H_
#define GEN8_LR_CONTEXT_ALIGN 4096
+#define GEN8_CSB_ENTRIES 6
+#define GEN8_CSB_PTR_MASK 0x07
/* Execlists regs */
#define RING_ELSP(ring) ((ring)->mmio_base+0x230)
-#define RING_EXECLIST_STATUS(ring) ((ring)->mmio_base+0x234)
+#define RING_EXECLIST_STATUS_LO(ring) ((ring)->mmio_base+0x234)
+#define RING_EXECLIST_STATUS_HI(ring) ((ring)->mmio_base+0x234 + 4)
#define RING_CONTEXT_CONTROL(ring) ((ring)->mmio_base+0x244)
#define CTX_CTRL_INHIBIT_SYN_CTX_SWITCH (1 << 3)
#define CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT (1 << 0)
#define CTX_CTRL_RS_CTX_ENABLE (1 << 1)
-#define RING_CONTEXT_STATUS_BUF(ring) ((ring)->mmio_base+0x370)
+#define RING_CONTEXT_STATUS_BUF_LO(ring, i) ((ring)->mmio_base+0x370 + (i) * 8)
+#define RING_CONTEXT_STATUS_BUF_HI(ring, i) ((ring)->mmio_base+0x370 + (i) * 8 + 4)
#define RING_CONTEXT_STATUS_PTR(ring) ((ring)->mmio_base+0x3a0)
/* Logical Rings */
}
/* Logical Ring Contexts */
+
+/* One extra page is added before LRC for GuC as shared data */
+#define LRC_GUCSHR_PN (0)
+#define LRC_PPHWSP_PN (LRC_GUCSHR_PN + 1)
+#define LRC_STATE_PN (LRC_PPHWSP_PN + 1)
+
void intel_lr_context_free(struct intel_context *ctx);
-int intel_lr_context_deferred_create(struct intel_context *ctx,
- struct intel_engine_cs *ring);
+int intel_lr_context_deferred_alloc(struct intel_context *ctx,
+ struct intel_engine_cs *ring);
void intel_lr_context_unpin(struct drm_i915_gem_request *req);
void intel_lr_context_reset(struct drm_device *dev,
struct intel_context *ctx);
+uint64_t intel_lr_context_descriptor(struct intel_context *ctx,
+ struct intel_engine_cs *ring);
/* Execlists */
int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists);
{
struct drm_device *dev = encoder->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- u32 lvds_reg, tmp, flags = 0;
+ struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
+ u32 tmp, flags = 0;
int dotclock;
- if (HAS_PCH_SPLIT(dev))
- lvds_reg = PCH_LVDS;
- else
- lvds_reg = LVDS;
-
- tmp = I915_READ(lvds_reg);
+ tmp = I915_READ(lvds_encoder->reg);
if (tmp & LVDS_HSYNC_POLARITY)
flags |= DRM_MODE_FLAG_NHSYNC;
else
struct drm_device *dev = encoder->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
- const struct drm_display_mode *adjusted_mode =
- &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
int pipe = crtc->pipe;
u32 temp;
{
struct intel_connector *intel_connector = to_intel_connector(connector);
struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode;
+ int max_pixclk = to_i915(connector->dev)->max_dotclk_freq;
if (mode->hdisplay > fixed_mode->hdisplay)
return MODE_PANEL;
if (mode->vdisplay > fixed_mode->vdisplay)
return MODE_PANEL;
+ if (fixed_mode->clock > max_pixclk)
+ return MODE_CLOCK_HIGH;
return MODE_OK;
}
struct drm_display_mode *downclock_mode = NULL;
struct edid *edid;
struct drm_crtc *crtc;
+ u32 lvds_reg;
u32 lvds;
int pipe;
u8 pin;
if (HAS_PCH_SPLIT(dev)) {
I915_WRITE(PCH_PP_CONTROL,
I915_READ(PCH_PP_CONTROL) | PANEL_UNLOCK_REGS);
- } else {
+ } else if (INTEL_INFO(dev_priv)->gen < 5) {
I915_WRITE(PP_CONTROL,
I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
}
if (dmi_check_system(intel_no_lvds))
return;
+ if (HAS_PCH_SPLIT(dev))
+ lvds_reg = PCH_LVDS;
+ else
+ lvds_reg = LVDS;
+
+ lvds = I915_READ(lvds_reg);
+
if (HAS_PCH_SPLIT(dev)) {
- if ((I915_READ(PCH_LVDS) & LVDS_DETECTED) == 0)
+ if ((lvds & LVDS_DETECTED) == 0)
return;
if (dev_priv->vbt.edp_support) {
DRM_DEBUG_KMS("disable LVDS for eDP support\n");
pin = GMBUS_PIN_PANEL;
if (!lvds_is_present_in_vbt(dev, &pin)) {
- u32 reg = HAS_PCH_SPLIT(dev) ? PCH_LVDS : LVDS;
- if ((I915_READ(reg) & LVDS_PORT_EN) == 0) {
+ if ((lvds & LVDS_PORT_EN) == 0) {
DRM_DEBUG_KMS("LVDS is not present in VBT\n");
return;
}
DRM_DEBUG_KMS("LVDS is not present in VBT, but enabled anyway\n");
}
+ /* Set the Panel Power On/Off timings if uninitialized. */
+ if (INTEL_INFO(dev_priv)->gen < 5 &&
+ I915_READ(PP_ON_DELAYS) == 0 && I915_READ(PP_OFF_DELAYS) == 0) {
+ /* Set T2 to 40ms and T5 to 200ms */
+ I915_WRITE(PP_ON_DELAYS, 0x019007d0);
+
+ /* Set T3 to 35ms and Tx to 200ms */
+ I915_WRITE(PP_OFF_DELAYS, 0x015e07d0);
+
+ DRM_DEBUG_KMS("Panel power timings uninitialized, setting defaults\n");
+ }
+
lvds_encoder = kzalloc(sizeof(*lvds_encoder), GFP_KERNEL);
if (!lvds_encoder)
return;
connector->interlace_allowed = false;
connector->doublescan_allowed = false;
- if (HAS_PCH_SPLIT(dev)) {
- lvds_encoder->reg = PCH_LVDS;
- } else {
- lvds_encoder->reg = LVDS;
- }
+ lvds_encoder->reg = lvds_reg;
/* create the scaling mode property */
drm_mode_create_scaling_mode_property(dev);
if (HAS_PCH_SPLIT(dev))
goto failed;
- lvds = I915_READ(LVDS);
pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0;
crtc = intel_get_crtc_for_pipe(dev, pipe);
drm_object_attach_property(&connector->base, prop, 0);
}
+
+void
+intel_attach_aspect_ratio_property(struct drm_connector *connector)
+{
+ if (!drm_mode_create_aspect_ratio_property(connector->dev))
+ drm_object_attach_property(&connector->base,
+ connector->dev->mode_config.aspect_ratio_property,
+ DRM_MODE_PICTURE_ASPECT_NONE);
+}
static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct opregion_swsci __iomem *swsci = dev_priv->opregion.swsci;
+ struct opregion_swsci *swsci = dev_priv->opregion.swsci;
u32 main_function, sub_function, scic;
u16 pci_swsci;
u32 dslp;
}
/* Driver sleep timeout in ms. */
- dslp = ioread32(&swsci->dslp);
+ dslp = swsci->dslp;
if (!dslp) {
/* The spec says 2ms should be the default, but it's too small
* for some machines. */
}
/* The spec tells us to do this, but we are the only user... */
- scic = ioread32(&swsci->scic);
+ scic = swsci->scic;
if (scic & SWSCI_SCIC_INDICATOR) {
DRM_DEBUG_DRIVER("SWSCI request already in progress\n");
return -EBUSY;
scic = function | SWSCI_SCIC_INDICATOR;
- iowrite32(parm, &swsci->parm);
- iowrite32(scic, &swsci->scic);
+ swsci->parm = parm;
+ swsci->scic = scic;
/* Ensure SCI event is selected and event trigger is cleared. */
pci_read_config_word(dev->pdev, PCI_SWSCI, &pci_swsci);
pci_write_config_word(dev->pdev, PCI_SWSCI, pci_swsci);
/* Poll for the result. */
-#define C (((scic = ioread32(&swsci->scic)) & SWSCI_SCIC_INDICATOR) == 0)
+#define C (((scic = swsci->scic) & SWSCI_SCIC_INDICATOR) == 0)
if (wait_for(C, dslp)) {
DRM_DEBUG_DRIVER("SWSCI request timed out\n");
return -ETIMEDOUT;
}
if (parm_out)
- *parm_out = ioread32(&swsci->parm);
+ *parm_out = swsci->parm;
return 0;
if (!HAS_DDI(dev))
return 0;
- port = intel_ddi_get_encoder_port(intel_encoder);
- if (port == PORT_E) {
+ if (intel_encoder->type == INTEL_OUTPUT_DSI)
+ port = 0;
+ else
+ port = intel_ddi_get_encoder_port(intel_encoder);
+
+ if (port == PORT_E) {
port = 0;
} else {
parm |= 1 << port;
type = DISPLAY_TYPE_EXTERNAL_FLAT_PANEL;
break;
case INTEL_OUTPUT_EDP:
+ case INTEL_OUTPUT_DSI:
type = DISPLAY_TYPE_INTERNAL_FLAT_PANEL;
break;
default:
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_connector *intel_connector;
- struct opregion_asle __iomem *asle = dev_priv->opregion.asle;
+ struct opregion_asle *asle = dev_priv->opregion.asle;
DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp);
DRM_DEBUG_KMS("updating opregion backlight %d/255\n", bclp);
list_for_each_entry(intel_connector, &dev->mode_config.connector_list, base.head)
intel_panel_set_backlight_acpi(intel_connector, bclp, 255);
- iowrite32(DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID, &asle->cblv);
+ asle->cblv = DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID;
drm_modeset_unlock(&dev->mode_config.connection_mutex);
struct drm_i915_private *dev_priv =
container_of(opregion, struct drm_i915_private, opregion);
struct drm_device *dev = dev_priv->dev;
- struct opregion_asle __iomem *asle = dev_priv->opregion.asle;
+ struct opregion_asle *asle = dev_priv->opregion.asle;
u32 aslc_stat = 0;
u32 aslc_req;
if (!asle)
return;
- aslc_req = ioread32(&asle->aslc);
+ aslc_req = asle->aslc;
if (!(aslc_req & ASLC_REQ_MSK)) {
DRM_DEBUG_DRIVER("No request on ASLC interrupt 0x%08x\n",
}
if (aslc_req & ASLC_SET_ALS_ILLUM)
- aslc_stat |= asle_set_als_illum(dev, ioread32(&asle->alsi));
+ aslc_stat |= asle_set_als_illum(dev, asle->alsi);
if (aslc_req & ASLC_SET_BACKLIGHT)
- aslc_stat |= asle_set_backlight(dev, ioread32(&asle->bclp));
+ aslc_stat |= asle_set_backlight(dev, asle->bclp);
if (aslc_req & ASLC_SET_PFIT)
- aslc_stat |= asle_set_pfit(dev, ioread32(&asle->pfit));
+ aslc_stat |= asle_set_pfit(dev, asle->pfit);
if (aslc_req & ASLC_SET_PWM_FREQ)
- aslc_stat |= asle_set_pwm_freq(dev, ioread32(&asle->pfmb));
+ aslc_stat |= asle_set_pwm_freq(dev, asle->pfmb);
if (aslc_req & ASLC_SUPPORTED_ROTATION_ANGLES)
aslc_stat |= asle_set_supported_rotation_angles(dev,
- ioread32(&asle->srot));
+ asle->srot);
if (aslc_req & ASLC_BUTTON_ARRAY)
- aslc_stat |= asle_set_button_array(dev, ioread32(&asle->iuer));
+ aslc_stat |= asle_set_button_array(dev, asle->iuer);
if (aslc_req & ASLC_CONVERTIBLE_INDICATOR)
- aslc_stat |= asle_set_convertible(dev, ioread32(&asle->iuer));
+ aslc_stat |= asle_set_convertible(dev, asle->iuer);
if (aslc_req & ASLC_DOCKING_INDICATOR)
- aslc_stat |= asle_set_docking(dev, ioread32(&asle->iuer));
+ aslc_stat |= asle_set_docking(dev, asle->iuer);
if (aslc_req & ASLC_ISCT_STATE_CHANGE)
aslc_stat |= asle_isct_state(dev);
- iowrite32(aslc_stat, &asle->aslc);
+ asle->aslc = aslc_stat;
}
void intel_opregion_asle_intr(struct drm_device *dev)
Linux, these are handled by the dock, button and video drivers.
*/
- struct opregion_acpi __iomem *acpi;
struct acpi_bus_event *event = data;
+ struct opregion_acpi *acpi;
int ret = NOTIFY_OK;
if (strcmp(event->device_class, ACPI_VIDEO_CLASS) != 0)
acpi = system_opregion->acpi;
- if (event->type == 0x80 &&
- (ioread32(&acpi->cevt) & 1) == 0)
+ if (event->type == 0x80 && ((acpi->cevt & 1) == 0))
ret = NOTIFY_BAD;
- iowrite32(0, &acpi->csts);
+ acpi->csts = 0;
return ret;
}
u32 did;
if (i < ARRAY_SIZE(opregion->acpi->didl)) {
- did = ioread32(&opregion->acpi->didl[i]);
+ did = opregion->acpi->didl[i];
} else {
i -= ARRAY_SIZE(opregion->acpi->didl);
if (WARN_ON(i >= ARRAY_SIZE(opregion->acpi->did2)))
return 0;
- did = ioread32(&opregion->acpi->did2[i]);
+ did = opregion->acpi->did2[i];
}
return did;
static void set_did(struct intel_opregion *opregion, int i, u32 val)
{
if (i < ARRAY_SIZE(opregion->acpi->didl)) {
- iowrite32(val, &opregion->acpi->didl[i]);
+ opregion->acpi->didl[i] = val;
} else {
i -= ARRAY_SIZE(opregion->acpi->didl);
if (WARN_ON(i >= ARRAY_SIZE(opregion->acpi->did2)))
return;
- iowrite32(val, &opregion->acpi->did2[i]);
+ opregion->acpi->did2[i] = val;
}
}
* there are less than eight devices. */
do {
disp_id = get_did(opregion, i);
- iowrite32(disp_id, &opregion->acpi->cadl[i]);
+ opregion->acpi->cadl[i] = disp_id;
} while (++i < 8 && disp_id != 0);
}
/* Notify BIOS we are ready to handle ACPI video ext notifs.
* Right now, all the events are handled by the ACPI video module.
* We don't actually need to do anything with them. */
- iowrite32(0, &opregion->acpi->csts);
- iowrite32(1, &opregion->acpi->drdy);
+ opregion->acpi->csts = 0;
+ opregion->acpi->drdy = 1;
system_opregion = opregion;
register_acpi_notifier(&intel_opregion_notifier);
}
if (opregion->asle) {
- iowrite32(ASLE_TCHE_BLC_EN, &opregion->asle->tche);
- iowrite32(ASLE_ARDY_READY, &opregion->asle->ardy);
+ opregion->asle->tche = ASLE_TCHE_BLC_EN;
+ opregion->asle->ardy = ASLE_ARDY_READY;
}
}
return;
if (opregion->asle)
- iowrite32(ASLE_ARDY_NOT_READY, &opregion->asle->ardy);
+ opregion->asle->ardy = ASLE_ARDY_NOT_READY;
cancel_work_sync(&dev_priv->opregion.asle_work);
if (opregion->acpi) {
- iowrite32(0, &opregion->acpi->drdy);
+ opregion->acpi->drdy = 0;
system_opregion = NULL;
unregister_acpi_notifier(&intel_opregion_notifier);
}
/* just clear all opregion memory pointers now */
- iounmap(opregion->header);
+ memunmap(opregion->header);
opregion->header = NULL;
opregion->acpi = NULL;
opregion->swsci = NULL;
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_opregion *opregion = &dev_priv->opregion;
- void __iomem *base;
u32 asls, mboxes;
char buf[sizeof(OPREGION_SIGNATURE)];
int err = 0;
+ void *base;
BUILD_BUG_ON(sizeof(struct opregion_header) != 0x100);
BUILD_BUG_ON(sizeof(struct opregion_acpi) != 0x100);
INIT_WORK(&opregion->asle_work, asle_work);
#endif
- base = acpi_os_ioremap(asls, OPREGION_SIZE);
+ base = memremap(asls, OPREGION_SIZE, MEMREMAP_WB);
if (!base)
return -ENOMEM;
- memcpy_fromio(buf, base, sizeof(buf));
+ memcpy(buf, base, sizeof(buf));
if (memcmp(buf, OPREGION_SIGNATURE, 16)) {
DRM_DEBUG_DRIVER("opregion signature mismatch\n");
opregion->lid_state = base + ACPI_CLID;
- mboxes = ioread32(&opregion->header->mboxes);
+ mboxes = opregion->header->mboxes;
if (mboxes & MBOX_ACPI) {
DRM_DEBUG_DRIVER("Public ACPI methods supported\n");
opregion->acpi = base + OPREGION_ACPI_OFFSET;
DRM_DEBUG_DRIVER("ASLE supported\n");
opregion->asle = base + OPREGION_ASLE_OFFSET;
- iowrite32(ASLE_ARDY_NOT_READY, &opregion->asle->ardy);
+ opregion->asle->ardy = ASLE_ARDY_NOT_READY;
}
return 0;
err_out:
- iounmap(base);
+ memunmap(base);
return err;
}
struct intel_crtc_state *pipe_config,
int fitting_mode)
{
- struct drm_display_mode *adjusted_mode;
- int x, y, width, height;
-
- adjusted_mode = &pipe_config->base.adjusted_mode;
-
- x = y = width = height = 0;
+ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+ int x = 0, y = 0, width = 0, height = 0;
/* Native modes don't need fitting */
- if (adjusted_mode->hdisplay == pipe_config->pipe_src_w &&
- adjusted_mode->vdisplay == pipe_config->pipe_src_h)
+ if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w &&
+ adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h)
goto done;
switch (fitting_mode) {
case DRM_MODE_SCALE_CENTER:
width = pipe_config->pipe_src_w;
height = pipe_config->pipe_src_h;
- x = (adjusted_mode->hdisplay - width + 1)/2;
- y = (adjusted_mode->vdisplay - height + 1)/2;
+ x = (adjusted_mode->crtc_hdisplay - width + 1)/2;
+ y = (adjusted_mode->crtc_vdisplay - height + 1)/2;
break;
case DRM_MODE_SCALE_ASPECT:
/* Scale but preserve the aspect ratio */
{
- u32 scaled_width = adjusted_mode->hdisplay
+ u32 scaled_width = adjusted_mode->crtc_hdisplay
* pipe_config->pipe_src_h;
u32 scaled_height = pipe_config->pipe_src_w
- * adjusted_mode->vdisplay;
+ * adjusted_mode->crtc_vdisplay;
if (scaled_width > scaled_height) { /* pillar */
width = scaled_height / pipe_config->pipe_src_h;
if (width & 1)
width++;
- x = (adjusted_mode->hdisplay - width + 1) / 2;
+ x = (adjusted_mode->crtc_hdisplay - width + 1) / 2;
y = 0;
- height = adjusted_mode->vdisplay;
+ height = adjusted_mode->crtc_vdisplay;
} else if (scaled_width < scaled_height) { /* letter */
height = scaled_width / pipe_config->pipe_src_w;
if (height & 1)
height++;
- y = (adjusted_mode->vdisplay - height + 1) / 2;
+ y = (adjusted_mode->crtc_vdisplay - height + 1) / 2;
x = 0;
- width = adjusted_mode->hdisplay;
+ width = adjusted_mode->crtc_hdisplay;
} else {
x = y = 0;
- width = adjusted_mode->hdisplay;
- height = adjusted_mode->vdisplay;
+ width = adjusted_mode->crtc_hdisplay;
+ height = adjusted_mode->crtc_vdisplay;
}
}
break;
case DRM_MODE_SCALE_FULLSCREEN:
x = y = 0;
- width = adjusted_mode->hdisplay;
- height = adjusted_mode->vdisplay;
+ width = adjusted_mode->crtc_hdisplay;
+ height = adjusted_mode->crtc_vdisplay;
break;
default:
}
static void
-centre_horizontally(struct drm_display_mode *mode,
+centre_horizontally(struct drm_display_mode *adjusted_mode,
int width)
{
u32 border, sync_pos, blank_width, sync_width;
/* keep the hsync and hblank widths constant */
- sync_width = mode->crtc_hsync_end - mode->crtc_hsync_start;
- blank_width = mode->crtc_hblank_end - mode->crtc_hblank_start;
+ sync_width = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start;
+ blank_width = adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start;
sync_pos = (blank_width - sync_width + 1) / 2;
- border = (mode->hdisplay - width + 1) / 2;
+ border = (adjusted_mode->crtc_hdisplay - width + 1) / 2;
border += border & 1; /* make the border even */
- mode->crtc_hdisplay = width;
- mode->crtc_hblank_start = width + border;
- mode->crtc_hblank_end = mode->crtc_hblank_start + blank_width;
+ adjusted_mode->crtc_hdisplay = width;
+ adjusted_mode->crtc_hblank_start = width + border;
+ adjusted_mode->crtc_hblank_end = adjusted_mode->crtc_hblank_start + blank_width;
- mode->crtc_hsync_start = mode->crtc_hblank_start + sync_pos;
- mode->crtc_hsync_end = mode->crtc_hsync_start + sync_width;
+ adjusted_mode->crtc_hsync_start = adjusted_mode->crtc_hblank_start + sync_pos;
+ adjusted_mode->crtc_hsync_end = adjusted_mode->crtc_hsync_start + sync_width;
}
static void
-centre_vertically(struct drm_display_mode *mode,
+centre_vertically(struct drm_display_mode *adjusted_mode,
int height)
{
u32 border, sync_pos, blank_width, sync_width;
/* keep the vsync and vblank widths constant */
- sync_width = mode->crtc_vsync_end - mode->crtc_vsync_start;
- blank_width = mode->crtc_vblank_end - mode->crtc_vblank_start;
+ sync_width = adjusted_mode->crtc_vsync_end - adjusted_mode->crtc_vsync_start;
+ blank_width = adjusted_mode->crtc_vblank_end - adjusted_mode->crtc_vblank_start;
sync_pos = (blank_width - sync_width + 1) / 2;
- border = (mode->vdisplay - height + 1) / 2;
+ border = (adjusted_mode->crtc_vdisplay - height + 1) / 2;
- mode->crtc_vdisplay = height;
- mode->crtc_vblank_start = height + border;
- mode->crtc_vblank_end = mode->crtc_vblank_start + blank_width;
+ adjusted_mode->crtc_vdisplay = height;
+ adjusted_mode->crtc_vblank_start = height + border;
+ adjusted_mode->crtc_vblank_end = adjusted_mode->crtc_vblank_start + blank_width;
- mode->crtc_vsync_start = mode->crtc_vblank_start + sync_pos;
- mode->crtc_vsync_end = mode->crtc_vsync_start + sync_width;
+ adjusted_mode->crtc_vsync_start = adjusted_mode->crtc_vblank_start + sync_pos;
+ adjusted_mode->crtc_vsync_end = adjusted_mode->crtc_vsync_start + sync_width;
}
static inline u32 panel_fitter_scaling(u32 source, u32 target)
static void i965_scale_aspect(struct intel_crtc_state *pipe_config,
u32 *pfit_control)
{
- struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
- u32 scaled_width = adjusted_mode->hdisplay *
+ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+ u32 scaled_width = adjusted_mode->crtc_hdisplay *
pipe_config->pipe_src_h;
u32 scaled_height = pipe_config->pipe_src_w *
- adjusted_mode->vdisplay;
+ adjusted_mode->crtc_vdisplay;
/* 965+ is easy, it does everything in hw */
if (scaled_width > scaled_height)
else if (scaled_width < scaled_height)
*pfit_control |= PFIT_ENABLE |
PFIT_SCALING_LETTER;
- else if (adjusted_mode->hdisplay != pipe_config->pipe_src_w)
+ else if (adjusted_mode->crtc_hdisplay != pipe_config->pipe_src_w)
*pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO;
}
u32 *border)
{
struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
- u32 scaled_width = adjusted_mode->hdisplay *
+ u32 scaled_width = adjusted_mode->crtc_hdisplay *
pipe_config->pipe_src_h;
u32 scaled_height = pipe_config->pipe_src_w *
- adjusted_mode->vdisplay;
+ adjusted_mode->crtc_vdisplay;
u32 bits;
/*
pipe_config->pipe_src_h);
*border = LVDS_BORDER_ENABLE;
- if (pipe_config->pipe_src_h != adjusted_mode->vdisplay) {
+ if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay) {
bits = panel_fitter_scaling(pipe_config->pipe_src_h,
- adjusted_mode->vdisplay);
+ adjusted_mode->crtc_vdisplay);
*pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT |
bits << PFIT_VERT_SCALE_SHIFT);
pipe_config->pipe_src_w);
*border = LVDS_BORDER_ENABLE;
- if (pipe_config->pipe_src_w != adjusted_mode->hdisplay) {
+ if (pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) {
bits = panel_fitter_scaling(pipe_config->pipe_src_w,
- adjusted_mode->hdisplay);
+ adjusted_mode->crtc_hdisplay);
*pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT |
bits << PFIT_VERT_SCALE_SHIFT);
{
struct drm_device *dev = intel_crtc->base.dev;
u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0;
- struct drm_display_mode *adjusted_mode;
-
- adjusted_mode = &pipe_config->base.adjusted_mode;
+ struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
/* Native modes don't need fitting */
- if (adjusted_mode->hdisplay == pipe_config->pipe_src_w &&
- adjusted_mode->vdisplay == pipe_config->pipe_src_h)
+ if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w &&
+ adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h)
goto out;
switch (fitting_mode) {
* Full scaling, even if it changes the aspect ratio.
* Fortunately this is all done for us in hw.
*/
- if (pipe_config->pipe_src_h != adjusted_mode->vdisplay ||
- pipe_config->pipe_src_w != adjusted_mode->hdisplay) {
+ if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay ||
+ pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) {
pfit_control |= PFIT_ENABLE;
if (INTEL_INFO(dev)->gen >= 4)
pfit_control |= PFIT_SCALING_AUTO;
/* Assume that the BIOS does not lie through the OpRegion... */
if (!i915.panel_ignore_lid && dev_priv->opregion.lid_state) {
- return ioread32(dev_priv->opregion.lid_state) & 0x1 ?
+ return *dev_priv->opregion.lid_state & 0x1 ?
connector_status_connected :
connector_status_disconnected;
}
return val;
}
-static u32 bdw_get_backlight(struct intel_connector *connector)
+static u32 lpt_get_backlight(struct intel_connector *connector)
{
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
static u32 bxt_get_backlight(struct intel_connector *connector)
{
struct drm_device *dev = connector->base.dev;
+ struct intel_panel *panel = &connector->panel;
struct drm_i915_private *dev_priv = dev->dev_private;
- return I915_READ(BXT_BLC_PWM_DUTY1);
+ return I915_READ(BXT_BLC_PWM_DUTY(panel->backlight.controller));
}
static u32 pwm_get_backlight(struct intel_connector *connector)
mutex_lock(&dev_priv->backlight_lock);
if (panel->backlight.enabled) {
- val = dev_priv->display.get_backlight(connector);
+ val = panel->backlight.get(connector);
val = intel_panel_compute_brightness(connector, val);
}
return val;
}
-static void bdw_set_backlight(struct intel_connector *connector, u32 level)
+static void lpt_set_backlight(struct intel_connector *connector, u32 level)
{
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
{
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_panel *panel = &connector->panel;
- I915_WRITE(BXT_BLC_PWM_DUTY1, level);
+ I915_WRITE(BXT_BLC_PWM_DUTY(panel->backlight.controller), level);
}
static void pwm_set_backlight(struct intel_connector *connector, u32 level)
static void
intel_panel_actually_set_backlight(struct intel_connector *connector, u32 level)
{
- struct drm_device *dev = connector->base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_panel *panel = &connector->panel;
DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level);
level = intel_panel_compute_brightness(connector, level);
- dev_priv->display.set_backlight(connector, level);
+ panel->backlight.set(connector, level);
}
/* set backlight brightness to level in range [0..max], scaling wrt hw min */
mutex_unlock(&dev_priv->backlight_lock);
}
+static void lpt_disable_backlight(struct intel_connector *connector)
+{
+ struct drm_device *dev = connector->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u32 tmp;
+
+ intel_panel_actually_set_backlight(connector, 0);
+
+ tmp = I915_READ(BLC_PWM_PCH_CTL1);
+ I915_WRITE(BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE);
+}
+
static void pch_disable_backlight(struct intel_connector *connector)
{
struct drm_device *dev = connector->base.dev;
{
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- u32 tmp;
+ struct intel_panel *panel = &connector->panel;
+ u32 tmp, val;
intel_panel_actually_set_backlight(connector, 0);
- tmp = I915_READ(BXT_BLC_PWM_CTL1);
- I915_WRITE(BXT_BLC_PWM_CTL1, tmp & ~BXT_BLC_PWM_ENABLE);
+ tmp = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
+ I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller),
+ tmp & ~BXT_BLC_PWM_ENABLE);
+
+ if (panel->backlight.controller == 1) {
+ val = I915_READ(UTIL_PIN_CTL);
+ val &= ~UTIL_PIN_ENABLE;
+ I915_WRITE(UTIL_PIN_CTL, val);
+ }
}
static void pwm_disable_backlight(struct intel_connector *connector)
return;
/*
- * Do not disable backlight on the vgaswitcheroo path. When switching
+ * Do not disable backlight on the vga_switcheroo path. When switching
* away from i915, the other client may depend on i915 to handle the
* backlight. This will leave the backlight on unnecessarily when
* another client is not activated.
if (panel->backlight.device)
panel->backlight.device->props.power = FB_BLANK_POWERDOWN;
panel->backlight.enabled = false;
- dev_priv->display.disable_backlight(connector);
+ panel->backlight.disable(connector);
mutex_unlock(&dev_priv->backlight_lock);
}
-static void bdw_enable_backlight(struct intel_connector *connector)
+static void lpt_enable_backlight(struct intel_connector *connector)
{
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_panel *panel = &connector->panel;
- u32 pwm_ctl;
+ enum pipe pipe = intel_get_pipe_from_connector(connector);
+ u32 pwm_ctl, val;
+
+ /* To use 2nd set of backlight registers, utility pin has to be
+ * enabled with PWM mode.
+ * The field should only be changed when the utility pin is disabled
+ */
+ if (panel->backlight.controller == 1) {
+ val = I915_READ(UTIL_PIN_CTL);
+ if (val & UTIL_PIN_ENABLE) {
+ DRM_DEBUG_KMS("util pin already enabled\n");
+ val &= ~UTIL_PIN_ENABLE;
+ I915_WRITE(UTIL_PIN_CTL, val);
+ }
- pwm_ctl = I915_READ(BXT_BLC_PWM_CTL1);
+ val = 0;
+ if (panel->backlight.util_pin_active_low)
+ val |= UTIL_PIN_POLARITY;
+ I915_WRITE(UTIL_PIN_CTL, val | UTIL_PIN_PIPE(pipe) |
+ UTIL_PIN_MODE_PWM | UTIL_PIN_ENABLE);
+ }
+
+ pwm_ctl = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
if (pwm_ctl & BXT_BLC_PWM_ENABLE) {
DRM_DEBUG_KMS("backlight already enabled\n");
pwm_ctl &= ~BXT_BLC_PWM_ENABLE;
- I915_WRITE(BXT_BLC_PWM_CTL1, pwm_ctl);
+ I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller),
+ pwm_ctl);
}
- I915_WRITE(BXT_BLC_PWM_FREQ1, panel->backlight.max);
+ I915_WRITE(BXT_BLC_PWM_FREQ(panel->backlight.controller),
+ panel->backlight.max);
intel_panel_actually_set_backlight(connector, panel->backlight.level);
if (panel->backlight.active_low_pwm)
pwm_ctl |= BXT_BLC_PWM_POLARITY;
- I915_WRITE(BXT_BLC_PWM_CTL1, pwm_ctl);
- POSTING_READ(BXT_BLC_PWM_CTL1);
- I915_WRITE(BXT_BLC_PWM_CTL1, pwm_ctl | BXT_BLC_PWM_ENABLE);
+ I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller), pwm_ctl);
+ POSTING_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
+ I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller),
+ pwm_ctl | BXT_BLC_PWM_ENABLE);
}
static void pwm_enable_backlight(struct intel_connector *connector)
panel->backlight.device->props.max_brightness);
}
- dev_priv->display.enable_backlight(connector);
+ panel->backlight.enable(connector);
panel->backlight.enabled = true;
if (panel->backlight.device)
panel->backlight.device->props.power = FB_BLANK_UNBLANK;
* callback needs to take this into account.
*/
if (panel->backlight.enabled) {
- if (panel->backlight_power) {
+ if (panel->backlight.power) {
bool enable = bd->props.power == FB_BLANK_UNBLANK &&
bd->props.brightness != 0;
- panel->backlight_power(connector, enable);
+ panel->backlight.power(connector, enable);
}
} else {
bd->props.power = FB_BLANK_POWERDOWN;
#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
/*
- * Note: The setup hooks can't assume pipe is set!
+ * SPT: This value represents the period of the PWM stream in clock periods
+ * multiplied by 16 (default increment) or 128 (alternate increment selected in
+ * SCHICKEN_1 bit 0). PWM clock is 24 MHz.
+ */
+static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+ struct drm_device *dev = connector->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u32 mul, clock;
+
+ if (I915_READ(SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY)
+ mul = 128;
+ else
+ mul = 16;
+
+ clock = MHz(24);
+
+ return clock / (pwm_freq_hz * mul);
+}
+
+/*
+ * LPT: This value represents the period of the PWM stream in clock periods
+ * multiplied by 128 (default increment) or 16 (alternate increment, selected in
+ * LPT SOUTH_CHICKEN2 register bit 5).
+ */
+static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+ struct drm_device *dev = connector->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u32 mul, clock;
+
+ if (I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY)
+ mul = 16;
+ else
+ mul = 128;
+
+ if (dev_priv->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
+ clock = MHz(135); /* LPT:H */
+ else
+ clock = MHz(24); /* LPT:LP */
+
+ return clock / (pwm_freq_hz * mul);
+}
+
+/*
+ * ILK/SNB/IVB: This value represents the period of the PWM stream in PCH
+ * display raw clocks multiplied by 128.
+ */
+static u32 pch_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+ struct drm_device *dev = connector->base.dev;
+ int clock = MHz(intel_pch_rawclk(dev));
+
+ return clock / (pwm_freq_hz * 128);
+}
+
+/*
+ * Gen2: This field determines the number of time base events (display core
+ * clock frequency/32) in total for a complete cycle of modulated backlight
+ * control.
*
- * XXX: Query mode clock or hardware clock and program PWM modulation frequency
- * appropriately when it's 0. Use VBT and/or sane defaults.
+ * Gen3: A time base event equals the display core clock ([DevPNV] HRAW clock)
+ * divided by 32.
+ */
+static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+ struct drm_device *dev = connector->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int clock;
+
+ if (IS_PINEVIEW(dev))
+ clock = intel_hrawclk(dev);
+ else
+ clock = 1000 * dev_priv->display.get_display_clock_speed(dev);
+
+ return clock / (pwm_freq_hz * 32);
+}
+
+/*
+ * Gen4: This value represents the period of the PWM stream in display core
+ * clocks multiplied by 128.
+ */
+static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+ struct drm_device *dev = connector->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int clock = 1000 * dev_priv->display.get_display_clock_speed(dev);
+
+ return clock / (pwm_freq_hz * 128);
+}
+
+/*
+ * VLV: This value represents the period of the PWM stream in display core
+ * clocks ([DevCTG] 200MHz HRAW clocks) multiplied by 128 or 25MHz S0IX clocks
+ * multiplied by 16. CHV uses a 19.2MHz S0IX clock.
+ */
+static u32 vlv_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+ struct drm_device *dev = connector->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int clock;
+
+ if ((I915_READ(CBR1_VLV) & CBR_PWM_CLOCK_MUX_SELECT) == 0) {
+ if (IS_CHERRYVIEW(dev))
+ return KHz(19200) / (pwm_freq_hz * 16);
+ else
+ return MHz(25) / (pwm_freq_hz * 16);
+ } else {
+ clock = intel_hrawclk(dev);
+ return MHz(clock) / (pwm_freq_hz * 128);
+ }
+}
+
+static u32 get_backlight_max_vbt(struct intel_connector *connector)
+{
+ struct drm_device *dev = connector->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_panel *panel = &connector->panel;
+ u16 pwm_freq_hz = dev_priv->vbt.backlight.pwm_freq_hz;
+ u32 pwm;
+
+ if (!pwm_freq_hz) {
+ DRM_DEBUG_KMS("backlight frequency not specified in VBT\n");
+ return 0;
+ }
+
+ if (!panel->backlight.hz_to_pwm) {
+ DRM_DEBUG_KMS("backlight frequency setting from VBT currently not supported on this platform\n");
+ return 0;
+ }
+
+ pwm = panel->backlight.hz_to_pwm(connector, pwm_freq_hz);
+ if (!pwm) {
+ DRM_DEBUG_KMS("backlight frequency conversion failed\n");
+ return 0;
+ }
+
+ DRM_DEBUG_KMS("backlight frequency %u Hz from VBT\n", pwm_freq_hz);
+
+ return pwm;
+}
+
+/*
+ * Note: The setup hooks can't assume pipe is set!
*/
static u32 get_backlight_min_vbt(struct intel_connector *connector)
{
return scale(min, 0, 255, 0, panel->backlight.max);
}
-static int bdw_setup_backlight(struct intel_connector *connector, enum pipe unused)
+static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unused)
{
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
pch_ctl2 = I915_READ(BLC_PWM_PCH_CTL2);
panel->backlight.max = pch_ctl2 >> 16;
+
+ if (!panel->backlight.max)
+ panel->backlight.max = get_backlight_max_vbt(connector);
+
if (!panel->backlight.max)
return -ENODEV;
panel->backlight.min = get_backlight_min_vbt(connector);
- val = bdw_get_backlight(connector);
+ val = lpt_get_backlight(connector);
panel->backlight.level = intel_panel_compute_brightness(connector, val);
panel->backlight.enabled = (pch_ctl1 & BLM_PCH_PWM_ENABLE) &&
pch_ctl2 = I915_READ(BLC_PWM_PCH_CTL2);
panel->backlight.max = pch_ctl2 >> 16;
+
+ if (!panel->backlight.max)
+ panel->backlight.max = get_backlight_max_vbt(connector);
+
if (!panel->backlight.max)
return -ENODEV;
panel->backlight.active_low_pwm = ctl & BLM_POLARITY_PNV;
panel->backlight.max = ctl >> 17;
- if (panel->backlight.combination_mode)
- panel->backlight.max *= 0xff;
+
+ if (!panel->backlight.max) {
+ panel->backlight.max = get_backlight_max_vbt(connector);
+ panel->backlight.max >>= 1;
+ }
if (!panel->backlight.max)
return -ENODEV;
+ if (panel->backlight.combination_mode)
+ panel->backlight.max *= 0xff;
+
panel->backlight.min = get_backlight_min_vbt(connector);
val = i9xx_get_backlight(connector);
ctl = I915_READ(BLC_PWM_CTL);
panel->backlight.max = ctl >> 16;
- if (panel->backlight.combination_mode)
- panel->backlight.max *= 0xff;
+
+ if (!panel->backlight.max)
+ panel->backlight.max = get_backlight_max_vbt(connector);
if (!panel->backlight.max)
return -ENODEV;
+ if (panel->backlight.combination_mode)
+ panel->backlight.max *= 0xff;
+
panel->backlight.min = get_backlight_min_vbt(connector);
val = i9xx_get_backlight(connector);
struct drm_device *dev = connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_panel *panel = &connector->panel;
- enum pipe p;
u32 ctl, ctl2, val;
- for_each_pipe(dev_priv, p) {
- u32 cur_val = I915_READ(VLV_BLC_PWM_CTL(p));
-
- /* Skip if the modulation freq is already set */
- if (cur_val & ~BACKLIGHT_DUTY_CYCLE_MASK)
- continue;
-
- cur_val &= BACKLIGHT_DUTY_CYCLE_MASK;
- I915_WRITE(VLV_BLC_PWM_CTL(p), (0xf42 << 16) |
- cur_val);
- }
-
if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B))
return -ENODEV;
ctl = I915_READ(VLV_BLC_PWM_CTL(pipe));
panel->backlight.max = ctl >> 16;
+
+ if (!panel->backlight.max)
+ panel->backlight.max = get_backlight_max_vbt(connector);
+
if (!panel->backlight.max)
return -ENODEV;
struct intel_panel *panel = &connector->panel;
u32 pwm_ctl, val;
- pwm_ctl = I915_READ(BXT_BLC_PWM_CTL1);
+ /*
+ * For BXT hard coding the Backlight controller to 0.
+ * TODO : Read the controller value from VBT and generalize
+ */
+ panel->backlight.controller = 0;
+
+ pwm_ctl = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller));
+
+ /* Keeping the check if controller 1 is to be programmed.
+ * This will come into affect once the VBT parsing
+ * is fixed for controller selection, and controller 1 is used
+ * for a prticular display configuration.
+ */
+ if (panel->backlight.controller == 1) {
+ val = I915_READ(UTIL_PIN_CTL);
+ panel->backlight.util_pin_active_low =
+ val & UTIL_PIN_POLARITY;
+ }
+
panel->backlight.active_low_pwm = pwm_ctl & BXT_BLC_PWM_POLARITY;
+ panel->backlight.max =
+ I915_READ(BXT_BLC_PWM_FREQ(panel->backlight.controller));
+
+ if (!panel->backlight.max)
+ panel->backlight.max = get_backlight_max_vbt(connector);
- panel->backlight.max = I915_READ(BXT_BLC_PWM_FREQ1);
if (!panel->backlight.max)
return -ENODEV;
}
}
+ /* ensure intel_panel has been initialized first */
+ if (WARN_ON(!panel->backlight.setup))
+ return -ENODEV;
+
/* set level and max in panel struct */
mutex_lock(&dev_priv->backlight_lock);
- ret = dev_priv->display.setup_backlight(intel_connector, pipe);
+ ret = panel->backlight.setup(intel_connector, pipe);
mutex_unlock(&dev_priv->backlight_lock);
if (ret) {
}
/* Set up chip specific backlight functions */
-void intel_panel_init_backlight_funcs(struct drm_device *dev)
+static void
+intel_panel_init_backlight_funcs(struct intel_panel *panel)
{
+ struct intel_connector *intel_connector =
+ container_of(panel, struct intel_connector, panel);
+ struct drm_device *dev = intel_connector->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
if (IS_BROXTON(dev)) {
- dev_priv->display.setup_backlight = bxt_setup_backlight;
- dev_priv->display.enable_backlight = bxt_enable_backlight;
- dev_priv->display.disable_backlight = bxt_disable_backlight;
- dev_priv->display.set_backlight = bxt_set_backlight;
- dev_priv->display.get_backlight = bxt_get_backlight;
- } else if (IS_BROADWELL(dev) || IS_SKYLAKE(dev)) {
- dev_priv->display.setup_backlight = bdw_setup_backlight;
- dev_priv->display.enable_backlight = bdw_enable_backlight;
- dev_priv->display.disable_backlight = pch_disable_backlight;
- dev_priv->display.set_backlight = bdw_set_backlight;
- dev_priv->display.get_backlight = bdw_get_backlight;
+ panel->backlight.setup = bxt_setup_backlight;
+ panel->backlight.enable = bxt_enable_backlight;
+ panel->backlight.disable = bxt_disable_backlight;
+ panel->backlight.set = bxt_set_backlight;
+ panel->backlight.get = bxt_get_backlight;
+ } else if (HAS_PCH_LPT(dev) || HAS_PCH_SPT(dev)) {
+ panel->backlight.setup = lpt_setup_backlight;
+ panel->backlight.enable = lpt_enable_backlight;
+ panel->backlight.disable = lpt_disable_backlight;
+ panel->backlight.set = lpt_set_backlight;
+ panel->backlight.get = lpt_get_backlight;
+ if (HAS_PCH_LPT(dev))
+ panel->backlight.hz_to_pwm = lpt_hz_to_pwm;
+ else
+ panel->backlight.hz_to_pwm = spt_hz_to_pwm;
} else if (HAS_PCH_SPLIT(dev)) {
- dev_priv->display.setup_backlight = pch_setup_backlight;
- dev_priv->display.enable_backlight = pch_enable_backlight;
- dev_priv->display.disable_backlight = pch_disable_backlight;
- dev_priv->display.set_backlight = pch_set_backlight;
- dev_priv->display.get_backlight = pch_get_backlight;
+ panel->backlight.setup = pch_setup_backlight;
+ panel->backlight.enable = pch_enable_backlight;
+ panel->backlight.disable = pch_disable_backlight;
+ panel->backlight.set = pch_set_backlight;
+ panel->backlight.get = pch_get_backlight;
+ panel->backlight.hz_to_pwm = pch_hz_to_pwm;
} else if (IS_VALLEYVIEW(dev)) {
if (dev_priv->vbt.has_mipi) {
- dev_priv->display.setup_backlight = pwm_setup_backlight;
- dev_priv->display.enable_backlight = pwm_enable_backlight;
- dev_priv->display.disable_backlight = pwm_disable_backlight;
- dev_priv->display.set_backlight = pwm_set_backlight;
- dev_priv->display.get_backlight = pwm_get_backlight;
+ panel->backlight.setup = pwm_setup_backlight;
+ panel->backlight.enable = pwm_enable_backlight;
+ panel->backlight.disable = pwm_disable_backlight;
+ panel->backlight.set = pwm_set_backlight;
+ panel->backlight.get = pwm_get_backlight;
} else {
- dev_priv->display.setup_backlight = vlv_setup_backlight;
- dev_priv->display.enable_backlight = vlv_enable_backlight;
- dev_priv->display.disable_backlight = vlv_disable_backlight;
- dev_priv->display.set_backlight = vlv_set_backlight;
- dev_priv->display.get_backlight = vlv_get_backlight;
+ panel->backlight.setup = vlv_setup_backlight;
+ panel->backlight.enable = vlv_enable_backlight;
+ panel->backlight.disable = vlv_disable_backlight;
+ panel->backlight.set = vlv_set_backlight;
+ panel->backlight.get = vlv_get_backlight;
+ panel->backlight.hz_to_pwm = vlv_hz_to_pwm;
}
} else if (IS_GEN4(dev)) {
- dev_priv->display.setup_backlight = i965_setup_backlight;
- dev_priv->display.enable_backlight = i965_enable_backlight;
- dev_priv->display.disable_backlight = i965_disable_backlight;
- dev_priv->display.set_backlight = i9xx_set_backlight;
- dev_priv->display.get_backlight = i9xx_get_backlight;
+ panel->backlight.setup = i965_setup_backlight;
+ panel->backlight.enable = i965_enable_backlight;
+ panel->backlight.disable = i965_disable_backlight;
+ panel->backlight.set = i9xx_set_backlight;
+ panel->backlight.get = i9xx_get_backlight;
+ panel->backlight.hz_to_pwm = i965_hz_to_pwm;
} else {
- dev_priv->display.setup_backlight = i9xx_setup_backlight;
- dev_priv->display.enable_backlight = i9xx_enable_backlight;
- dev_priv->display.disable_backlight = i9xx_disable_backlight;
- dev_priv->display.set_backlight = i9xx_set_backlight;
- dev_priv->display.get_backlight = i9xx_get_backlight;
+ panel->backlight.setup = i9xx_setup_backlight;
+ panel->backlight.enable = i9xx_enable_backlight;
+ panel->backlight.disable = i9xx_disable_backlight;
+ panel->backlight.set = i9xx_set_backlight;
+ panel->backlight.get = i9xx_get_backlight;
+ panel->backlight.hz_to_pwm = i9xx_hz_to_pwm;
}
}
struct drm_display_mode *fixed_mode,
struct drm_display_mode *downclock_mode)
{
+ intel_panel_init_backlight_funcs(panel);
+
panel->fixed_mode = fixed_mode;
panel->downclock_mode = downclock_mode;
#define INTEL_RC6p_ENABLE (1<<1)
#define INTEL_RC6pp_ENABLE (1<<2)
-static void gen9_init_clock_gating(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- /* WaEnableLbsSlaRetryTimerDecrement:skl */
- I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
- GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
-
- /* WaDisableKillLogic:bxt,skl */
- I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
- ECOCHK_DIS_TLB);
-}
-
-static void skl_init_clock_gating(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- gen9_init_clock_gating(dev);
-
- if (INTEL_REVID(dev) <= SKL_REVID_B0) {
- /*
- * WaDisableSDEUnitClockGating:skl
- * WaSetGAPSunitClckGateDisable:skl
- */
- I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
- GEN8_GAPSUNIT_CLOCK_GATE_DISABLE |
- GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
-
- /* WaDisableVFUnitClockGating:skl */
- I915_WRITE(GEN6_UCGCTL2, I915_READ(GEN6_UCGCTL2) |
- GEN6_VFUNIT_CLOCK_GATE_DISABLE);
- }
-
- if (INTEL_REVID(dev) <= SKL_REVID_D0) {
- /* WaDisableHDCInvalidation:skl */
- I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
- BDW_DISABLE_HDC_INVALIDATION);
-
- /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
- I915_WRITE(FF_SLICE_CS_CHICKEN2,
- _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
- }
-
- /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
- * involving this register should also be added to WA batch as required.
- */
- if (INTEL_REVID(dev) <= SKL_REVID_E0)
- /* WaDisableLSQCROPERFforOCL:skl */
- I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
- GEN8_LQSC_RO_PERF_DIS);
-
- /* WaEnableGapsTsvCreditFix:skl */
- if (IS_SKYLAKE(dev) && (INTEL_REVID(dev) >= SKL_REVID_C0)) {
- I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
- GEN9_GAPS_TSV_CREDIT_DISABLE));
- }
-}
-
static void bxt_init_clock_gating(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- gen9_init_clock_gating(dev);
+ /* WaDisableSDEUnitClockGating:bxt */
+ I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
+ GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
/*
* FIXME:
- * GEN8_SDEUNIT_CLOCK_GATE_DISABLE applies on A0 only.
* GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
*/
- /* WaDisableSDEUnitClockGating:bxt */
I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
- GEN8_SDEUNIT_CLOCK_GATE_DISABLE |
GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
-
- /* FIXME: apply on A0 only */
- I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
}
static void i915_pineview_get_mem_freq(struct drm_device *dev)
crtc = single_enabled_crtc(dev);
if (crtc) {
- const struct drm_display_mode *adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
- int clock;
-
- adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
- clock = adjusted_mode->crtc_clock;
+ int clock = adjusted_mode->crtc_clock;
/* Display SR */
wm = intel_calculate_wm(clock, &pineview_display_wm,
if (crtc) {
/* self-refresh has much higher latency */
static const int sr_latency_ns = 12000;
- const struct drm_display_mode *adjusted_mode =
- &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
int clock = adjusted_mode->crtc_clock;
int htotal = adjusted_mode->crtc_htotal;
int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
if (HAS_FW_BLC(dev) && enabled) {
/* self-refresh has much higher latency */
static const int sr_latency_ns = 6000;
- const struct drm_display_mode *adjusted_mode =
- &to_intel_crtc(enabled)->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
int clock = adjusted_mode->crtc_clock;
int htotal = adjusted_mode->crtc_htotal;
int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
uint32_t pipe_htotal;
uint32_t pixel_rate; /* in KHz */
struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
- struct intel_plane_wm_parameters cursor;
-};
-
-struct ilk_pipe_wm_parameters {
- bool active;
- uint32_t pipe_htotal;
- uint32_t pixel_rate;
- struct intel_plane_wm_parameters pri;
- struct intel_plane_wm_parameters spr;
- struct intel_plane_wm_parameters cur;
};
struct ilk_wm_maximums {
* For both WM_PIPE and WM_LP.
* mem_value must be in 0.1us units.
*/
-static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
+static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
+ const struct intel_plane_state *pstate,
uint32_t mem_value,
bool is_lp)
{
+ int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
uint32_t method1, method2;
- if (!params->active || !params->pri.enabled)
+ if (!cstate->base.active || !pstate->visible)
return 0;
- method1 = ilk_wm_method1(params->pixel_rate,
- params->pri.bytes_per_pixel,
- mem_value);
+ method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
if (!is_lp)
return method1;
- method2 = ilk_wm_method2(params->pixel_rate,
- params->pipe_htotal,
- params->pri.horiz_pixels,
- params->pri.bytes_per_pixel,
+ method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
+ cstate->base.adjusted_mode.crtc_htotal,
+ drm_rect_width(&pstate->dst),
+ bpp,
mem_value);
return min(method1, method2);
* For both WM_PIPE and WM_LP.
* mem_value must be in 0.1us units.
*/
-static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
+static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
+ const struct intel_plane_state *pstate,
uint32_t mem_value)
{
+ int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
uint32_t method1, method2;
- if (!params->active || !params->spr.enabled)
+ if (!cstate->base.active || !pstate->visible)
return 0;
- method1 = ilk_wm_method1(params->pixel_rate,
- params->spr.bytes_per_pixel,
- mem_value);
- method2 = ilk_wm_method2(params->pixel_rate,
- params->pipe_htotal,
- params->spr.horiz_pixels,
- params->spr.bytes_per_pixel,
+ method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
+ method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
+ cstate->base.adjusted_mode.crtc_htotal,
+ drm_rect_width(&pstate->dst),
+ bpp,
mem_value);
return min(method1, method2);
}
* For both WM_PIPE and WM_LP.
* mem_value must be in 0.1us units.
*/
-static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
+static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
+ const struct intel_plane_state *pstate,
uint32_t mem_value)
{
- if (!params->active || !params->cur.enabled)
+ int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
+
+ if (!cstate->base.active || !pstate->visible)
return 0;
- return ilk_wm_method2(params->pixel_rate,
- params->pipe_htotal,
- params->cur.horiz_pixels,
- params->cur.bytes_per_pixel,
+ return ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
+ cstate->base.adjusted_mode.crtc_htotal,
+ drm_rect_width(&pstate->dst),
+ bpp,
mem_value);
}
/* Only for WM_LP. */
-static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
+static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
+ const struct intel_plane_state *pstate,
uint32_t pri_val)
{
- if (!params->active || !params->pri.enabled)
+ int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
+
+ if (!cstate->base.active || !pstate->visible)
return 0;
- return ilk_wm_fbc(pri_val,
- params->pri.horiz_pixels,
- params->pri.bytes_per_pixel);
+ return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), bpp);
}
static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
}
static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
+ const struct intel_crtc *intel_crtc,
int level,
- const struct ilk_pipe_wm_parameters *p,
+ struct intel_crtc_state *cstate,
struct intel_wm_level *result)
{
+ struct intel_plane *intel_plane;
uint16_t pri_latency = dev_priv->wm.pri_latency[level];
uint16_t spr_latency = dev_priv->wm.spr_latency[level];
uint16_t cur_latency = dev_priv->wm.cur_latency[level];
cur_latency *= 5;
}
- result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
- result->spr_val = ilk_compute_spr_wm(p, spr_latency);
- result->cur_val = ilk_compute_cur_wm(p, cur_latency);
- result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
+ for_each_intel_plane_on_crtc(dev_priv->dev, intel_crtc, intel_plane) {
+ struct intel_plane_state *pstate =
+ to_intel_plane_state(intel_plane->base.state);
+
+ switch (intel_plane->base.type) {
+ case DRM_PLANE_TYPE_PRIMARY:
+ result->pri_val = ilk_compute_pri_wm(cstate, pstate,
+ pri_latency,
+ level);
+ result->fbc_val = ilk_compute_fbc_wm(cstate, pstate,
+ result->pri_val);
+ break;
+ case DRM_PLANE_TYPE_OVERLAY:
+ result->spr_val = ilk_compute_spr_wm(cstate, pstate,
+ spr_latency);
+ break;
+ case DRM_PLANE_TYPE_CURSOR:
+ result->cur_val = ilk_compute_cur_wm(cstate, pstate,
+ cur_latency);
+ break;
+ }
+ }
+
result->enable = true;
}
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
u32 linetime, ips_linetime;
if (!intel_crtc->active)
/* The WM are computed with base on how long it takes to fill a single
* row at the given clock rate, multiplied by 8.
* */
- linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
- mode->crtc_clock);
- ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
+ linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
+ adjusted_mode->crtc_clock);
+ ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
dev_priv->cdclk_freq);
return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
}
-static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
- struct ilk_pipe_wm_parameters *p)
-{
- struct drm_device *dev = crtc->dev;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- enum pipe pipe = intel_crtc->pipe;
- struct drm_plane *plane;
-
- if (!intel_crtc->active)
- return;
-
- p->active = true;
- p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
- p->pixel_rate = ilk_pipe_pixel_rate(intel_crtc->config);
-
- if (crtc->primary->state->fb)
- p->pri.bytes_per_pixel =
- crtc->primary->state->fb->bits_per_pixel / 8;
- else
- p->pri.bytes_per_pixel = 4;
-
- p->cur.bytes_per_pixel = 4;
- /*
- * TODO: for now, assume primary and cursor planes are always enabled.
- * Setting them to false makes the screen flicker.
- */
- p->pri.enabled = true;
- p->cur.enabled = true;
-
- p->pri.horiz_pixels = intel_crtc->config->pipe_src_w;
- p->cur.horiz_pixels = intel_crtc->base.cursor->state->crtc_w;
-
- drm_for_each_legacy_plane(plane, dev) {
- struct intel_plane *intel_plane = to_intel_plane(plane);
-
- if (intel_plane->pipe == pipe) {
- p->spr = intel_plane->wm;
- break;
- }
- }
-}
-
static void ilk_compute_wm_config(struct drm_device *dev,
struct intel_wm_config *config)
{
}
/* Compute new watermarks for the pipe */
-static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
- const struct ilk_pipe_wm_parameters *params,
+static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
struct intel_pipe_wm *pipe_wm)
{
+ struct drm_crtc *crtc = cstate->base.crtc;
struct drm_device *dev = crtc->dev;
const struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct intel_plane *intel_plane;
+ struct intel_plane_state *sprstate = NULL;
int level, max_level = ilk_wm_max_level(dev);
/* LP0 watermark maximums depend on this pipe alone */
struct intel_wm_config config = {
.num_pipes_active = 1,
- .sprites_enabled = params->spr.enabled,
- .sprites_scaled = params->spr.scaled,
};
struct ilk_wm_maximums max;
- pipe_wm->pipe_enabled = params->active;
- pipe_wm->sprites_enabled = params->spr.enabled;
- pipe_wm->sprites_scaled = params->spr.scaled;
+ for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+ if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY) {
+ sprstate = to_intel_plane_state(intel_plane->base.state);
+ break;
+ }
+ }
+
+ config.sprites_enabled = sprstate->visible;
+ config.sprites_scaled = sprstate->visible &&
+ (drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 ||
+ drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
+
+ pipe_wm->pipe_enabled = cstate->base.active;
+ pipe_wm->sprites_enabled = sprstate->visible;
+ pipe_wm->sprites_scaled = config.sprites_scaled;
/* ILK/SNB: LP2+ watermarks only w/o sprites */
- if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
+ if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible)
max_level = 1;
/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
- if (params->spr.scaled)
+ if (config.sprites_scaled)
max_level = 0;
- ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
+ ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, &pipe_wm->wm[0]);
if (IS_HASWELL(dev) || IS_BROADWELL(dev))
pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
for (level = 1; level <= max_level; level++) {
struct intel_wm_level wm = {};
- ilk_compute_wm_level(dev_priv, level, params, &wm);
+ ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, &wm);
/*
* Disable any watermark level that exceeds the
}
val = I915_READ(CUR_BUF_CFG(pipe));
- skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
+ skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR],
+ val);
}
}
alloc_size = skl_ddb_entry_size(alloc);
if (alloc_size == 0) {
memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
- memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
+ memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
+ sizeof(ddb->plane[pipe][PLANE_CURSOR]));
return;
}
cursor_blocks = skl_cursor_allocation(config);
- ddb->cursor[pipe].start = alloc->end - cursor_blocks;
- ddb->cursor[pipe].end = alloc->end;
+ ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
+ ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
alloc_size -= cursor_blocks;
alloc->end -= cursor_blocks;
sizeof(new_ddb->plane[pipe])))
return true;
- if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
- sizeof(new_ddb->cursor[pipe])))
+ if (memcmp(&new_ddb->plane[pipe][PLANE_CURSOR], &cur_ddb->plane[pipe][PLANE_CURSOR],
+ sizeof(new_ddb->plane[pipe][PLANE_CURSOR])))
return true;
return false;
if (fb) {
p->plane[0].enabled = true;
p->plane[0].bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
- drm_format_plane_cpp(fb->pixel_format, 1) : fb->bits_per_pixel / 8;
+ drm_format_plane_cpp(fb->pixel_format, 1) :
+ drm_format_plane_cpp(fb->pixel_format, 0);
p->plane[0].y_bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
drm_format_plane_cpp(fb->pixel_format, 0) : 0;
p->plane[0].tiling = fb->modifier[0];
p->plane[0].rotation = crtc->primary->state->rotation;
fb = crtc->cursor->state->fb;
- p->cursor.y_bytes_per_pixel = 0;
+ p->plane[PLANE_CURSOR].y_bytes_per_pixel = 0;
if (fb) {
- p->cursor.enabled = true;
- p->cursor.bytes_per_pixel = fb->bits_per_pixel / 8;
- p->cursor.horiz_pixels = crtc->cursor->state->crtc_w;
- p->cursor.vert_pixels = crtc->cursor->state->crtc_h;
+ p->plane[PLANE_CURSOR].enabled = true;
+ p->plane[PLANE_CURSOR].bytes_per_pixel = fb->bits_per_pixel / 8;
+ p->plane[PLANE_CURSOR].horiz_pixels = crtc->cursor->state->crtc_w;
+ p->plane[PLANE_CURSOR].vert_pixels = crtc->cursor->state->crtc_h;
} else {
- p->cursor.enabled = false;
- p->cursor.bytes_per_pixel = 0;
- p->cursor.horiz_pixels = 64;
- p->cursor.vert_pixels = 64;
+ p->plane[PLANE_CURSOR].enabled = false;
+ p->plane[PLANE_CURSOR].bytes_per_pixel = 0;
+ p->plane[PLANE_CURSOR].horiz_pixels = 64;
+ p->plane[PLANE_CURSOR].vert_pixels = 64;
}
}
&result->plane_res_l[i]);
}
- ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
- result->cursor_en = skl_compute_plane_wm(dev_priv, p, &p->cursor,
+ ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][PLANE_CURSOR]);
+ result->plane_en[PLANE_CURSOR] = skl_compute_plane_wm(dev_priv, p,
+ &p->plane[PLANE_CURSOR],
ddb_blocks, level,
- &result->cursor_res_b,
- &result->cursor_res_l);
+ &result->plane_res_b[PLANE_CURSOR],
+ &result->plane_res_l[PLANE_CURSOR]);
}
static uint32_t
/* Until we know more, just disable transition WMs */
for (i = 0; i < intel_num_planes(intel_crtc); i++)
trans_wm->plane_en[i] = false;
- trans_wm->cursor_en = false;
+ trans_wm->plane_en[PLANE_CURSOR] = false;
}
static void skl_compute_pipe_wm(struct drm_crtc *crtc,
temp = 0;
- temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
- temp |= p_wm->wm[level].cursor_res_b;
+ temp |= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
+ temp |= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
- if (p_wm->wm[level].cursor_en)
+ if (p_wm->wm[level].plane_en[PLANE_CURSOR])
temp |= PLANE_WM_EN;
- r->cursor[pipe][level] = temp;
+ r->plane[pipe][PLANE_CURSOR][level] = temp;
}
}
temp = 0;
- temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
- temp |= p_wm->trans_wm.cursor_res_b;
- if (p_wm->trans_wm.cursor_en)
+ temp |= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
+ temp |= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
+ if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
temp |= PLANE_WM_EN;
- r->cursor_trans[pipe] = temp;
+ r->plane_trans[pipe][PLANE_CURSOR] = temp;
r->wm_linetime[pipe] = p_wm->linetime;
}
I915_WRITE(PLANE_WM(pipe, i, level),
new->plane[pipe][i][level]);
I915_WRITE(CUR_WM(pipe, level),
- new->cursor[pipe][level]);
+ new->plane[pipe][PLANE_CURSOR][level]);
}
for (i = 0; i < intel_num_planes(crtc); i++)
I915_WRITE(PLANE_WM_TRANS(pipe, i),
new->plane_trans[pipe][i]);
- I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
+ I915_WRITE(CUR_WM_TRANS(pipe),
+ new->plane_trans[pipe][PLANE_CURSOR]);
for (i = 0; i < intel_num_planes(crtc); i++) {
skl_ddb_entry_write(dev_priv,
}
skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
- &new->ddb.cursor[pipe]);
+ &new->ddb.plane[pipe][PLANE_CURSOR]);
}
}
}
}
+static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
+{
+ watermarks->wm_linetime[pipe] = 0;
+ memset(watermarks->plane[pipe], 0,
+ sizeof(uint32_t) * 8 * I915_MAX_PLANES);
+ memset(watermarks->plane_trans[pipe],
+ 0, sizeof(uint32_t) * I915_MAX_PLANES);
+ watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
+
+ /* Clear ddb entries for pipe */
+ memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
+ memset(&watermarks->ddb.plane[pipe], 0,
+ sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
+ memset(&watermarks->ddb.y_plane[pipe], 0,
+ sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
+ memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
+ sizeof(struct skl_ddb_entry));
+
+}
+
static void skl_update_wm(struct drm_crtc *crtc)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct skl_pipe_wm pipe_wm = {};
struct intel_wm_config config = {};
- memset(results, 0, sizeof(*results));
+
+ /* Clear all dirty flags */
+ memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
+
+ skl_clear_wm(results, intel_crtc->pipe);
skl_compute_wm_global_parameters(dev, &config);
static void ilk_update_wm(struct drm_crtc *crtc)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct ilk_wm_maximums max;
- struct ilk_pipe_wm_parameters params = {};
struct ilk_wm_values results = {};
enum intel_ddb_partitioning partitioning;
struct intel_pipe_wm pipe_wm = {};
struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
struct intel_wm_config config = {};
- ilk_compute_wm_parameters(crtc, ¶ms);
+ WARN_ON(cstate->base.active != intel_crtc->active);
- intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm);
+ intel_compute_pipe_wm(cstate, &pipe_wm);
if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
return;
struct drm_device *dev = plane->dev;
struct intel_plane *intel_plane = to_intel_plane(plane);
- intel_plane->wm.enabled = enabled;
- intel_plane->wm.scaled = scaled;
- intel_plane->wm.horiz_pixels = sprite_width;
- intel_plane->wm.vert_pixels = sprite_width;
- intel_plane->wm.bytes_per_pixel = pixel_size;
-
/*
* IVB workaround: must disable low power watermarks for at least
* one frame before enabling scaling. LP watermarks can be re-enabled
(val >> PLANE_WM_LINES_SHIFT) &
PLANE_WM_LINES_MASK;
} else {
- active->wm[level].cursor_en = is_enabled;
- active->wm[level].cursor_res_b =
+ active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
+ active->wm[level].plane_res_b[PLANE_CURSOR] =
val & PLANE_WM_BLOCKS_MASK;
- active->wm[level].cursor_res_l =
+ active->wm[level].plane_res_l[PLANE_CURSOR] =
(val >> PLANE_WM_LINES_SHIFT) &
PLANE_WM_LINES_MASK;
}
(val >> PLANE_WM_LINES_SHIFT) &
PLANE_WM_LINES_MASK;
} else {
- active->trans_wm.cursor_en = is_enabled;
- active->trans_wm.cursor_res_b =
+ active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
+ active->trans_wm.plane_res_b[PLANE_CURSOR] =
val & PLANE_WM_BLOCKS_MASK;
- active->trans_wm.cursor_res_l =
+ active->trans_wm.plane_res_l[PLANE_CURSOR] =
(val >> PLANE_WM_LINES_SHIFT) &
PLANE_WM_LINES_MASK;
}
for (i = 0; i < intel_num_planes(intel_crtc); i++)
hw->plane[pipe][i][level] =
I915_READ(PLANE_WM(pipe, i, level));
- hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
+ hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
}
for (i = 0; i < intel_num_planes(intel_crtc); i++)
hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
- hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
+ hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
if (!intel_crtc->active)
return;
skl_pipe_wm_active_state(temp, active, false,
false, i, level);
}
- temp = hw->cursor[pipe][level];
+ temp = hw->plane[pipe][PLANE_CURSOR][level];
skl_pipe_wm_active_state(temp, active, false, true, i, level);
}
skl_pipe_wm_active_state(temp, active, true, false, i, 0);
}
- temp = hw->cursor_trans[pipe];
+ temp = hw->plane_trans[pipe][PLANE_CURSOR];
skl_pipe_wm_active_state(temp, active, true, true, i, 0);
}
fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
MEMMODE_FSTART_SHIFT;
- vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
+ vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
PXVFREQ_PX_SHIFT;
dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
ironlake_set_drps(dev, fstart);
- dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
- I915_READ(0x112e0);
+ dev_priv->ips.last_count1 = I915_READ(DMIEC) +
+ I915_READ(DDREC) + I915_READ(CSIEC);
dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
- dev_priv->ips.last_count2 = I915_READ(0x112f4);
+ dev_priv->ips.last_count2 = I915_READ(GFXEC);
dev_priv->ips.last_time2 = ktime_get_raw_ns();
spin_unlock_irq(&mchdev_lock);
{
struct drm_i915_private *dev_priv = dev->dev_private;
+ /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
+ if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0))
+ return;
+
WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
WARN_ON(val > dev_priv->rps.max_freq);
WARN_ON(val < dev_priv->rps.min_freq);
gen6_init_rps_frequencies(dev);
+ /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
+ if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) {
+ intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+ return;
+ }
+
/* Program defaults and thresholds for RPS*/
I915_WRITE(GEN6_RC_VIDEO_FREQ,
GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
I915_WRITE(GEN6_RC_CONTROL, 0);
/* 2b: Program RC6 thresholds.*/
- I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
+
+ /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
+ if (IS_SKYLAKE(dev) && !((IS_SKL_GT3(dev) || IS_SKL_GT4(dev)) &&
+ (INTEL_REVID(dev) <= SKL_REVID_E0)))
+ I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
+ else
+ I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_ring(ring, dev_priv, unused)
I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
+
+ if (HAS_GUC_UCODE(dev))
+ I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
+
I915_WRITE(GEN6_RC_SLEEP, 0);
- I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
/* 2c: Program Coarse Power Gating Policies. */
I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
"on" : "off");
- I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
- GEN6_RC_CTL_EI_MODE(1) |
- rc6_mask);
+ /* WaRsUseTimeoutMode */
+ if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_D0) ||
+ (IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_A0)) {
+ I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
+ I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
+ GEN7_RC_CTL_TO_MODE |
+ rc6_mask);
+ } else {
+ I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
+ I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
+ GEN6_RC_CTL_EI_MODE(1) |
+ rc6_mask);
+ }
/*
* 3b: Enable Coarse Power Gating only when RC6 is enabled.
- * WaDisableRenderPowerGating:skl,bxt - Render PG need to be disabled with RC6.
+ * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
*/
- I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
- GEN9_MEDIA_PG_ENABLE : 0);
-
+ if ((IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) ||
+ ((IS_SKL_GT3(dev) || IS_SKL_GT4(dev)) && (INTEL_REVID(dev) <= SKL_REVID_E0)))
+ I915_WRITE(GEN9_PG_ENABLE, 0);
+ else
+ I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
+ (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0);
intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
struct drm_device *dev = dev_priv->dev;
u32 val, rp0;
- if (dev->pdev->revision >= 0x20) {
- val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
+ val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
- switch (INTEL_INFO(dev)->eu_total) {
- case 8:
- /* (2 * 4) config */
- rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
- break;
- case 12:
- /* (2 * 6) config */
- rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
- break;
- case 16:
- /* (2 * 8) config */
- default:
- /* Setting (2 * 8) Min RP0 for any other combination */
- rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
- break;
- }
- rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
- } else {
- /* For pre-production hardware */
- val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
- rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
- PUNIT_GPU_STATUS_MAX_FREQ_MASK;
+ switch (INTEL_INFO(dev)->eu_total) {
+ case 8:
+ /* (2 * 4) config */
+ rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
+ break;
+ case 12:
+ /* (2 * 6) config */
+ rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
+ break;
+ case 16:
+ /* (2 * 8) config */
+ default:
+ /* Setting (2 * 8) Min RP0 for any other combination */
+ rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
+ break;
}
+
+ rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
+
return rp0;
}
static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
{
- struct drm_device *dev = dev_priv->dev;
u32 val, rp1;
- if (dev->pdev->revision >= 0x20) {
- val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
- rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
- } else {
- /* For pre-production hardware */
- val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
- rp1 = ((val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
- PUNIT_GPU_STATUS_MAX_FREQ_MASK);
- }
+ val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
+ rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
+
return rp1;
}
mutex_unlock(&dev_priv->sb_lock);
switch ((val >> 2) & 0x7) {
- case 0:
- case 1:
- dev_priv->rps.cz_freq = 200;
- dev_priv->mem_freq = 1600;
- break;
- case 2:
- dev_priv->rps.cz_freq = 267;
- dev_priv->mem_freq = 1600;
- break;
case 3:
- dev_priv->rps.cz_freq = 333;
dev_priv->mem_freq = 2000;
break;
- case 4:
- dev_priv->rps.cz_freq = 320;
- dev_priv->mem_freq = 1600;
- break;
- case 5:
- dev_priv->rps.cz_freq = 400;
+ default:
dev_priv->mem_freq = 1600;
break;
}
/* RPS code assumes GPLL is used */
WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
- DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
+ DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
dev_priv->rps.cur_freq = (val >> 8) & 0xff;
/* RPS code assumes GPLL is used */
WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
- DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
+ DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
dev_priv->rps.cur_freq = (val >> 8) & 0xff;
assert_spin_locked(&mchdev_lock);
- pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
+ pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
pxvid = (pxvid >> 24) & 0x7f;
ext_v = pvid_to_extvid(dev_priv, pxvid);
I915_WRITE(CSIEW2, 0x04000004);
for (i = 0; i < 5; i++)
- I915_WRITE(PEW + (i * 4), 0);
+ I915_WRITE(PEW(i), 0);
for (i = 0; i < 3; i++)
- I915_WRITE(DEW + (i * 4), 0);
+ I915_WRITE(DEW(i), 0);
/* Program P-state weights to account for frequency power adjustment */
for (i = 0; i < 16; i++) {
- u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
+ u32 pxvidfreq = I915_READ(PXVFREQ(i));
unsigned long freq = intel_pxfreq(pxvidfreq);
unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
PXVFREQ_PX_SHIFT;
for (i = 0; i < 4; i++) {
u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
(pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
- I915_WRITE(PXW + (i * 4), val);
+ I915_WRITE(PXW(i), val);
}
/* Adjust magic regs to magic values (more experimental results) */
I915_WRITE(EG7, 0);
for (i = 0; i < 8; i++)
- I915_WRITE(PXWL + (i * 4), 0);
+ I915_WRITE(PXWL(i), 0);
/* Enable PMON + select events */
I915_WRITE(ECR, 0x80000019);
* TODO: this bit should only be enabled when really needed, then
* disabled when not needed anymore in order to save power.
*/
- if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
+ if (HAS_PCH_LPT_LP(dev))
I915_WRITE(SOUTH_DSPCLK_GATE_D,
I915_READ(SOUTH_DSPCLK_GATE_D) |
PCH_LP_PARTITION_LEVEL_DISABLE);
/* WADPOClockGatingDisable:hsw */
- I915_WRITE(_TRANSA_CHICKEN1,
- I915_READ(_TRANSA_CHICKEN1) |
+ I915_WRITE(TRANS_CHICKEN1(PIPE_A),
+ I915_READ(TRANS_CHICKEN1(PIPE_A)) |
TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
}
{
struct drm_i915_private *dev_priv = dev->dev_private;
- if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
+ if (HAS_PCH_LPT_LP(dev)) {
uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
if (IS_BROXTON(dev))
dev_priv->display.init_clock_gating =
bxt_init_clock_gating;
- else if (IS_SKYLAKE(dev))
- dev_priv->display.init_clock_gating =
- skl_init_clock_gating;
dev_priv->display.update_wm = skl_update_wm;
dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
} else if (HAS_PCH_SPLIT(dev)) {
static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
{
- int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
+ int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
div = vlv_gpu_freq_div(czclk_freq);
if (div < 0)
static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
{
- int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
+ int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
mul = vlv_gpu_freq_div(czclk_freq);
if (mul < 0)
static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
{
- int div, czclk_freq = dev_priv->rps.cz_freq;
+ int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
div = vlv_gpu_freq_div(czclk_freq) / 2;
if (div < 0)
static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
{
- int mul, czclk_freq = dev_priv->rps.cz_freq;
+ int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
mul = vlv_gpu_freq_div(czclk_freq) / 2;
if (mul < 0)
}
static void intel_psr_write_vsc(struct intel_dp *intel_dp,
- struct edp_vsc_psr *vsc_psr)
+ const struct edp_vsc_psr *vsc_psr)
{
struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
struct drm_device *dev = dig_port->base.base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc);
- u32 ctl_reg = HSW_TVIDEO_DIP_CTL(crtc->config->cpu_transcoder);
- u32 data_reg = HSW_TVIDEO_DIP_VSC_DATA(crtc->config->cpu_transcoder);
+ enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
+ u32 ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
uint32_t *data = (uint32_t *) vsc_psr;
unsigned int i;
I915_WRITE(ctl_reg, 0);
POSTING_READ(ctl_reg);
- for (i = 0; i < VIDEO_DIP_VSC_DATA_SIZE; i += 4) {
- if (i < sizeof(struct edp_vsc_psr))
- I915_WRITE(data_reg + i, *data++);
- else
- I915_WRITE(data_reg + i, 0);
+ for (i = 0; i < sizeof(*vsc_psr); i += 4) {
+ I915_WRITE(HSW_TVIDEO_DIP_VSC_DATA(cpu_transcoder,
+ i >> 2), *data);
+ data++;
}
+ for (; i < VIDEO_DIP_VSC_DATA_SIZE; i += 4)
+ I915_WRITE(HSW_TVIDEO_DIP_VSC_DATA(cpu_transcoder,
+ i >> 2), 0);
I915_WRITE(ctl_reg, VIDEO_DIP_ENABLE_VSC_HSW);
POSTING_READ(ctl_reg);
struct drm_i915_private *dev_priv = dev->dev_private;
struct i915_workarounds *w = &dev_priv->workarounds;
- if (WARN_ON_ONCE(w->count == 0))
+ if (w->count == 0)
return 0;
ring->gpu_caches_dirty = true;
#define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
-static int bdw_init_workarounds(struct intel_engine_cs *ring)
+static int gen8_init_workarounds(struct intel_engine_cs *ring)
{
struct drm_device *dev = ring->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
- /* WaDisableAsyncFlipPerfMode:bdw */
+ /* WaDisableAsyncFlipPerfMode:bdw,chv */
WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
- /* WaDisablePartialInstShootdown:bdw */
- /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
+ /* WaDisablePartialInstShootdown:bdw,chv */
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
- PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE |
- STALL_DOP_GATING_DISABLE);
-
- /* WaDisableDopClockGating:bdw */
- WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
- DOP_CLOCK_GATING_DISABLE);
-
- WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
- GEN8_SAMPLER_POWER_BYPASS_DIS);
+ PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
/* Use Force Non-Coherent whenever executing a 3D context. This is a
* workaround for for a possible hang in the unlikely event a TLB
* invalidation occurs during a PSD flush.
*/
+ /* WaForceEnableNonCoherent:bdw,chv */
+ /* WaHdcDisableFetchWhenMasked:bdw,chv */
WA_SET_BIT_MASKED(HDC_CHICKEN0,
- /* WaForceEnableNonCoherent:bdw */
- HDC_FORCE_NON_COHERENT |
- /* WaForceContextSaveRestoreNonCoherent:bdw */
- HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
- /* WaHdcDisableFetchWhenMasked:bdw */
HDC_DONOT_FETCH_MEM_WHEN_MASKED |
- /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
- (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
+ HDC_FORCE_NON_COHERENT);
/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
* "The Hierarchical Z RAW Stall Optimization allows non-overlapping
* stalling waiting for the earlier ones to write to Hierarchical Z
* buffer."
*
- * This optimization is off by default for Broadwell; turn it on.
+ * This optimization is off by default for BDW and CHV; turn it on.
*/
WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
- /* Wa4x4STCOptimizationDisable:bdw */
- WA_SET_BIT_MASKED(CACHE_MODE_1,
- GEN8_4x4_STC_OPTIMIZATION_DISABLE);
+ /* Wa4x4STCOptimizationDisable:bdw,chv */
+ WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
/*
* BSpec recommends 8x4 when MSAA is used,
return 0;
}
-static int chv_init_workarounds(struct intel_engine_cs *ring)
+static int bdw_init_workarounds(struct intel_engine_cs *ring)
{
+ int ret;
struct drm_device *dev = ring->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
+ ret = gen8_init_workarounds(ring);
+ if (ret)
+ return ret;
- /* WaDisableAsyncFlipPerfMode:chv */
- WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
+ /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
+ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
- /* WaDisablePartialInstShootdown:chv */
- /* WaDisableThreadStallDopClockGating:chv */
- WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
- PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE |
- STALL_DOP_GATING_DISABLE);
+ /* WaDisableDopClockGating:bdw */
+ WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
+ DOP_CLOCK_GATING_DISABLE);
+
+ WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
+ GEN8_SAMPLER_POWER_BYPASS_DIS);
- /* Use Force Non-Coherent whenever executing a 3D context. This is a
- * workaround for a possible hang in the unlikely event a TLB
- * invalidation occurs during a PSD flush.
- */
- /* WaForceEnableNonCoherent:chv */
- /* WaHdcDisableFetchWhenMasked:chv */
WA_SET_BIT_MASKED(HDC_CHICKEN0,
- HDC_FORCE_NON_COHERENT |
- HDC_DONOT_FETCH_MEM_WHEN_MASKED);
+ /* WaForceContextSaveRestoreNonCoherent:bdw */
+ HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
+ /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
+ (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
- /* According to the CACHE_MODE_0 default value documentation, some
- * CHV platforms disable this optimization by default. Turn it on.
- */
- WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
+ return 0;
+}
- /* Wa4x4STCOptimizationDisable:chv */
- WA_SET_BIT_MASKED(CACHE_MODE_1,
- GEN8_4x4_STC_OPTIMIZATION_DISABLE);
+static int chv_init_workarounds(struct intel_engine_cs *ring)
+{
+ int ret;
+ struct drm_device *dev = ring->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ ret = gen8_init_workarounds(ring);
+ if (ret)
+ return ret;
+
+ /* WaDisableThreadStallDopClockGating:chv */
+ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
/* Improve HiZ throughput on CHV. */
WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
- /*
- * BSpec recommends 8x4 when MSAA is used,
- * however in practice 16x4 seems fastest.
- *
- * Note that PS/WM thread counts depend on the WIZ hashing
- * disable bit, which we don't touch here, but it's good
- * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
- */
- WA_SET_FIELD_MASKED(GEN7_GT_MODE,
- GEN6_WIZ_HASHING_MASK,
- GEN6_WIZ_HASHING_16x4);
-
return 0;
}
struct drm_i915_private *dev_priv = dev->dev_private;
uint32_t tmp;
+ /* WaEnableLbsSlaRetryTimerDecrement:skl */
+ I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
+ GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
+
+ /* WaDisableKillLogic:bxt,skl */
+ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
+ ECOCHK_DIS_TLB);
+
/* WaDisablePartialInstShootdown:skl,bxt */
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
}
/* Wa4x4STCOptimizationDisable:skl,bxt */
- WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
-
/* WaDisablePartialResolveInVc:skl,bxt */
- WA_SET_BIT_MASKED(CACHE_MODE_1, GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
+ WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
+ GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
/* WaCcsTlbPrefetchDisable:skl,bxt */
WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
tmp |= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE;
WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp);
+ /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */
+ if (IS_SKYLAKE(dev) ||
+ (IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_B0)) {
+ WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
+ GEN8_SAMPLER_POWER_BYPASS_DIS);
+ }
+
+ /* WaDisableSTUnitPowerOptimization:skl,bxt */
+ WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
+
return 0;
}
return 0;
}
-
static int skl_init_workarounds(struct intel_engine_cs *ring)
{
+ int ret;
struct drm_device *dev = ring->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- gen9_init_workarounds(ring);
+ ret = gen9_init_workarounds(ring);
+ if (ret)
+ return ret;
+
+ if (INTEL_REVID(dev) <= SKL_REVID_D0) {
+ /* WaDisableHDCInvalidation:skl */
+ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
+ BDW_DISABLE_HDC_INVALIDATION);
+
+ /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
+ I915_WRITE(FF_SLICE_CS_CHICKEN2,
+ _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
+ }
+
+ /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
+ * involving this register should also be added to WA batch as required.
+ */
+ if (INTEL_REVID(dev) <= SKL_REVID_E0)
+ /* WaDisableLSQCROPERFforOCL:skl */
+ I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
+ GEN8_LQSC_RO_PERF_DIS);
+
+ /* WaEnableGapsTsvCreditFix:skl */
+ if (IS_SKYLAKE(dev) && (INTEL_REVID(dev) >= SKL_REVID_C0)) {
+ I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
+ GEN9_GAPS_TSV_CREDIT_DISABLE));
+ }
/* WaDisablePowerCompilerClockGating:skl */
if (INTEL_REVID(dev) == SKL_REVID_B0)
static int bxt_init_workarounds(struct intel_engine_cs *ring)
{
+ int ret;
struct drm_device *dev = ring->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- gen9_init_workarounds(ring);
+ ret = gen9_init_workarounds(ring);
+ if (ret)
+ return ret;
+
+ /* WaStoreMultiplePTEenable:bxt */
+ /* This is a requirement according to Hardware specification */
+ if (INTEL_REVID(dev) == BXT_REVID_A0)
+ I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
+
+ /* WaSetClckGatingDisableMedia:bxt */
+ if (INTEL_REVID(dev) == BXT_REVID_A0) {
+ I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
+ ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
+ }
/* WaDisableThreadStallDopClockGating:bxt */
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
return 0;
}
-void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
+static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
{
drm_gem_object_unreference(&ringbuf->obj->base);
ringbuf->obj = NULL;
}
-int intel_alloc_ringbuffer_obj(struct drm_device *dev,
- struct intel_ringbuffer *ringbuf)
+static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
+ struct intel_ringbuffer *ringbuf)
{
struct drm_i915_gem_object *obj;
return 0;
}
+struct intel_ringbuffer *
+intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
+{
+ struct intel_ringbuffer *ring;
+ int ret;
+
+ ring = kzalloc(sizeof(*ring), GFP_KERNEL);
+ if (ring == NULL)
+ return ERR_PTR(-ENOMEM);
+
+ ring->ring = engine;
+
+ ring->size = size;
+ /* Workaround an erratum on the i830 which causes a hang if
+ * the TAIL pointer points to within the last 2 cachelines
+ * of the buffer.
+ */
+ ring->effective_size = size;
+ if (IS_I830(engine->dev) || IS_845G(engine->dev))
+ ring->effective_size -= 2 * CACHELINE_BYTES;
+
+ ring->last_retired_head = -1;
+ intel_ring_update_space(ring);
+
+ ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
+ if (ret) {
+ DRM_ERROR("Failed to allocate ringbuffer %s: %d\n",
+ engine->name, ret);
+ kfree(ring);
+ return ERR_PTR(ret);
+ }
+
+ return ring;
+}
+
+void
+intel_ringbuffer_free(struct intel_ringbuffer *ring)
+{
+ intel_destroy_ringbuffer_obj(ring);
+ kfree(ring);
+}
+
static int intel_init_ring_buffer(struct drm_device *dev,
struct intel_engine_cs *ring)
{
WARN_ON(ring->buffer);
- ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
- if (!ringbuf)
- return -ENOMEM;
- ring->buffer = ringbuf;
-
ring->dev = dev;
INIT_LIST_HEAD(&ring->active_list);
INIT_LIST_HEAD(&ring->request_list);
INIT_LIST_HEAD(&ring->execlist_queue);
i915_gem_batch_pool_init(dev, &ring->batch_pool);
- ringbuf->size = 32 * PAGE_SIZE;
- ringbuf->ring = ring;
memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
init_waitqueue_head(&ring->irq_queue);
+ ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE);
+ if (IS_ERR(ringbuf))
+ return PTR_ERR(ringbuf);
+ ring->buffer = ringbuf;
+
if (I915_NEED_GFX_HWS(dev)) {
ret = init_status_page(ring);
if (ret)
goto error;
}
- WARN_ON(ringbuf->obj);
-
- ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
- if (ret) {
- DRM_ERROR("Failed to allocate ringbuffer %s: %d\n",
- ring->name, ret);
- goto error;
- }
-
ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
if (ret) {
DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
goto error;
}
- /* Workaround an erratum on the i830 which causes a hang if
- * the TAIL pointer points to within the last 2 cachelines
- * of the buffer.
- */
- ringbuf->effective_size = ringbuf->size;
- if (IS_I830(dev) || IS_845G(dev))
- ringbuf->effective_size -= 2 * CACHELINE_BYTES;
-
ret = i915_cmd_parser_init_ring(ring);
if (ret)
goto error;
return 0;
error:
- kfree(ringbuf);
+ intel_ringbuffer_free(ringbuf);
ring->buffer = NULL;
return ret;
}
void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
{
struct drm_i915_private *dev_priv;
- struct intel_ringbuffer *ringbuf;
if (!intel_ring_initialized(ring))
return;
dev_priv = to_i915(ring->dev);
- ringbuf = ring->buffer;
intel_stop_ring_buffer(ring);
WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
- intel_unpin_ringbuffer_obj(ringbuf);
- intel_destroy_ringbuffer_obj(ringbuf);
+ intel_unpin_ringbuffer_obj(ring->buffer);
+ intel_ringbuffer_free(ring->buffer);
+ ring->buffer = NULL;
if (ring->cleanup)
ring->cleanup(ring);
i915_cmd_parser_fini_ring(ring);
i915_gem_batch_pool_fini(&ring->batch_pool);
-
- kfree(ringbuf);
- ring->buffer = NULL;
}
static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
GEN8_RING_SEMAPHORE_INIT;
}
} else if (INTEL_INFO(dev)->gen >= 6) {
+ ring->init_context = intel_rcs_ctx_init;
ring->add_request = gen6_add_request;
ring->flush = gen7_render_ring_flush;
if (INTEL_INFO(dev)->gen == 6)
return idx;
}
+static inline void
+intel_flush_status_page(struct intel_engine_cs *ring, int reg)
+{
+ drm_clflush_virt_range(&ring->status_page.page_addr[reg],
+ sizeof(uint32_t));
+}
+
static inline u32
intel_read_status_page(struct intel_engine_cs *ring,
int reg)
#define I915_GEM_HWS_SCRATCH_INDEX 0x40
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
-void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
+struct intel_ringbuffer *
+intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size);
int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
struct intel_ringbuffer *ringbuf);
-void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
-int intel_alloc_ringbuffer_obj(struct drm_device *dev,
- struct intel_ringbuffer *ringbuf);
+void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
+void intel_ringbuffer_free(struct intel_ringbuffer *ring);
void intel_stop_ring_buffer(struct intel_engine_cs *ring);
void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
}
if (power_well->data == SKL_DISP_PW_1) {
- intel_prepare_ddi(dev);
+ if (!dev_priv->power_domains.initializing)
+ intel_prepare_ddi(dev);
gen8_irq_power_well_post_enable(dev_priv, 1 << PIPE_A);
}
}
bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
SKL_DISP_PW_2);
- WARN(!IS_SKYLAKE(dev), "Platform doesn't support DC5.\n");
- WARN(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n");
- WARN(pg2_enabled, "PG2 not disabled to enable DC5.\n");
+ WARN_ONCE(!IS_SKYLAKE(dev), "Platform doesn't support DC5.\n");
+ WARN_ONCE(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n");
+ WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
- WARN((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
- "DC5 already programmed to be enabled.\n");
- WARN(dev_priv->pm.suspended,
- "DC5 cannot be enabled, if platform is runtime-suspended.\n");
+ WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
+ "DC5 already programmed to be enabled.\n");
+ WARN_ONCE(dev_priv->pm.suspended,
+ "DC5 cannot be enabled, if platform is runtime-suspended.\n");
assert_csr_loaded(dev_priv);
}
if (dev_priv->power_domains.initializing)
return;
- WARN(!pg2_enabled, "PG2 not enabled to disable DC5.\n");
- WARN(dev_priv->pm.suspended,
+ WARN_ONCE(!pg2_enabled, "PG2 not enabled to disable DC5.\n");
+ WARN_ONCE(dev_priv->pm.suspended,
"Disabling of DC5 while platform is runtime-suspended should never happen.\n");
}
{
struct drm_device *dev = dev_priv->dev;
- WARN(!IS_SKYLAKE(dev), "Platform doesn't support DC6.\n");
- WARN(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n");
- WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
- "Backlight is not disabled.\n");
- WARN((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
- "DC6 already programmed to be enabled.\n");
+ WARN_ONCE(!IS_SKYLAKE(dev), "Platform doesn't support DC6.\n");
+ WARN_ONCE(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n");
+ WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
+ "Backlight is not disabled.\n");
+ WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
+ "DC6 already programmed to be enabled.\n");
assert_csr_loaded(dev_priv);
}
return;
assert_csr_loaded(dev_priv);
- WARN(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
- "DC6 already programmed to be disabled.\n");
+ WARN_ONCE(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
+ "DC6 already programmed to be disabled.\n");
}
static void skl_enable_dc6(struct drm_i915_private *dev_priv)
}
} else {
if (enable_requested) {
- I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask);
- POSTING_READ(HSW_PWR_WELL_DRIVER);
- DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
+ if (IS_SKYLAKE(dev) &&
+ (power_well->data == SKL_DISP_PW_1) &&
+ (intel_csr_load_status_get(dev_priv) == FW_LOADED))
+ DRM_DEBUG_KMS("Not Disabling PW1, dmc will handle\n");
+ else {
+ I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask);
+ POSTING_READ(HSW_PWR_WELL_DRIVER);
+ DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
+ }
if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) &&
power_well->data == SKL_DISP_PW_2) {
wait_for((state = intel_csr_load_status_get(dev_priv)) !=
FW_UNINITIALIZED, 1000);
if (state != FW_LOADED)
- DRM_ERROR("CSR firmware not ready (%d)\n",
+ DRM_DEBUG("CSR firmware not ready (%d)\n",
state);
else
if (SKL_ENABLE_DC6(dev))
static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
{
+ enum pipe pipe;
+
+ /*
+ * Enable the CRI clock source so we can get at the
+ * display and the reference clock for VGA
+ * hotplug / manual detection. Supposedly DSI also
+ * needs the ref clock up and running.
+ *
+ * CHV DPLL B/C have some issues if VGA mode is enabled.
+ */
+ for_each_pipe(dev_priv->dev, pipe) {
+ u32 val = I915_READ(DPLL(pipe));
+
+ val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
+ if (pipe != PIPE_A)
+ val |= DPLL_INTEGRATED_CRI_CLK_VLV;
+
+ I915_WRITE(DPLL(pipe), val);
+ }
spin_lock_irq(&dev_priv->irq_lock);
valleyview_enable_display_irqs(dev_priv);
{
WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
- /*
- * Enable the CRI clock source so we can get at the
- * display and the reference clock for VGA
- * hotplug / manual detection.
- */
- I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | DPLL_VGA_MODE_DIS |
- DPLL_REF_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
+ /* since ref/cri clock was enabled */
udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
vlv_set_power_well(dev_priv, power_well, true);
vlv_set_power_well(dev_priv, power_well, false);
}
+#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
+
+static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
+ int power_well_id)
+{
+ struct i915_power_domains *power_domains = &dev_priv->power_domains;
+ struct i915_power_well *power_well;
+ int i;
+
+ for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
+ if (power_well->data == power_well_id)
+ return power_well;
+ }
+
+ return NULL;
+}
+
+#define BITS_SET(val, bits) (((val) & (bits)) == (bits))
+
+static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
+{
+ struct i915_power_well *cmn_bc =
+ lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
+ struct i915_power_well *cmn_d =
+ lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
+ u32 phy_control = dev_priv->chv_phy_control;
+ u32 phy_status = 0;
+ u32 phy_status_mask = 0xffffffff;
+ u32 tmp;
+
+ /*
+ * The BIOS can leave the PHY is some weird state
+ * where it doesn't fully power down some parts.
+ * Disable the asserts until the PHY has been fully
+ * reset (ie. the power well has been disabled at
+ * least once).
+ */
+ if (!dev_priv->chv_phy_assert[DPIO_PHY0])
+ phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
+ PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
+ PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
+ PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
+ PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
+ PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
+
+ if (!dev_priv->chv_phy_assert[DPIO_PHY1])
+ phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
+ PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
+ PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
+
+ if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
+ phy_status |= PHY_POWERGOOD(DPIO_PHY0);
+
+ /* this assumes override is only used to enable lanes */
+ if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
+ phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
+
+ if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
+ phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
+
+ /* CL1 is on whenever anything is on in either channel */
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
+ PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
+ phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
+
+ /*
+ * The DPLLB check accounts for the pipe B + port A usage
+ * with CL2 powered up but all the lanes in the second channel
+ * powered down.
+ */
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
+ (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
+ phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
+
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
+ phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
+ phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
+
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
+ phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
+ phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
+ }
+
+ if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
+ phy_status |= PHY_POWERGOOD(DPIO_PHY1);
+
+ /* this assumes override is only used to enable lanes */
+ if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
+ phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
+
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
+ phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
+
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
+ phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
+ if (BITS_SET(phy_control,
+ PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
+ phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
+ }
+
+ phy_status &= phy_status_mask;
+
+ /*
+ * The PHY may be busy with some initial calibration and whatnot,
+ * so the power state can take a while to actually change.
+ */
+ if (wait_for((tmp = I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask) == phy_status, 10))
+ WARN(phy_status != tmp,
+ "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
+ tmp, phy_status, dev_priv->chv_phy_control);
+}
+
+#undef BITS_SET
+
static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
struct i915_power_well *power_well)
{
enum dpio_phy phy;
+ enum pipe pipe;
+ uint32_t tmp;
WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
- /*
- * Enable the CRI clock source so we can get at the
- * display and the reference clock for VGA
- * hotplug / manual detection.
- */
if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
+ pipe = PIPE_A;
phy = DPIO_PHY0;
- I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | DPLL_VGA_MODE_DIS |
- DPLL_REF_CLK_ENABLE_VLV);
- I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | DPLL_VGA_MODE_DIS |
- DPLL_REF_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
} else {
+ pipe = PIPE_C;
phy = DPIO_PHY1;
- I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) | DPLL_VGA_MODE_DIS |
- DPLL_REF_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
}
+
+ /* since ref/cri clock was enabled */
udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
vlv_set_power_well(dev_priv, power_well, true);
if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1))
DRM_ERROR("Display PHY %d is not power up\n", phy);
+ mutex_lock(&dev_priv->sb_lock);
+
+ /* Enable dynamic power down */
+ tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
+ tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
+ DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
+ vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
+
+ if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
+ tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
+ tmp |= DPIO_DYNPWRDOWNEN_CH1;
+ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
+ } else {
+ /*
+ * Force the non-existing CL2 off. BXT does this
+ * too, so maybe it saves some power even though
+ * CL2 doesn't exist?
+ */
+ tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
+ tmp |= DPIO_CL2_LDOFUSE_PWRENB;
+ vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
+ }
+
+ mutex_unlock(&dev_priv->sb_lock);
+
dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
+
+ DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
+ phy, dev_priv->chv_phy_control);
+
+ assert_chv_phy_status(dev_priv);
}
static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
vlv_set_power_well(dev_priv, power_well, false);
+
+ DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
+ phy, dev_priv->chv_phy_control);
+
+ /* PHY is fully reset now, so we can enable the PHY state asserts */
+ dev_priv->chv_phy_assert[phy] = true;
+
+ assert_chv_phy_status(dev_priv);
+}
+
+static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
+ enum dpio_channel ch, bool override, unsigned int mask)
+{
+ enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
+ u32 reg, val, expected, actual;
+
+ /*
+ * The BIOS can leave the PHY is some weird state
+ * where it doesn't fully power down some parts.
+ * Disable the asserts until the PHY has been fully
+ * reset (ie. the power well has been disabled at
+ * least once).
+ */
+ if (!dev_priv->chv_phy_assert[phy])
+ return;
+
+ if (ch == DPIO_CH0)
+ reg = _CHV_CMN_DW0_CH0;
+ else
+ reg = _CHV_CMN_DW6_CH1;
+
+ mutex_lock(&dev_priv->sb_lock);
+ val = vlv_dpio_read(dev_priv, pipe, reg);
+ mutex_unlock(&dev_priv->sb_lock);
+
+ /*
+ * This assumes !override is only used when the port is disabled.
+ * All lanes should power down even without the override when
+ * the port is disabled.
+ */
+ if (!override || mask == 0xf) {
+ expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
+ /*
+ * If CH1 common lane is not active anymore
+ * (eg. for pipe B DPLL) the entire channel will
+ * shut down, which causes the common lane registers
+ * to read as 0. That means we can't actually check
+ * the lane power down status bits, but as the entire
+ * register reads as 0 it's a good indication that the
+ * channel is indeed entirely powered down.
+ */
+ if (ch == DPIO_CH1 && val == 0)
+ expected = 0;
+ } else if (mask != 0x0) {
+ expected = DPIO_ANYDL_POWERDOWN;
+ } else {
+ expected = 0;
+ }
+
+ if (ch == DPIO_CH0)
+ actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
+ else
+ actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
+ actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
+
+ WARN(actual != expected,
+ "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
+ !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
+ !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
+ reg, val);
+}
+
+bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
+ enum dpio_channel ch, bool override)
+{
+ struct i915_power_domains *power_domains = &dev_priv->power_domains;
+ bool was_override;
+
+ mutex_lock(&power_domains->lock);
+
+ was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+
+ if (override == was_override)
+ goto out;
+
+ if (override)
+ dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+ else
+ dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+
+ I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
+
+ DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
+ phy, ch, dev_priv->chv_phy_control);
+
+ assert_chv_phy_status(dev_priv);
+
+out:
+ mutex_unlock(&power_domains->lock);
+
+ return was_override;
+}
+
+void chv_phy_powergate_lanes(struct intel_encoder *encoder,
+ bool override, unsigned int mask)
+{
+ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+ struct i915_power_domains *power_domains = &dev_priv->power_domains;
+ enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
+ enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
+
+ mutex_lock(&power_domains->lock);
+
+ dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
+ dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
+
+ if (override)
+ dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+ else
+ dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+
+ I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
+
+ DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
+ phy, ch, mask, dev_priv->chv_phy_control);
+
+ assert_chv_phy_status(dev_priv);
+
+ assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
+
+ mutex_unlock(&power_domains->lock);
}
static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
intel_runtime_pm_put(dev_priv);
}
-#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
-
#define HSW_ALWAYS_ON_POWER_DOMAINS ( \
BIT(POWER_DOMAIN_PIPE_A) | \
BIT(POWER_DOMAIN_TRANSCODER_EDP) | \
},
};
-static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
- int power_well_id)
-{
- struct i915_power_domains *power_domains = &dev_priv->power_domains;
- struct i915_power_well *power_well;
- int i;
-
- for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
- if (power_well->data == power_well_id)
- return power_well;
- }
-
- return NULL;
-}
-
bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
int power_well_id)
{
/* Make sure we're not suspended first. */
pm_runtime_get_sync(device);
- pm_runtime_disable(device);
}
/**
* DISPLAY_PHY_CONTROL can get corrupted if read. As a
* workaround never ever read DISPLAY_PHY_CONTROL, and
* instead maintain a shadow copy ourselves. Use the actual
- * power well state to reconstruct the expected initial
- * value.
+ * power well state and lane status to reconstruct the
+ * expected initial value.
*/
dev_priv->chv_phy_control =
PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
- PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY0, DPIO_CH0) |
- PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY0, DPIO_CH1) |
- PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY1, DPIO_CH0);
- if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc))
+ PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
+ PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
+ PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
+
+ /*
+ * If all lanes are disabled we leave the override disabled
+ * with all power down bits cleared to match the state we
+ * would use after disabling the port. Otherwise enable the
+ * override and set the lane powerdown bits accding to the
+ * current lane status.
+ */
+ if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
+ uint32_t status = I915_READ(DPLL(PIPE_A));
+ unsigned int mask;
+
+ mask = status & DPLL_PORTB_READY_MASK;
+ if (mask == 0xf)
+ mask = 0x0;
+ else
+ dev_priv->chv_phy_control |=
+ PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
+
+ dev_priv->chv_phy_control |=
+ PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
+
+ mask = (status & DPLL_PORTC_READY_MASK) >> 4;
+ if (mask == 0xf)
+ mask = 0x0;
+ else
+ dev_priv->chv_phy_control |=
+ PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
+
+ dev_priv->chv_phy_control |=
+ PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
+
dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
- if (cmn_d->ops->is_enabled(dev_priv, cmn_d))
+
+ dev_priv->chv_phy_assert[DPIO_PHY0] = false;
+ } else {
+ dev_priv->chv_phy_assert[DPIO_PHY0] = true;
+ }
+
+ if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
+ uint32_t status = I915_READ(DPIO_PHY_STATUS);
+ unsigned int mask;
+
+ mask = status & DPLL_PORTD_READY_MASK;
+
+ if (mask == 0xf)
+ mask = 0x0;
+ else
+ dev_priv->chv_phy_control |=
+ PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
+
+ dev_priv->chv_phy_control |=
+ PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
+
dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
+
+ dev_priv->chv_phy_assert[DPIO_PHY1] = false;
+ } else {
+ dev_priv->chv_phy_assert[DPIO_PHY1] = true;
+ }
+
+ I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
+
+ DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
+ dev_priv->chv_phy_control);
}
static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
power_domains->initializing = true;
if (IS_CHERRYVIEW(dev)) {
+ mutex_lock(&power_domains->lock);
chv_phy_control_init(dev_priv);
+ mutex_unlock(&power_domains->lock);
} else if (IS_VALLEYVIEW(dev)) {
mutex_lock(&power_domains->lock);
vlv_cmnlane_wa(dev_priv);
if (!HAS_RUNTIME_PM(dev))
return;
- pm_runtime_set_active(device);
-
/*
* RPM depends on RC6 to save restore the GT HW context, so make RC6 a
* requirement.
#define IS_DIGITAL(c) (c->output_flag & (SDVO_TMDS_MASK | SDVO_LVDS_MASK))
-static const char *tv_format_names[] = {
+static const char * const tv_format_names[] = {
"NTSC_M" , "NTSC_J" , "NTSC_443",
"PAL_B" , "PAL_D" , "PAL_G" ,
"PAL_H" , "PAL_I" , "PAL_M" ,
"SECAM_60"
};
-#define TV_FORMAT_NUM (sizeof(tv_format_names) / sizeof(*tv_format_names))
+#define TV_FORMAT_NUM ARRAY_SIZE(tv_format_names)
struct intel_sdvo {
struct intel_encoder base;
uint32_t color_range;
bool color_range_auto;
+ /**
+ * HDMI user specified aspect ratio
+ */
+ enum hdmi_picture_aspect aspect_ratio;
+
/**
* This is set if we're going to treat the device as TV-out.
*
DRM_DEBUG_KMS("%s: W: %02X %s\n", SDVO_NAME(intel_sdvo), cmd, buffer);
}
-static const char *cmd_status_names[] = {
+static const char * const cmd_status_names[] = {
"Power on",
"Success",
"Not supported",
return false;
}
-static int intel_sdvo_get_pixel_multiplier(struct drm_display_mode *mode)
+static int intel_sdvo_get_pixel_multiplier(const struct drm_display_mode *adjusted_mode)
{
- if (mode->clock >= 100000)
+ if (adjusted_mode->crtc_clock >= 100000)
return 1;
- else if (mode->clock >= 50000)
+ else if (adjusted_mode->crtc_clock >= 50000)
return 2;
else
return 4;
if (intel_sdvo->is_tv)
i9xx_adjust_sdvo_tv_clock(pipe_config);
+ /* Set user selected PAR to incoming mode's member */
+ if (intel_sdvo->is_hdmi)
+ adjusted_mode->picture_aspect_ratio = intel_sdvo->aspect_ratio;
+
return true;
}
struct drm_device *dev = intel_encoder->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *crtc = to_intel_crtc(intel_encoder->base.crtc);
- struct drm_display_mode *adjusted_mode =
- &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
struct drm_display_mode *mode = &crtc->config->base.mode;
struct intel_sdvo *intel_sdvo = to_sdvo(intel_encoder);
u32 sdvox;
goto done;
}
+ if (property == connector->dev->mode_config.aspect_ratio_property) {
+ switch (val) {
+ case DRM_MODE_PICTURE_ASPECT_NONE:
+ intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
+ break;
+ case DRM_MODE_PICTURE_ASPECT_4_3:
+ intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_4_3;
+ break;
+ case DRM_MODE_PICTURE_ASPECT_16_9:
+ intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_16_9;
+ break;
+ default:
+ return -EINVAL;
+ }
+ goto done;
+ }
+
#define CHECK_PROPERTY(name, NAME) \
if (intel_sdvo_connector->name == property) { \
if (intel_sdvo_connector->cur_##name == temp_value) return 0; \
*/
static void
intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv,
- struct intel_sdvo *sdvo, u32 reg)
+ struct intel_sdvo *sdvo)
{
struct sdvo_device_mapping *mapping;
static void
intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv,
- struct intel_sdvo *sdvo, u32 reg)
+ struct intel_sdvo *sdvo)
{
struct sdvo_device_mapping *mapping;
u8 pin;
intel_attach_broadcast_rgb_property(&connector->base.base);
intel_sdvo->color_range_auto = true;
}
+ intel_attach_aspect_ratio_property(&connector->base.base);
+ intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
}
static struct intel_sdvo_connector *intel_sdvo_connector_alloc(void)
intel_sdvo->sdvo_reg = sdvo_reg;
intel_sdvo->is_sdvob = is_sdvob;
intel_sdvo->slave_addr = intel_sdvo_get_slave_addr(dev, intel_sdvo) >> 1;
- intel_sdvo_select_i2c_bus(dev_priv, intel_sdvo, sdvo_reg);
+ intel_sdvo_select_i2c_bus(dev_priv, intel_sdvo);
if (!intel_sdvo_init_ddc_proxy(intel_sdvo, dev))
goto err_i2c_bus;
*/
intel_sdvo->base.cloneable = 0;
- intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo, sdvo_reg);
+ intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo);
/* Set the input timing to the screen. Assume always input 0. */
if (!intel_sdvo_set_target_input(intel_sdvo))
}
}
-static int usecs_to_scanlines(const struct drm_display_mode *mode, int usecs)
+static int usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
+ int usecs)
{
/* paranoia */
- if (!mode->crtc_htotal)
+ if (!adjusted_mode->crtc_htotal)
return 1;
- return DIV_ROUND_UP(usecs * mode->crtc_clock, 1000 * mode->crtc_htotal);
+ return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
+ 1000 * adjusted_mode->crtc_htotal);
}
/**
* avoid random delays. The value written to @start_vbl_count should be
* supplied to intel_pipe_update_end() for error checking.
*/
-void intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl_count)
+void intel_pipe_update_start(struct intel_crtc *crtc)
{
struct drm_device *dev = crtc->base.dev;
- const struct drm_display_mode *mode = &crtc->config->base.adjusted_mode;
+ const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
enum pipe pipe = crtc->pipe;
long timeout = msecs_to_jiffies_timeout(1);
int scanline, min, max, vblank_start;
wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
DEFINE_WAIT(wait);
- vblank_start = mode->crtc_vblank_start;
- if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+ vblank_start = adjusted_mode->crtc_vblank_start;
+ if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
vblank_start = DIV_ROUND_UP(vblank_start, 2);
/* FIXME needs to be calibrated sensibly */
- min = vblank_start - usecs_to_scanlines(mode, 100);
+ min = vblank_start - usecs_to_scanlines(adjusted_mode, 100);
max = vblank_start - 1;
local_irq_disable();
- *start_vbl_count = 0;
if (min <= 0 || max <= 0)
return;
if (WARN_ON(drm_crtc_vblank_get(&crtc->base)))
return;
- trace_i915_pipe_update_start(crtc, min, max);
+ crtc->debug.min_vbl = min;
+ crtc->debug.max_vbl = max;
+ trace_i915_pipe_update_start(crtc);
for (;;) {
/*
drm_crtc_vblank_put(&crtc->base);
- *start_vbl_count = dev->driver->get_vblank_counter(dev, pipe);
+ crtc->debug.scanline_start = scanline;
+ crtc->debug.start_vbl_time = ktime_get();
+ crtc->debug.start_vbl_count =
+ dev->driver->get_vblank_counter(dev, pipe);
- trace_i915_pipe_update_vblank_evaded(crtc, min, max, *start_vbl_count);
+ trace_i915_pipe_update_vblank_evaded(crtc);
}
/**
* re-enables interrupts and verifies the update was actually completed
* before a vblank using the value of @start_vbl_count.
*/
-void intel_pipe_update_end(struct intel_crtc *crtc, u32 start_vbl_count)
+void intel_pipe_update_end(struct intel_crtc *crtc)
{
struct drm_device *dev = crtc->base.dev;
enum pipe pipe = crtc->pipe;
+ int scanline_end = intel_get_crtc_scanline(crtc);
u32 end_vbl_count = dev->driver->get_vblank_counter(dev, pipe);
+ ktime_t end_vbl_time = ktime_get();
- trace_i915_pipe_update_end(crtc, end_vbl_count);
+ trace_i915_pipe_update_end(crtc, end_vbl_count, scanline_end);
local_irq_enable();
- if (start_vbl_count && start_vbl_count != end_vbl_count)
- DRM_ERROR("Atomic update failure on pipe %c (start=%u end=%u)\n",
- pipe_name(pipe), start_vbl_count, end_vbl_count);
+ if (crtc->debug.start_vbl_count &&
+ crtc->debug.start_vbl_count != end_vbl_count) {
+ DRM_ERROR("Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
+ pipe_name(pipe), crtc->debug.start_vbl_count,
+ end_vbl_count,
+ ktime_us_delta(end_vbl_time, crtc->debug.start_vbl_time),
+ crtc->debug.min_vbl, crtc->debug.max_vbl,
+ crtc->debug.scanline_start, scanline_end);
+ }
}
static void
int scaler_id;
plane_ctl = PLANE_CTL_ENABLE |
+ PLANE_CTL_PIPE_GAMMA_ENABLE |
PLANE_CTL_PIPE_CSC_ENABLE;
plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
else if (key->flags & I915_SET_COLORKEY_SOURCE)
plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
- surf_addr = intel_plane_obj_offset(intel_plane, obj);
+ surf_addr = intel_plane_obj_offset(intel_plane, obj, 0);
if (intel_rotation_90_or_270(rotation)) {
/* stride: Surface height in tiles */
tile_height = intel_tile_height(dev, fb->pixel_format,
- fb->modifier[0]);
+ fb->modifier[0], 0);
stride = DIV_ROUND_UP(fb->height, tile_height);
plane_size = (src_w << 16) | src_h;
x_offset = stride * tile_height - y - (src_h + 1);
struct intel_plane *intel_plane = to_intel_plane(plane);
int pipe = intel_plane->pipe;
- I915_WRITE(SPRCTL(pipe), I915_READ(SPRCTL(pipe)) & ~SPRITE_ENABLE);
+ I915_WRITE(SPRCTL(pipe), 0);
/* Can't leave the scaler enabled... */
if (intel_plane->can_scale)
I915_WRITE(SPRSCALE(pipe), 0);
crtc = crtc ? crtc : plane->crtc;
- plane->fb = fb;
-
if (!crtc->state->active)
return;
intel_plane->pipe = pipe;
intel_plane->plane = plane;
- intel_plane->frontbuffer_bit = INTEL_FRONTBUFFER_SPRITE(pipe);
+ intel_plane->frontbuffer_bit = INTEL_FRONTBUFFER_SPRITE(pipe, plane);
intel_plane->check_plane = intel_check_sprite_plane;
intel_plane->commit_plane = intel_commit_sprite_plane;
possible_crtcs = (1 << pipe);
j = 0;
for (i = 0; i < 60; i++)
- I915_WRITE(TV_H_LUMA_0 + (i<<2), tv_mode->filter_table[j++]);
+ I915_WRITE(TV_H_LUMA(i), tv_mode->filter_table[j++]);
for (i = 0; i < 60; i++)
- I915_WRITE(TV_H_CHROMA_0 + (i<<2), tv_mode->filter_table[j++]);
+ I915_WRITE(TV_H_CHROMA(i), tv_mode->filter_table[j++]);
for (i = 0; i < 43; i++)
- I915_WRITE(TV_V_LUMA_0 + (i<<2), tv_mode->filter_table[j++]);
+ I915_WRITE(TV_V_LUMA(i), tv_mode->filter_table[j++]);
for (i = 0; i < 43; i++)
- I915_WRITE(TV_V_CHROMA_0 + (i<<2), tv_mode->filter_table[j++]);
+ I915_WRITE(TV_V_CHROMA(i), tv_mode->filter_table[j++]);
I915_WRITE(TV_DAC, I915_READ(TV_DAC) & TV_DAC_SAVE);
I915_WRITE(TV_CTL, tv_ctl);
}
return;
- for (i = 0; i < sizeof(tv_modes) / sizeof(*tv_modes); i++) {
+ for (i = 0; i < ARRAY_SIZE(tv_modes); i++) {
tv_mode = tv_modes + i;
if ((intel_tv->type == DRM_MODE_CONNECTOR_Component) ==
struct intel_encoder *intel_encoder;
struct intel_connector *intel_connector;
u32 tv_dac_on, tv_dac_off, save_tv_dac;
- char *tv_format_names[ARRAY_SIZE(tv_modes)];
+ const char *tv_format_names[ARRAY_SIZE(tv_modes)];
int i, initial_mode = 0;
if ((I915_READ(TV_CTL) & TV_FUSE_STATE_MASK) == TV_FUSE_STATE_DISABLED)
/* Create TV properties then attach current values */
for (i = 0; i < ARRAY_SIZE(tv_modes); i++)
- tv_format_names[i] = (char *)tv_modes[i].name;
+ tv_format_names[i] = tv_modes[i].name;
drm_mode_create_tv_properties(dev,
ARRAY_SIZE(tv_modes),
tv_format_names);
#include <linux/pm_runtime.h>
-#define FORCEWAKE_ACK_TIMEOUT_MS 2
+#define FORCEWAKE_ACK_TIMEOUT_MS 50
#define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
#define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
const char *
intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
{
- BUILD_BUG_ON((sizeof(forcewake_domain_names)/sizeof(const char *)) !=
- FW_DOMAIN_ID_COUNT);
+ BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
return forcewake_domain_names[id];
}
/* We give fast paths for the really cool registers */
-#define NEEDS_FORCE_WAKE(dev_priv, reg) \
+#define NEEDS_FORCE_WAKE(reg) \
((reg) < 0x40000 && (reg) != FORCEWAKE)
#define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end))
gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
GEN6_READ_HEADER(x); \
hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
- if (NEEDS_FORCE_WAKE((dev_priv), (reg))) \
+ if (NEEDS_FORCE_WAKE(reg)) \
__force_wake_get(dev_priv, FORCEWAKE_RENDER); \
val = __raw_i915_read##x(dev_priv, reg); \
hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
GEN6_READ_FOOTER; \
}
-#define SKL_NEEDS_FORCE_WAKE(dev_priv, reg) \
+#define SKL_NEEDS_FORCE_WAKE(reg) \
((reg) < 0x40000 && !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg))
#define __gen9_read(x) \
gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
enum forcewake_domains fw_engine; \
GEN6_READ_HEADER(x); \
- if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) \
+ hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
+ if (!SKL_NEEDS_FORCE_WAKE(reg)) \
fw_engine = 0; \
- else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
+ else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
fw_engine = FORCEWAKE_RENDER; \
else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
fw_engine = FORCEWAKE_MEDIA; \
if (fw_engine) \
__force_wake_get(dev_priv, fw_engine); \
val = __raw_i915_read##x(dev_priv, reg); \
+ hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
GEN6_READ_FOOTER; \
}
gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
u32 __fifo_ret = 0; \
GEN6_WRITE_HEADER; \
- if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
+ if (NEEDS_FORCE_WAKE(reg)) { \
__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
} \
__raw_i915_write##x(dev_priv, reg, val); \
hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
u32 __fifo_ret = 0; \
GEN6_WRITE_HEADER; \
- if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
+ if (NEEDS_FORCE_WAKE(reg)) { \
__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
} \
hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
bool trace) { \
enum forcewake_domains fw_engine; \
GEN6_WRITE_HEADER; \
- if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \
+ hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
+ if (!SKL_NEEDS_FORCE_WAKE(reg) || \
is_gen9_shadowed(dev_priv, reg)) \
fw_engine = 0; \
else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
if (fw_engine) \
__force_wake_get(dev_priv, fw_engine); \
__raw_i915_write##x(dev_priv, reg, val); \
+ hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
+ hsw_unclaimed_reg_detect(dev_priv); \
GEN6_WRITE_FOOTER; \
}
switch (INTEL_INFO(dev)->gen) {
default:
- MISSING_CASE(INTEL_INFO(dev)->gen);
- return;
case 9:
ASSIGN_WRITE_MMIO_VFUNCS(gen9);
ASSIGN_READ_MMIO_VFUNCS(gen9);
struct drm_i915_private *dev_priv = dev->dev_private;
int ret;
- I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
+ I915_WRITE(ILK_GDSR,
ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
- ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
+ ret = wait_for((I915_READ(ILK_GDSR) &
ILK_GRDOM_RESET_ENABLE) == 0, 500);
if (ret)
return ret;
- I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
+ I915_WRITE(ILK_GDSR,
ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
- ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
+ ret = wait_for((I915_READ(ILK_GDSR) &
ILK_GRDOM_RESET_ENABLE) == 0, 500);
if (ret)
return ret;
- I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 0);
+ I915_WRITE(ILK_GDSR, 0);
return 0;
}
{ 0x40a2, 0x000a },
},
}, {
- ~0UL, {
+ 216000000, {
{ 0x00a0, 0x000a },
{ 0x2001, 0x000f },
{ 0x4002, 0x000f },
},
+ }, {
+ ~0UL, {
+ { 0x0000, 0x0000 },
+ { 0x0000, 0x0000 },
+ { 0x0000, 0x0000 },
+ },
}
};
*/
static const struct dw_hdmi_phy_config imx_phy_config[] = {
/*pixelclk symbol term vlev */
- { 148500000, 0x800d, 0x0005, 0x01ad},
+ { 216000000, 0x800d, 0x0005, 0x01ad},
{ ~0UL, 0x0000, 0x0000, 0x0000}
};
{
if (mode->clock < 13500)
return MODE_CLOCK_LOW;
- if (mode->clock > 266000)
+ /* FIXME: Hardware is capable of 266MHz, but setup data is missing. */
+ if (mode->clock > 216000)
return MODE_CLOCK_HIGH;
return MODE_OK;
{
if (mode->clock < 13500)
return MODE_CLOCK_LOW;
- if (mode->clock > 270000)
+ /* FIXME: Hardware is capable of 270MHz, but setup data is missing. */
+ if (mode->clock > 216000)
return MODE_CLOCK_HIGH;
return MODE_OK;
}
EXPORT_SYMBOL_GPL(imx_drm_handle_vblank);
-static int imx_drm_enable_vblank(struct drm_device *drm, int crtc)
+static int imx_drm_enable_vblank(struct drm_device *drm, unsigned int pipe)
{
struct imx_drm_device *imxdrm = drm->dev_private;
- struct imx_drm_crtc *imx_drm_crtc = imxdrm->crtc[crtc];
+ struct imx_drm_crtc *imx_drm_crtc = imxdrm->crtc[pipe];
int ret;
if (!imx_drm_crtc)
return ret;
}
-static void imx_drm_disable_vblank(struct drm_device *drm, int crtc)
+static void imx_drm_disable_vblank(struct drm_device *drm, unsigned int pipe)
{
struct imx_drm_device *imxdrm = drm->dev_private;
- struct imx_drm_crtc *imx_drm_crtc = imxdrm->crtc[crtc];
+ struct imx_drm_crtc *imx_drm_crtc = imxdrm->crtc[pipe];
if (!imx_drm_crtc)
return;
.gem_prime_vmap = drm_gem_cma_prime_vmap,
.gem_prime_vunmap = drm_gem_cma_prime_vunmap,
.gem_prime_mmap = drm_gem_cma_prime_mmap,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = imx_drm_enable_vblank,
.disable_vblank = imx_drm_disable_vblank,
.ioctls = imx_drm_ioctls,
static int imx_drm_platform_probe(struct platform_device *pdev)
{
- struct device_node *ep, *port, *remote;
- struct component_match *match = NULL;
- int ret;
- int i;
-
- /*
- * Bind the IPU display interface ports first, so that
- * imx_drm_encoder_parse_of called from encoder .bind callbacks
- * works as expected.
- */
- for (i = 0; ; i++) {
- port = of_parse_phandle(pdev->dev.of_node, "ports", i);
- if (!port)
- break;
-
- component_match_add(&pdev->dev, &match, compare_of, port);
- }
+ int ret = drm_of_component_probe(&pdev->dev, compare_of, &imx_drm_ops);
- if (i == 0) {
- dev_err(&pdev->dev, "missing 'ports' property\n");
- return -ENODEV;
- }
+ if (!ret)
+ ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- /* Then bind all encoders */
- for (i = 0; ; i++) {
- port = of_parse_phandle(pdev->dev.of_node, "ports", i);
- if (!port)
- break;
-
- for_each_child_of_node(port, ep) {
- remote = of_graph_get_remote_port_parent(ep);
- if (!remote || !of_device_is_available(remote)) {
- of_node_put(remote);
- continue;
- } else if (!of_device_is_available(remote->parent)) {
- dev_warn(&pdev->dev, "parent device of %s is not available\n",
- remote->full_name);
- of_node_put(remote);
- continue;
- }
-
- component_match_add(&pdev->dev, &match, compare_of,
- remote);
- of_node_put(remote);
- }
- of_node_put(port);
- }
-
- ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (ret)
- return ret;
-
- return component_master_add_with_match(&pdev->dev, &imx_drm_ops, match);
+ return ret;
}
static int imx_drm_platform_remove(struct platform_device *pdev)
#define to_ipu_plane(x) container_of(x, struct ipu_plane, base)
static const uint32_t ipu_plane_formats[] = {
+ DRM_FORMAT_ARGB1555,
DRM_FORMAT_XRGB1555,
+ DRM_FORMAT_ABGR1555,
DRM_FORMAT_XBGR1555,
+ DRM_FORMAT_RGBA5551,
+ DRM_FORMAT_BGRA5551,
+ DRM_FORMAT_ARGB4444,
DRM_FORMAT_ARGB8888,
DRM_FORMAT_XRGB8888,
DRM_FORMAT_ABGR8888,
DRM_FORMAT_XBGR8888,
+ DRM_FORMAT_RGBA8888,
+ DRM_FORMAT_RGBX8888,
+ DRM_FORMAT_BGRA8888,
+ DRM_FORMAT_BGRA8888,
DRM_FORMAT_YUYV,
DRM_FORMAT_YVYU,
DRM_FORMAT_YUV420,
ipu_dp_set_window_pos(ipu_plane->dp, crtc_x, crtc_y);
/* Enable local alpha on partial plane */
switch (fb->pixel_format) {
+ case DRM_FORMAT_ARGB1555:
+ case DRM_FORMAT_ABGR1555:
+ case DRM_FORMAT_RGBA5551:
+ case DRM_FORMAT_BGRA5551:
+ case DRM_FORMAT_ARGB4444:
case DRM_FORMAT_ARGB8888:
case DRM_FORMAT_ABGR8888:
+ case DRM_FORMAT_RGBA8888:
+ case DRM_FORMAT_BGRA8888:
ipu_dp_set_global_alpha(ipu_plane->dp, false, 0, false);
break;
default:
return 0;
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
/**
* Bootstrap the driver for AGP DMA.
*
drm_legacy_ioremapfree(dev->agp_buffer_map, dev);
if (dev_priv->used_new_dma_init) {
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->agp_handle != 0) {
struct drm_agp_binding unbind_req;
struct drm_agp_buffer free_req;
extern int mga_warp_init(drm_mga_private_t *dev_priv);
/* mga_irq.c */
-extern int mga_enable_vblank(struct drm_device *dev, int crtc);
-extern void mga_disable_vblank(struct drm_device *dev, int crtc);
-extern u32 mga_get_vblank_counter(struct drm_device *dev, int crtc);
+extern int mga_enable_vblank(struct drm_device *dev, unsigned int pipe);
+extern void mga_disable_vblank(struct drm_device *dev, unsigned int pipe);
+extern u32 mga_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
extern int mga_driver_fence_wait(struct drm_device *dev, unsigned int *sequence);
extern int mga_driver_vblank_wait(struct drm_device *dev, unsigned int *sequence);
extern irqreturn_t mga_driver_irq_handler(int irq, void *arg);
#include <drm/mga_drm.h>
#include "mga_drv.h"
-u32 mga_get_vblank_counter(struct drm_device *dev, int crtc)
+u32 mga_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
const drm_mga_private_t *const dev_priv =
(drm_mga_private_t *) dev->dev_private;
- if (crtc != 0)
+ if (pipe != 0)
return 0;
return atomic_read(&dev_priv->vbl_received);
return IRQ_NONE;
}
-int mga_enable_vblank(struct drm_device *dev, int crtc)
+int mga_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
- if (crtc != 0) {
- DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
- crtc);
+ if (pipe != 0) {
+ DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
+ pipe);
return 0;
}
}
-void mga_disable_vblank(struct drm_device *dev, int crtc)
+void mga_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
- if (crtc != 0) {
- DRM_ERROR("tried to disable vblank on non-existent crtc %d\n",
- crtc);
+ if (pipe != 0) {
+ DRM_ERROR("tried to disable vblank on non-existent crtc %u\n",
+ pipe);
}
/* Do *NOT* disable the vertical refresh interrupt. MGA doesn't have
sysram = vmalloc(size);
if (!sysram)
- return -ENOMEM;
+ goto err_sysram;
info = drm_fb_helper_alloc_fbi(helper);
- if (IS_ERR(info))
- return PTR_ERR(info);
+ if (IS_ERR(info)) {
+ ret = PTR_ERR(info);
+ goto err_alloc_fbi;
+ }
info->par = mfbdev;
ret = mgag200_framebuffer_init(dev, &mfbdev->mfb, &mode_cmd, gobj);
if (ret)
- return ret;
+ goto err_framebuffer_init;
mfbdev->sysram = sysram;
mfbdev->size = size;
DRM_DEBUG_KMS("allocated %dx%d\n",
fb->width, fb->height);
+
return 0;
+
+err_framebuffer_init:
+ drm_fb_helper_release_fbi(helper);
+err_alloc_fbi:
+ vfree(sysram);
+err_sysram:
+ drm_gem_object_unreference_unlocked(gobj);
+
+ return ret;
}
static int mga_fbdev_destroy(struct drm_device *dev,
ret = drm_fb_helper_init(mdev->dev, &mfbdev->helper,
mdev->num_crtc, MGAG200FB_CONN_LIMIT);
if (ret)
- return ret;
+ goto err_fb_helper;
ret = drm_fb_helper_single_add_all_connectors(&mfbdev->helper);
if (ret)
- goto fini;
+ goto err_fb_setup;
/* disable all the possible outputs/crtcs before entering KMS mode */
drm_helper_disable_unused_functions(mdev->dev);
ret = drm_fb_helper_initial_config(&mfbdev->helper, bpp_sel);
if (ret)
- goto fini;
+ goto err_fb_setup;
return 0;
-fini:
+err_fb_setup:
drm_fb_helper_fini(&mfbdev->helper);
+err_fb_helper:
+ mdev->mfbdev = NULL;
+
return ret;
}
}
r = mgag200_mm_init(mdev);
if (r)
- goto out;
+ goto err_mm;
drm_mode_config_init(dev);
dev->mode_config.funcs = (void *)&mga_mode_funcs;
r = mgag200_modeset_init(mdev);
if (r) {
dev_err(&dev->pdev->dev, "Fatal error during modeset init: %d\n", r);
- goto out;
+ goto err_modeset;
}
/* Make small buffers to store a hardware cursor (double buffered icon updates) */
&mdev->cursor.pixels_1);
mgag200_bo_create(dev, roundup(48*64, PAGE_SIZE), 0, 0,
&mdev->cursor.pixels_2);
- if (!mdev->cursor.pixels_2 || !mdev->cursor.pixels_1)
- goto cursor_nospace;
- mdev->cursor.pixels_current = mdev->cursor.pixels_1;
- mdev->cursor.pixels_prev = mdev->cursor.pixels_2;
- goto cursor_done;
- cursor_nospace:
- mdev->cursor.pixels_1 = NULL;
- mdev->cursor.pixels_2 = NULL;
- dev_warn(&dev->pdev->dev, "Could not allocate space for cursors. Not doing hardware cursors.\n");
- cursor_done:
-
-out:
- if (r)
- mgag200_driver_unload(dev);
+ if (!mdev->cursor.pixels_2 || !mdev->cursor.pixels_1) {
+ mdev->cursor.pixels_1 = NULL;
+ mdev->cursor.pixels_2 = NULL;
+ dev_warn(&dev->pdev->dev,
+ "Could not allocate space for cursors. Not doing hardware cursors.\n");
+ } else {
+ mdev->cursor.pixels_current = mdev->cursor.pixels_1;
+ mdev->cursor.pixels_prev = mdev->cursor.pixels_2;
+ }
+
+ return 0;
+
+err_modeset:
+ drm_mode_config_cleanup(dev);
+ mgag200_mm_fini(mdev);
+err_mm:
+ dev->dev_private = NULL;
+
return r;
}
help
DRM/KMS driver for MSM/snapdragon.
-config DRM_MSM_FBDEV
- bool "Enable legacy fbdev support for MSM modesetting driver"
- depends on DRM_MSM
- select DRM_KMS_FB_HELPER
- select FB_SYS_FILLRECT
- select FB_SYS_COPYAREA
- select FB_SYS_IMAGEBLIT
- select FB_SYS_FOPS
- default y
- help
- Choose this option if you have a need for the legacy fbdev
- support. Note that this support also provide the linux console
- support on top of the MSM modesetting driver.
-
config DRM_MSM_REGISTER_LOGGING
bool "MSM DRM register logging"
depends on DRM_MSM
msm_rd.o \
msm_ringbuffer.o
-msm-$(CONFIG_DRM_MSM_FBDEV) += msm_fbdev.o
+msm-$(CONFIG_DRM_FBDEV_EMULATION) += msm_fbdev.o
msm-$(CONFIG_COMMON_CLK) += mdp/mdp4/mdp4_lvds_pll.o
msm-$(CONFIG_DRM_MSM_DSI) += dsi/dsi.o \
git clone https://github.com/freedreno/envytools.git
The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 364 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10551 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67120 bytes, from 2015-08-14 23:22:03)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63785 bytes, from 2015-08-14 18:27:06)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00)
Copyright (C) 2013-2015 by the following authors:
- Rob Clark <robdclark@gmail.com> (robclark)
git clone https://github.com/freedreno/envytools.git
The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 364 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10551 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67120 bytes, from 2015-08-14 23:22:03)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63785 bytes, from 2015-08-14 18:27:06)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00)
Copyright (C) 2013-2015 by the following authors:
- Rob Clark <robdclark@gmail.com> (robclark)
enum a3xx_intp_mode {
SMOOTH = 0,
FLAT = 1,
+ ZERO = 2,
+ ONE = 3,
};
enum a3xx_repl_mode {
#define A3XX_GRAS_CL_CLIP_CNTL_VP_CLIP_CODE_IGNORE 0x00080000
#define A3XX_GRAS_CL_CLIP_CNTL_VP_XFORM_DISABLE 0x00100000
#define A3XX_GRAS_CL_CLIP_CNTL_PERSP_DIVISION_DISABLE 0x00200000
+#define A3XX_GRAS_CL_CLIP_CNTL_ZERO_GB_SCALE_Z 0x00400000
#define A3XX_GRAS_CL_CLIP_CNTL_ZCOORD 0x00800000
#define A3XX_GRAS_CL_CLIP_CNTL_WCOORD 0x01000000
#define A3XX_GRAS_CL_CLIP_CNTL_ZCLIP_DISABLE 0x02000000
+#define A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__MASK 0x1c000000
+#define A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__SHIFT 26
+static inline uint32_t A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES(uint32_t val)
+{
+ return ((val) << A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__SHIFT) & A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__MASK;
+}
#define REG_A3XX_GRAS_CL_GB_CLIP_ADJ 0x00002044
#define A3XX_GRAS_CL_GB_CLIP_ADJ_HORZ__MASK 0x000003ff
#define A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__SHIFT 0
static inline uint32_t A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL(float val)
{
- return ((((int32_t)(val * 16384.0))) << A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__SHIFT) & A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__MASK;
+ return ((((int32_t)(val * 1048576.0))) << A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__SHIFT) & A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__MASK;
}
#define REG_A3XX_GRAS_SU_POLY_OFFSET_OFFSET 0x0000206d
#define A3XX_RB_MODE_CONTROL_PACKER_TIMER_ENABLE 0x00010000
#define REG_A3XX_RB_RENDER_CONTROL 0x000020c1
+#define A3XX_RB_RENDER_CONTROL_DUAL_COLOR_IN_ENABLE 0x00000001
+#define A3XX_RB_RENDER_CONTROL_YUV_IN_ENABLE 0x00000002
+#define A3XX_RB_RENDER_CONTROL_COV_VALUE_INPUT_ENABLE 0x00000004
#define A3XX_RB_RENDER_CONTROL_FACENESS 0x00000008
#define A3XX_RB_RENDER_CONTROL_BIN_WIDTH__MASK 0x00000ff0
#define A3XX_RB_RENDER_CONTROL_BIN_WIDTH__SHIFT 4
#define A3XX_RB_RENDER_CONTROL_YCOORD 0x00008000
#define A3XX_RB_RENDER_CONTROL_ZCOORD 0x00010000
#define A3XX_RB_RENDER_CONTROL_WCOORD 0x00020000
+#define A3XX_RB_RENDER_CONTROL_I_CLAMP_ENABLE 0x00080000
+#define A3XX_RB_RENDER_CONTROL_COV_VALUE_OUTPUT_ENABLE 0x00100000
#define A3XX_RB_RENDER_CONTROL_ALPHA_TEST 0x00400000
#define A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__MASK 0x07000000
#define A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__SHIFT 24
{
return ((val) << A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__MASK;
}
+#define A3XX_RB_RENDER_CONTROL_ALPHA_TO_COVERAGE 0x40000000
+#define A3XX_RB_RENDER_CONTROL_ALPHA_TO_ONE 0x80000000
#define REG_A3XX_RB_MSAA_CONTROL 0x000020c2
#define A3XX_RB_MSAA_CONTROL_DISABLE 0x00000400
git clone https://github.com/freedreno/envytools.git
The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 364 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10551 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67120 bytes, from 2015-08-14 23:22:03)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63785 bytes, from 2015-08-14 18:27:06)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00)
Copyright (C) 2013-2015 by the following authors:
- Rob Clark <robdclark@gmail.com> (robclark)
TFMT4_8_UNORM = 4,
TFMT4_8_8_UNORM = 14,
TFMT4_8_8_8_8_UNORM = 28,
+ TFMT4_8_SNORM = 5,
TFMT4_8_8_SNORM = 15,
TFMT4_8_8_8_8_SNORM = 29,
+ TFMT4_8_UINT = 6,
TFMT4_8_8_UINT = 16,
TFMT4_8_8_8_8_UINT = 30,
+ TFMT4_8_SINT = 7,
TFMT4_8_8_SINT = 17,
TFMT4_8_8_8_8_SINT = 31,
TFMT4_16_UINT = 21,
A4XX_TEX_REPEAT = 0,
A4XX_TEX_CLAMP_TO_EDGE = 1,
A4XX_TEX_MIRROR_REPEAT = 2,
- A4XX_TEX_CLAMP_NONE = 3,
+ A4XX_TEX_CLAMP_TO_BORDER = 3,
+ A4XX_TEX_MIRROR_CLAMP = 4,
};
enum a4xx_tex_aniso {
git clone https://github.com/freedreno/envytools.git
The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 364 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10551 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67120 bytes, from 2015-08-14 23:22:03)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63785 bytes, from 2015-08-14 18:27:06)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00)
Copyright (C) 2013-2015 by the following authors:
- Rob Clark <robdclark@gmail.com> (robclark)
FACTOR_CONSTANT_ALPHA = 14,
FACTOR_ONE_MINUS_CONSTANT_ALPHA = 15,
FACTOR_SRC_ALPHA_SATURATE = 16,
+ FACTOR_SRC1_COLOR = 20,
+ FACTOR_ONE_MINUS_SRC1_COLOR = 21,
+ FACTOR_SRC1_ALPHA = 22,
+ FACTOR_ONE_MINUS_SRC1_ALPHA = 23,
};
enum adreno_rb_surface_endian {
git clone https://github.com/freedreno/envytools.git
The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 364 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10551 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55)
- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67120 bytes, from 2015-08-14 23:22:03)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63785 bytes, from 2015-08-14 18:27:06)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00)
Copyright (C) 2013-2015 by the following authors:
- Rob Clark <robdclark@gmail.com> (robclark)
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
#define REG_DSI_8x60_PHY_CAL_STATUS 0x000000fc
#define DSI_8x60_PHY_CAL_STATUS_CAL_BUSY 0x10000000
-static inline uint32_t REG_DSI_8960_LN(uint32_t i0) { return 0x00000300 + 0x40*i0; }
+static inline uint32_t REG_DSI_28nm_8960_PHY_LN(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-static inline uint32_t REG_DSI_8960_LN_CFG_0(uint32_t i0) { return 0x00000300 + 0x40*i0; }
+static inline uint32_t REG_DSI_28nm_8960_PHY_LN_CFG_0(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-static inline uint32_t REG_DSI_8960_LN_CFG_1(uint32_t i0) { return 0x00000304 + 0x40*i0; }
+static inline uint32_t REG_DSI_28nm_8960_PHY_LN_CFG_1(uint32_t i0) { return 0x00000004 + 0x40*i0; }
-static inline uint32_t REG_DSI_8960_LN_CFG_2(uint32_t i0) { return 0x00000308 + 0x40*i0; }
+static inline uint32_t REG_DSI_28nm_8960_PHY_LN_CFG_2(uint32_t i0) { return 0x00000008 + 0x40*i0; }
-static inline uint32_t REG_DSI_8960_LN_TEST_DATAPATH(uint32_t i0) { return 0x0000030c + 0x40*i0; }
+static inline uint32_t REG_DSI_28nm_8960_PHY_LN_TEST_DATAPATH(uint32_t i0) { return 0x0000000c + 0x40*i0; }
-static inline uint32_t REG_DSI_8960_LN_TEST_STR_0(uint32_t i0) { return 0x00000314 + 0x40*i0; }
+static inline uint32_t REG_DSI_28nm_8960_PHY_LN_TEST_STR_0(uint32_t i0) { return 0x00000014 + 0x40*i0; }
-static inline uint32_t REG_DSI_8960_LN_TEST_STR_1(uint32_t i0) { return 0x00000318 + 0x40*i0; }
+static inline uint32_t REG_DSI_28nm_8960_PHY_LN_TEST_STR_1(uint32_t i0) { return 0x00000018 + 0x40*i0; }
-#define REG_DSI_8960_PHY_LNCK_CFG_0 0x00000400
+#define REG_DSI_28nm_8960_PHY_LNCK_CFG_0 0x00000100
-#define REG_DSI_8960_PHY_LNCK_CFG_1 0x00000404
+#define REG_DSI_28nm_8960_PHY_LNCK_CFG_1 0x00000104
-#define REG_DSI_8960_PHY_LNCK_CFG_2 0x00000408
+#define REG_DSI_28nm_8960_PHY_LNCK_CFG_2 0x00000108
-#define REG_DSI_8960_PHY_LNCK_TEST_DATAPATH 0x0000040c
+#define REG_DSI_28nm_8960_PHY_LNCK_TEST_DATAPATH 0x0000010c
-#define REG_DSI_8960_PHY_LNCK_TEST_STR0 0x00000414
+#define REG_DSI_28nm_8960_PHY_LNCK_TEST_STR0 0x00000114
-#define REG_DSI_8960_PHY_LNCK_TEST_STR1 0x00000418
+#define REG_DSI_28nm_8960_PHY_LNCK_TEST_STR1 0x00000118
-#define REG_DSI_8960_PHY_TIMING_CTRL_0 0x00000440
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_0 0x00000140
+#define DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_1 0x00000144
+#define DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_2 0x00000148
+#define DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_3 0x0000014c
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_4 0x00000150
+#define DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_5 0x00000154
+#define DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_6 0x00000158
+#define DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_7 0x0000015c
+#define DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_8 0x00000160
+#define DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_9 0x00000164
+#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__MASK 0x00000007
+#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__MASK;
+}
+#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__MASK 0x00000070
+#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__SHIFT 4
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_10 0x00000168
+#define DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__MASK 0x00000007
+#define DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_11 0x0000016c
+#define DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK 0x000000ff
+#define DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT 0
+static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD(uint32_t val)
+{
+ return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK;
+}
+
+#define REG_DSI_28nm_8960_PHY_CTRL_0 0x00000170
+
+#define REG_DSI_28nm_8960_PHY_CTRL_1 0x00000174
+
+#define REG_DSI_28nm_8960_PHY_CTRL_2 0x00000178
+
+#define REG_DSI_28nm_8960_PHY_CTRL_3 0x0000017c
+
+#define REG_DSI_28nm_8960_PHY_STRENGTH_0 0x00000180
+
+#define REG_DSI_28nm_8960_PHY_STRENGTH_1 0x00000184
+
+#define REG_DSI_28nm_8960_PHY_STRENGTH_2 0x00000188
+
+#define REG_DSI_28nm_8960_PHY_BIST_CTRL_0 0x0000018c
+
+#define REG_DSI_28nm_8960_PHY_BIST_CTRL_1 0x00000190
+
+#define REG_DSI_28nm_8960_PHY_BIST_CTRL_2 0x00000194
+
+#define REG_DSI_28nm_8960_PHY_BIST_CTRL_3 0x00000198
+
+#define REG_DSI_28nm_8960_PHY_BIST_CTRL_4 0x0000019c
-#define REG_DSI_8960_PHY_TIMING_CTRL_1 0x00000444
+#define REG_DSI_28nm_8960_PHY_LDO_CTRL 0x000001b0
-#define REG_DSI_8960_PHY_TIMING_CTRL_2 0x00000448
+#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_0 0x00000000
-#define REG_DSI_8960_PHY_TIMING_CTRL_3 0x0000044c
+#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_1 0x00000004
-#define REG_DSI_8960_PHY_TIMING_CTRL_4 0x00000450
+#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_2 0x00000008
-#define REG_DSI_8960_PHY_TIMING_CTRL_5 0x00000454
+#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_3 0x0000000c
-#define REG_DSI_8960_PHY_TIMING_CTRL_6 0x00000458
+#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_4 0x00000010
-#define REG_DSI_8960_PHY_TIMING_CTRL_7 0x0000045c
+#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_5 0x00000014
-#define REG_DSI_8960_PHY_TIMING_CTRL_8 0x00000460
+#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CAL_PWR_CFG 0x00000018
-#define REG_DSI_8960_PHY_TIMING_CTRL_9 0x00000464
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_TRIGGER 0x00000028
-#define REG_DSI_8960_PHY_TIMING_CTRL_10 0x00000468
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_0 0x0000002c
-#define REG_DSI_8960_PHY_TIMING_CTRL_11 0x0000046c
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_1 0x00000030
-#define REG_DSI_8960_PHY_CTRL_0 0x00000470
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_2 0x00000034
-#define REG_DSI_8960_PHY_CTRL_1 0x00000474
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_0 0x00000038
-#define REG_DSI_8960_PHY_CTRL_2 0x00000478
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_1 0x0000003c
-#define REG_DSI_8960_PHY_CTRL_3 0x0000047c
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_2 0x00000040
-#define REG_DSI_8960_PHY_STRENGTH_0 0x00000480
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_3 0x00000044
-#define REG_DSI_8960_PHY_STRENGTH_1 0x00000484
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_4 0x00000048
-#define REG_DSI_8960_PHY_STRENGTH_2 0x00000488
+#define REG_DSI_28nm_8960_PHY_MISC_CAL_STATUS 0x00000050
+#define DSI_28nm_8960_PHY_MISC_CAL_STATUS_CAL_BUSY 0x00000010
-#define REG_DSI_8960_PHY_BIST_CTRL_0 0x0000048c
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_0 0x00000000
+#define DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE 0x00000001
-#define REG_DSI_8960_PHY_BIST_CTRL_1 0x00000490
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_1 0x00000004
-#define REG_DSI_8960_PHY_BIST_CTRL_2 0x00000494
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_2 0x00000008
-#define REG_DSI_8960_PHY_BIST_CTRL_3 0x00000498
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_3 0x0000000c
-#define REG_DSI_8960_PHY_BIST_CTRL_4 0x0000049c
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_4 0x00000010
-#define REG_DSI_8960_PHY_LDO_CTRL 0x000004b0
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_5 0x00000014
-#define REG_DSI_8960_PHY_REGULATOR_CTRL_0 0x00000500
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_6 0x00000018
-#define REG_DSI_8960_PHY_REGULATOR_CTRL_1 0x00000504
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_7 0x0000001c
-#define REG_DSI_8960_PHY_REGULATOR_CTRL_2 0x00000508
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_8 0x00000020
-#define REG_DSI_8960_PHY_REGULATOR_CTRL_3 0x0000050c
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_9 0x00000024
-#define REG_DSI_8960_PHY_REGULATOR_CTRL_4 0x00000510
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_10 0x00000028
-#define REG_DSI_8960_PHY_REGULATOR_CAL_PWR_CFG 0x00000518
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_11 0x0000002c
-#define REG_DSI_8960_PHY_CAL_HW_TRIGGER 0x00000528
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_12 0x00000030
-#define REG_DSI_8960_PHY_CAL_SW_CFG_0 0x0000052c
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_13 0x00000034
-#define REG_DSI_8960_PHY_CAL_SW_CFG_1 0x00000530
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_14 0x00000038
-#define REG_DSI_8960_PHY_CAL_SW_CFG_2 0x00000534
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_15 0x0000003c
-#define REG_DSI_8960_PHY_CAL_HW_CFG_0 0x00000538
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_16 0x00000040
-#define REG_DSI_8960_PHY_CAL_HW_CFG_1 0x0000053c
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_17 0x00000044
-#define REG_DSI_8960_PHY_CAL_HW_CFG_2 0x00000540
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_18 0x00000048
-#define REG_DSI_8960_PHY_CAL_HW_CFG_3 0x00000544
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_19 0x0000004c
-#define REG_DSI_8960_PHY_CAL_HW_CFG_4 0x00000548
+#define REG_DSI_28nm_8960_PHY_PLL_CTRL_20 0x00000050
-#define REG_DSI_8960_PHY_CAL_STATUS 0x00000550
-#define DSI_8960_PHY_CAL_STATUS_CAL_BUSY 0x00000010
+#define REG_DSI_28nm_8960_PHY_PLL_RDY 0x00000080
+#define DSI_28nm_8960_PHY_PLL_RDY_PLL_RDY 0x00000001
static inline uint32_t REG_DSI_28nm_PHY_LN(uint32_t i0) { return 0x00000000 + 0x40*i0; }
}
for (i = 0; i < num; i++) {
- if ((regs[i].min_voltage >= 0) && (regs[i].max_voltage >= 0)) {
+ if (regulator_can_change_voltage(s[i].consumer)) {
ret = regulator_set_voltage(s[i].consumer,
regs[i].min_voltage, regs[i].max_voltage);
if (ret < 0) {
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
}
for (i = 0; i < num; i++) {
- if ((regs[i].min_voltage >= 0) && (regs[i].max_voltage >= 0)) {
+ if (regulator_can_change_voltage(s[i].consumer)) {
ret = regulator_set_voltage(s[i].consumer,
regs[i].min_voltage, regs[i].max_voltage);
if (ret < 0) {
dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_1(i), 0);
dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_2(i), 0);
dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_3(i), 0);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_4(i), 0);
dsi_phy_write(base + REG_DSI_28nm_PHY_LN_TEST_DATAPATH(i), 0);
dsi_phy_write(base + REG_DSI_28nm_PHY_LN_DEBUG_SEL(i), 0);
dsi_phy_write(base + REG_DSI_28nm_PHY_LN_TEST_STR_0(i), 0x1);
dsi_phy_write(base + REG_DSI_28nm_PHY_LN_TEST_STR_1(i), 0x97);
}
- dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_4(0), 0);
- dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_4(1), 0x5);
- dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_4(2), 0xa);
- dsi_phy_write(base + REG_DSI_28nm_PHY_LN_CFG_4(3), 0xf);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_LNCK_CFG_4, 0);
dsi_phy_write(base + REG_DSI_28nm_PHY_LNCK_CFG_1, 0xc0);
dsi_phy_write(base + REG_DSI_28nm_PHY_LNCK_TEST_STR0, 0x1);
dsi_phy_write(base + REG_DSI_28nm_PHY_LNCK_TEST_STR1, 0xbb);
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
*/
-#define REG_SFPB_CFG 0x00000058
+enum sfpb_ahb_arb_master_port_en {
+ SFPB_MASTER_PORT_ENABLE = 3,
+ SFPB_MASTER_PORT_DISABLE = 0,
+};
+
+#define REG_SFPB_GPREG 0x00000058
+#define SFPB_GPREG_MASTER_PORT_EN__MASK 0x00001800
+#define SFPB_GPREG_MASTER_PORT_EN__SHIFT 11
+static inline uint32_t SFPB_GPREG_MASTER_PORT_EN(enum sfpb_ahb_arb_master_port_en val)
+{
+ return ((val) << SFPB_GPREG_MASTER_PORT_EN__SHIFT) & SFPB_GPREG_MASTER_PORT_EN__MASK;
+}
#endif /* SFPB_XML */
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
.item ## _names = item ##_names_ ## entry, \
.item ## _cnt = ARRAY_SIZE(item ## _names_ ## entry)
+static const char *pwr_reg_names_none[] = {};
+static const char *hpd_reg_names_none[] = {};
+
static struct hdmi_platform_config hdmi_tx_8660_config = {
.phy_init = hdmi_phy_8x60_init,
};
.hpd_freq = hpd_clk_freq_8x74,
};
-static const char *hpd_reg_names_8x94[] = {};
-
static struct hdmi_platform_config hdmi_tx_8994_config = {
.phy_init = NULL, /* nothing to do for this HDMI PHY 20nm */
HDMI_CFG(pwr_reg, 8x74),
- HDMI_CFG(hpd_reg, 8x94),
+ HDMI_CFG(hpd_reg, none),
+ HDMI_CFG(pwr_clk, 8x74),
+ HDMI_CFG(hpd_clk, 8x74),
+ .hpd_freq = hpd_clk_freq_8x74,
+};
+
+static struct hdmi_platform_config hdmi_tx_8996_config = {
+ .phy_init = NULL,
+ HDMI_CFG(pwr_reg, none),
+ HDMI_CFG(hpd_reg, none),
HDMI_CFG(pwr_clk, 8x74),
HDMI_CFG(hpd_clk, 8x74),
.hpd_freq = hpd_clk_freq_8x74,
};
static const struct of_device_id dt_match[] = {
+ { .compatible = "qcom,hdmi-tx-8996", .data = &hdmi_tx_8996_config },
{ .compatible = "qcom,hdmi-tx-8994", .data = &hdmi_tx_8994_config },
{ .compatible = "qcom,hdmi-tx-8084", .data = &hdmi_tx_8084_config },
{ .compatible = "qcom,hdmi-tx-8974", .data = &hdmi_tx_8974_config },
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
};
static int mdp4_plane_prepare_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
struct mdp4_kms *mdp4_kms = get_kms(plane);
+ struct drm_framebuffer *fb = new_state->fb;
+
+ if (!fb)
+ return 0;
DBG("%s: prepare: FB[%u]", mdp4_plane->name, fb->base.id);
return msm_framebuffer_prepare(fb, mdp4_kms->id);
}
static void mdp4_plane_cleanup_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *old_state)
{
struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
struct mdp4_kms *mdp4_kms = get_kms(plane);
+ struct drm_framebuffer *fb = old_state->fb;
+
+ if (!fb)
+ return;
DBG("%s: cleanup: FB[%u]", mdp4_plane->name, fb->base.id);
msm_framebuffer_cleanup(fb, mdp4_kms->id);
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
#define MDP5_PIPE_SRC_OP_MODE_IGC_ROM_1 0x00040000
#define MDP5_PIPE_SRC_OP_MODE_DEINTERLACE 0x00400000
#define MDP5_PIPE_SRC_OP_MODE_DEINTERLACE_ODD 0x00800000
+#define MDP5_PIPE_SRC_OP_MODE_SW_PIX_EXT_OVERRIDE 0x80000000
static inline uint32_t REG_MDP5_PIPE_SRC_CONSTANT_COLOR(enum mdp5_pipe i0) { return 0x0000003c + __offset_PIPE(i0); }
return ((val) << MDP5_PIPE_DECIMATION_HORZ__SHIFT) & MDP5_PIPE_DECIMATION_HORZ__MASK;
}
+static inline uint32_t __offset_SW_PIX_EXT(enum mdp_component_type idx)
+{
+ switch (idx) {
+ case COMP_0: return 0x00000100;
+ case COMP_1_2: return 0x00000110;
+ case COMP_3: return 0x00000120;
+ default: return INVALID_IDX(idx);
+ }
+}
+static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000000 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
+
+static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT_LR(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000000 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
+#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__MASK 0x000000ff
+#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__SHIFT 0
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT(uint32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__MASK;
+}
+#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__MASK 0x0000ff00
+#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__SHIFT 8
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF(int32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__MASK;
+}
+#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__MASK 0x00ff0000
+#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__SHIFT 16
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT(uint32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__MASK;
+}
+#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__MASK 0xff000000
+#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__SHIFT 24
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF(int32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__MASK;
+}
+
+static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT_TB(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000004 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
+#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__MASK 0x000000ff
+#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__SHIFT 0
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT(uint32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__MASK;
+}
+#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__MASK 0x0000ff00
+#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__SHIFT 8
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF(int32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__MASK;
+}
+#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__MASK 0x00ff0000
+#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__SHIFT 16
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT(uint32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__MASK;
+}
+#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__MASK 0xff000000
+#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__SHIFT 24
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF(int32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__MASK;
+}
+
+static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000008 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
+#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__MASK 0x0000ffff
+#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__SHIFT 0
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT(uint32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__MASK;
+}
+#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__MASK 0xffff0000
+#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__SHIFT 16
+static inline uint32_t MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM(uint32_t val)
+{
+ return ((val) << MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__SHIFT) & MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__MASK;
+}
+
static inline uint32_t REG_MDP5_PIPE_SCALE_CONFIG(enum mdp5_pipe i0) { return 0x00000204 + __offset_PIPE(i0); }
#define MDP5_PIPE_SCALE_CONFIG_SCALEX_EN 0x00000001
#define MDP5_PIPE_SCALE_CONFIG_SCALEY_EN 0x00000002
.mdp = {
.count = 1,
.base = { 0x00100 },
+ .caps = MDP_CAP_SMP |
+ 0,
},
.smp = {
.mmb_count = 22,
.mdp = {
.count = 1,
.base = { 0x00100 },
+ .caps = MDP_CAP_SMP |
+ 0,
},
.smp = {
.mmb_count = 22,
.mdp = {
.count = 1,
.base = { 0x00100 },
+ .caps = MDP_CAP_SMP |
+ 0,
},
.smp = {
.mmb_count = 44,
.mdp = {
.count = 1,
.base = { 0x01000 },
+ .caps = MDP_CAP_SMP |
+ 0,
},
.smp = {
.mmb_count = 8,
.mdp = {
.count = 1,
.base = { 0x01000 },
+ .caps = MDP_CAP_SMP |
+ 0,
},
.smp = {
.mmb_count = 44,
[3] = INTF_HDMI,
},
},
- .max_clk = 320000000,
+ .max_clk = 400000000,
+};
+
+const struct mdp5_cfg_hw msm8x96_config = {
+ .name = "msm8x96",
+ .mdp = {
+ .count = 1,
+ .base = { 0x01000 },
+ .caps = MDP_CAP_DSC |
+ MDP_CAP_CDM |
+ 0,
+ },
+ .ctl = {
+ .count = 5,
+ .base = { 0x02000, 0x02200, 0x02400, 0x02600, 0x02800 },
+ .flush_hw_mask = 0xf4ffffff,
+ },
+ .pipe_vig = {
+ .count = 4,
+ .base = { 0x05000, 0x07000, 0x09000, 0x0b000 },
+ .caps = MDP_PIPE_CAP_HFLIP |
+ MDP_PIPE_CAP_VFLIP |
+ MDP_PIPE_CAP_SCALE |
+ MDP_PIPE_CAP_CSC |
+ MDP_PIPE_CAP_DECIMATION |
+ MDP_PIPE_CAP_SW_PIX_EXT |
+ 0,
+ },
+ .pipe_rgb = {
+ .count = 4,
+ .base = { 0x15000, 0x17000, 0x19000, 0x1b000 },
+ .caps = MDP_PIPE_CAP_HFLIP |
+ MDP_PIPE_CAP_VFLIP |
+ MDP_PIPE_CAP_SCALE |
+ MDP_PIPE_CAP_DECIMATION |
+ MDP_PIPE_CAP_SW_PIX_EXT |
+ 0,
+ },
+ .pipe_dma = {
+ .count = 2,
+ .base = { 0x25000, 0x27000 },
+ .caps = MDP_PIPE_CAP_HFLIP |
+ MDP_PIPE_CAP_VFLIP |
+ MDP_PIPE_CAP_SW_PIX_EXT |
+ 0,
+ },
+ .lm = {
+ .count = 6,
+ .base = { 0x45000, 0x46000, 0x47000, 0x48000, 0x49000, 0x4a000 },
+ .nb_stages = 8,
+ .max_width = 2560,
+ .max_height = 0xFFFF,
+ },
+ .dspp = {
+ .count = 2,
+ .base = { 0x55000, 0x57000 },
+ },
+ .ad = {
+ .count = 3,
+ .base = { 0x79000, 0x79800, 0x7a000 },
+ },
+ .pp = {
+ .count = 4,
+ .base = { 0x71000, 0x71800, 0x72000, 0x72800 },
+ },
+ .cdm = {
+ .count = 1,
+ .base = { 0x7a200 },
+ },
+ .dsc = {
+ .count = 2,
+ .base = { 0x81000, 0x81400 },
+ },
+ .intf = {
+ .base = { 0x6b000, 0x6b800, 0x6c000, 0x6c800, 0x6d000 },
+ .connect = {
+ [0] = INTF_DISABLED,
+ [1] = INTF_DSI,
+ [2] = INTF_DSI,
+ [3] = INTF_HDMI,
+ },
+ },
+ .max_clk = 412500000,
};
static const struct mdp5_cfg_handler cfg_handlers[] = {
{ .revision = 3, .config = { .hw = &apq8084_config } },
{ .revision = 6, .config = { .hw = &msm8x16_config } },
{ .revision = 9, .config = { .hw = &msm8x94_config } },
+ { .revision = 7, .config = { .hw = &msm8x96_config } },
};
static struct mdp5_cfg_platform *mdp5_get_config(struct platform_device *dev);
int mmb_size; /* MMB: size in bytes */
uint32_t clients[MAX_CLIENTS]; /* SMP port allocation /pipe */
mdp5_smp_state_t reserved_state;/* SMP MMBs statically allocated */
- int reserved[MAX_CLIENTS]; /* # of MMBs allocated per client */
+ uint8_t reserved[MAX_CLIENTS]; /* # of MMBs allocated per client */
+};
+
+struct mdp5_mdp_block {
+ MDP5_SUB_BLOCK_DEFINITION;
+ uint32_t caps; /* MDP capabilities: MDP_CAP_xxx bits */
};
#define MDP5_INTF_NUM_MAX 5
struct mdp5_cfg_hw {
char *name;
- struct mdp5_sub_block mdp;
+ struct mdp5_mdp_block mdp;
struct mdp5_smp_block smp;
struct mdp5_ctl_block ctl;
struct mdp5_pipe_block pipe_vig;
struct mdp5_sub_block dspp;
struct mdp5_sub_block ad;
struct mdp5_sub_block pp;
+ struct mdp5_sub_block dsc;
+ struct mdp5_sub_block cdm;
struct mdp5_intf_block intf;
uint32_t max_clk;
irq_set_chip_and_handler(irq, &mdp5_hw_irq_chip, handle_level_irq);
irq_set_chip_data(irq, mdp5_kms);
- set_irq_flags(irq, IRQF_VALID);
return 0;
}
}
static int get_clk(struct platform_device *pdev, struct clk **clkp,
- const char *name)
+ const char *name, bool mandatory)
{
struct device *dev = &pdev->dev;
struct clk *clk = devm_clk_get(dev, name);
- if (IS_ERR(clk)) {
+ if (IS_ERR(clk) && mandatory) {
dev_err(dev, "failed to get %s (%ld)\n", name, PTR_ERR(clk));
return PTR_ERR(clk);
}
- *clkp = clk;
+ if (IS_ERR(clk))
+ DBG("skipping %s", name);
+ else
+ *clkp = clk;
+
return 0;
}
goto fail;
}
- ret = get_clk(pdev, &mdp5_kms->axi_clk, "bus_clk");
+ /* mandatory clocks: */
+ ret = get_clk(pdev, &mdp5_kms->axi_clk, "bus_clk", true);
if (ret)
goto fail;
- ret = get_clk(pdev, &mdp5_kms->ahb_clk, "iface_clk");
+ ret = get_clk(pdev, &mdp5_kms->ahb_clk, "iface_clk", true);
if (ret)
goto fail;
- ret = get_clk(pdev, &mdp5_kms->src_clk, "core_clk_src");
+ ret = get_clk(pdev, &mdp5_kms->src_clk, "core_clk_src", true);
if (ret)
goto fail;
- ret = get_clk(pdev, &mdp5_kms->core_clk, "core_clk");
+ ret = get_clk(pdev, &mdp5_kms->core_clk, "core_clk", true);
if (ret)
goto fail;
- ret = get_clk(pdev, &mdp5_kms->lut_clk, "lut_clk");
- if (ret)
- DBG("failed to get (optional) lut_clk clock");
- ret = get_clk(pdev, &mdp5_kms->vsync_clk, "vsync_clk");
+ ret = get_clk(pdev, &mdp5_kms->vsync_clk, "vsync_clk", true);
if (ret)
goto fail;
+ /* optional clocks: */
+ get_clk(pdev, &mdp5_kms->lut_clk, "lut_clk", false);
+
/* we need to set a default rate before enabling. Set a safe
* rate first, then figure out hw revision, and then set a
* more optimal rate:
}
config = mdp5_cfg_get_config(mdp5_kms->cfg);
+ mdp5_kms->caps = config->hw->mdp.caps;
/* TODO: compute core clock rate at runtime */
clk_set_rate(mdp5_kms->src_clk, config->hw->max_clk);
- mdp5_kms->smp = mdp5_smp_init(mdp5_kms->dev, &config->hw->smp);
- if (IS_ERR(mdp5_kms->smp)) {
- ret = PTR_ERR(mdp5_kms->smp);
- mdp5_kms->smp = NULL;
- goto fail;
+ /*
+ * Some chipsets have a Shared Memory Pool (SMP), while others
+ * have dedicated latency buffering per source pipe instead;
+ * this section initializes the SMP:
+ */
+ if (mdp5_kms->caps & MDP_CAP_SMP) {
+ mdp5_kms->smp = mdp5_smp_init(mdp5_kms->dev, &config->hw->smp);
+ if (IS_ERR(mdp5_kms->smp)) {
+ ret = PTR_ERR(mdp5_kms->smp);
+ mdp5_kms->smp = NULL;
+ goto fail;
+ }
}
mdp5_kms->ctlm = mdp5_ctlm_init(dev, mdp5_kms->mmio, mdp5_kms->cfg);
if (IS_ERR(mmu)) {
ret = PTR_ERR(mmu);
dev_err(dev->dev, "failed to init iommu: %d\n", ret);
+ iommu_domain_free(config->platform.iommu);
goto fail;
}
struct drm_device *dev;
struct mdp5_cfg_handler *cfg;
+ uint32_t caps; /* MDP capabilities (MDP_CAP_XXX bits) */
+
/* mapper-id used to request GEM buffer mapped for scanout: */
int id;
};
static int mdp5_plane_prepare_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
struct mdp5_kms *mdp5_kms = get_kms(plane);
+ struct drm_framebuffer *fb = new_state->fb;
+
+ if (!new_state->fb)
+ return 0;
DBG("%s: prepare: FB[%u]", mdp5_plane->name, fb->base.id);
return msm_framebuffer_prepare(fb, mdp5_kms->id);
}
static void mdp5_plane_cleanup_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *old_state)
{
struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
struct mdp5_kms *mdp5_kms = get_kms(plane);
+ struct drm_framebuffer *fb = old_state->fb;
+
+ if (!fb)
+ return;
DBG("%s: cleanup: FB[%u]", mdp5_plane->name, fb->base.id);
msm_framebuffer_cleanup(fb, mdp5_kms->id);
static int calc_scalex_steps(struct drm_plane *plane,
uint32_t pixel_format, uint32_t src, uint32_t dest,
- uint32_t phasex_steps[2])
+ uint32_t phasex_steps[COMP_MAX])
{
struct mdp5_kms *mdp5_kms = get_kms(plane);
struct device *dev = mdp5_kms->dev->dev;
hsub = drm_format_horz_chroma_subsampling(pixel_format);
- phasex_steps[0] = phasex_step;
- phasex_steps[1] = phasex_step / hsub;
+ phasex_steps[COMP_0] = phasex_step;
+ phasex_steps[COMP_3] = phasex_step;
+ phasex_steps[COMP_1_2] = phasex_step / hsub;
return 0;
}
static int calc_scaley_steps(struct drm_plane *plane,
uint32_t pixel_format, uint32_t src, uint32_t dest,
- uint32_t phasey_steps[2])
+ uint32_t phasey_steps[COMP_MAX])
{
struct mdp5_kms *mdp5_kms = get_kms(plane);
struct device *dev = mdp5_kms->dev->dev;
vsub = drm_format_vert_chroma_subsampling(pixel_format);
- phasey_steps[0] = phasey_step;
- phasey_steps[1] = phasey_step / vsub;
+ phasey_steps[COMP_0] = phasey_step;
+ phasey_steps[COMP_3] = phasey_step;
+ phasey_steps[COMP_1_2] = phasey_step / vsub;
return 0;
}
-static uint32_t get_scale_config(enum mdp_chroma_samp_type chroma_sample,
- uint32_t src, uint32_t dest, bool hor)
+static uint32_t get_scale_config(const struct mdp_format *format,
+ uint32_t src, uint32_t dst, bool horz)
{
- uint32_t y_filter = (src <= dest) ? SCALE_FILTER_CA : SCALE_FILTER_PCMN;
- uint32_t y_a_filter = (src <= dest) ? SCALE_FILTER_BIL : SCALE_FILTER_PCMN;
- uint32_t uv_filter = ((src / 2) <= dest) ? /* 2x upsample */
- SCALE_FILTER_BIL : SCALE_FILTER_PCMN;
- uint32_t value = 0;
-
- if (chroma_sample == CHROMA_420 || chroma_sample == CHROMA_H2V1) {
- if (hor)
- value = MDP5_PIPE_SCALE_CONFIG_SCALEX_EN |
- MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0(y_filter) |
- MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3(y_a_filter) |
- MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_1_2(uv_filter);
- else
- value = MDP5_PIPE_SCALE_CONFIG_SCALEY_EN |
- MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0(y_filter) |
- MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3(y_a_filter) |
- MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2(uv_filter);
- } else if (src != dest) {
- if (hor)
- value = MDP5_PIPE_SCALE_CONFIG_SCALEX_EN |
- MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0(y_a_filter) |
- MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3(y_a_filter);
- else
- value = MDP5_PIPE_SCALE_CONFIG_SCALEY_EN |
- MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0(y_a_filter) |
- MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3(y_a_filter);
+ bool scaling = format->is_yuv ? true : (src != dst);
+ uint32_t sub, pix_fmt = format->base.pixel_format;
+ uint32_t ya_filter, uv_filter;
+ bool yuv = format->is_yuv;
+
+ if (!scaling)
+ return 0;
+
+ if (yuv) {
+ sub = horz ? drm_format_horz_chroma_subsampling(pix_fmt) :
+ drm_format_vert_chroma_subsampling(pix_fmt);
+ uv_filter = ((src / sub) <= dst) ?
+ SCALE_FILTER_BIL : SCALE_FILTER_PCMN;
}
+ ya_filter = (src <= dst) ? SCALE_FILTER_BIL : SCALE_FILTER_PCMN;
- return value;
+ if (horz)
+ return MDP5_PIPE_SCALE_CONFIG_SCALEX_EN |
+ MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0(ya_filter) |
+ MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3(ya_filter) |
+ COND(yuv, MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_1_2(uv_filter));
+ else
+ return MDP5_PIPE_SCALE_CONFIG_SCALEY_EN |
+ MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0(ya_filter) |
+ MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3(ya_filter) |
+ COND(yuv, MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2(uv_filter));
}
+static void calc_pixel_ext(const struct mdp_format *format,
+ uint32_t src, uint32_t dst, uint32_t phase_step[2],
+ int pix_ext_edge1[COMP_MAX], int pix_ext_edge2[COMP_MAX],
+ bool horz)
+{
+ bool scaling = format->is_yuv ? true : (src != dst);
+ int i;
+
+ /*
+ * Note:
+ * We assume here that:
+ * 1. PCMN filter is used for downscale
+ * 2. bilinear filter is used for upscale
+ * 3. we are in a single pipe configuration
+ */
+
+ for (i = 0; i < COMP_MAX; i++) {
+ pix_ext_edge1[i] = 0;
+ pix_ext_edge2[i] = scaling ? 1 : 0;
+ }
+}
+
+static void mdp5_write_pixel_ext(struct mdp5_kms *mdp5_kms, enum mdp5_pipe pipe,
+ const struct mdp_format *format,
+ uint32_t src_w, int pe_left[COMP_MAX], int pe_right[COMP_MAX],
+ uint32_t src_h, int pe_top[COMP_MAX], int pe_bottom[COMP_MAX])
+{
+ uint32_t pix_fmt = format->base.pixel_format;
+ uint32_t lr, tb, req;
+ int i;
+
+ for (i = 0; i < COMP_MAX; i++) {
+ uint32_t roi_w = src_w;
+ uint32_t roi_h = src_h;
+
+ if (format->is_yuv && i == COMP_1_2) {
+ roi_w /= drm_format_horz_chroma_subsampling(pix_fmt);
+ roi_h /= drm_format_vert_chroma_subsampling(pix_fmt);
+ }
+
+ lr = (pe_left[i] >= 0) ?
+ MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT(pe_left[i]) :
+ MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF(pe_left[i]);
+
+ lr |= (pe_right[i] >= 0) ?
+ MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT(pe_right[i]) :
+ MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF(pe_right[i]);
+
+ tb = (pe_top[i] >= 0) ?
+ MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT(pe_top[i]) :
+ MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF(pe_top[i]);
+
+ tb |= (pe_bottom[i] >= 0) ?
+ MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT(pe_bottom[i]) :
+ MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF(pe_bottom[i]);
+
+ req = MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT(roi_w +
+ pe_left[i] + pe_right[i]);
+
+ req |= MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM(roi_h +
+ pe_top[i] + pe_bottom[i]);
+
+ mdp5_write(mdp5_kms, REG_MDP5_PIPE_SW_PIX_EXT_LR(pipe, i), lr);
+ mdp5_write(mdp5_kms, REG_MDP5_PIPE_SW_PIX_EXT_TB(pipe, i), tb);
+ mdp5_write(mdp5_kms, REG_MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS(pipe, i), req);
+
+ DBG("comp-%d (L/R): rpt=%d/%d, ovf=%d/%d, req=%d", i,
+ FIELD(lr, MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT),
+ FIELD(lr, MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT),
+ FIELD(lr, MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF),
+ FIELD(lr, MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF),
+ FIELD(req, MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT));
+
+ DBG("comp-%d (T/B): rpt=%d/%d, ovf=%d/%d, req=%d", i,
+ FIELD(tb, MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT),
+ FIELD(tb, MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT),
+ FIELD(tb, MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF),
+ FIELD(tb, MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF),
+ FIELD(req, MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM));
+ }
+}
+
+
static int mdp5_plane_mode_set(struct drm_plane *plane,
struct drm_crtc *crtc, struct drm_framebuffer *fb,
int crtc_x, int crtc_y,
enum mdp5_pipe pipe = mdp5_plane->pipe;
const struct mdp_format *format;
uint32_t nplanes, config = 0;
- /* below array -> index 0: comp 0/3 ; index 1: comp 1/2 */
- uint32_t phasex_step[2] = {0,}, phasey_step[2] = {0,};
+ uint32_t phasex_step[COMP_MAX] = {0,}, phasey_step[COMP_MAX] = {0,};
+ bool pe = mdp5_plane->caps & MDP_PIPE_CAP_SW_PIX_EXT;
+ int pe_left[COMP_MAX], pe_right[COMP_MAX];
+ int pe_top[COMP_MAX], pe_bottom[COMP_MAX];
uint32_t hdecm = 0, vdecm = 0;
uint32_t pix_format;
bool vflip, hflip;
crtc->base.id, crtc_x, crtc_y, crtc_w, crtc_h);
/* Request some memory from the SMP: */
- ret = mdp5_smp_request(mdp5_kms->smp,
- mdp5_plane->pipe, format, src_w, false);
- if (ret)
- return ret;
+ if (mdp5_kms->smp) {
+ ret = mdp5_smp_request(mdp5_kms->smp,
+ mdp5_plane->pipe, format, src_w, false);
+ if (ret)
+ return ret;
+ }
/*
* Currently we update the hw for allocations/requests immediately,
* would move into atomic->check_plane_state(), while updating the
* hw would remain here:
*/
- mdp5_smp_configure(mdp5_kms->smp, pipe);
+ if (mdp5_kms->smp)
+ mdp5_smp_configure(mdp5_kms->smp, pipe);
ret = calc_scalex_steps(plane, pix_format, src_w, crtc_w, phasex_step);
if (ret)
if (ret)
return ret;
+ if (mdp5_plane->caps & MDP_PIPE_CAP_SW_PIX_EXT) {
+ calc_pixel_ext(format, src_w, crtc_w, phasex_step,
+ pe_left, pe_right, true);
+ calc_pixel_ext(format, src_h, crtc_h, phasey_step,
+ pe_top, pe_bottom, false);
+ }
+
/* TODO calc hdecm, vdecm */
/* SCALE is used to both scale and up-sample chroma components */
- config |= get_scale_config(format->chroma_sample, src_w, crtc_w, true);
- config |= get_scale_config(format->chroma_sample, src_h, crtc_h, false);
+ config |= get_scale_config(format, src_w, crtc_w, true);
+ config |= get_scale_config(format, src_h, crtc_h, false);
DBG("scale config = %x", config);
hflip = !!(pstate->rotation & BIT(DRM_REFLECT_X));
mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_OP_MODE(pipe),
(hflip ? MDP5_PIPE_SRC_OP_MODE_FLIP_LR : 0) |
(vflip ? MDP5_PIPE_SRC_OP_MODE_FLIP_UD : 0) |
+ COND(pe, MDP5_PIPE_SRC_OP_MODE_SW_PIX_EXT_OVERRIDE) |
MDP5_PIPE_SRC_OP_MODE_BWC(BWC_LOSSLESS));
/* not using secure mode: */
mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_ADDR_SW_STATUS(pipe), 0);
+ if (mdp5_plane->caps & MDP_PIPE_CAP_SW_PIX_EXT)
+ mdp5_write_pixel_ext(mdp5_kms, pipe, format,
+ src_w, pe_left, pe_right,
+ src_h, pe_top, pe_bottom);
+
if (mdp5_plane->caps & MDP_PIPE_CAP_SCALE) {
mdp5_write(mdp5_kms, REG_MDP5_PIPE_SCALE_PHASE_STEP_X(pipe),
- phasex_step[0]);
+ phasex_step[COMP_0]);
mdp5_write(mdp5_kms, REG_MDP5_PIPE_SCALE_PHASE_STEP_Y(pipe),
- phasey_step[0]);
+ phasey_step[COMP_0]);
mdp5_write(mdp5_kms, REG_MDP5_PIPE_SCALE_CR_PHASE_STEP_X(pipe),
- phasex_step[1]);
+ phasex_step[COMP_1_2]);
mdp5_write(mdp5_kms, REG_MDP5_PIPE_SCALE_CR_PHASE_STEP_Y(pipe),
- phasey_step[1]);
+ phasey_step[COMP_1_2]);
mdp5_write(mdp5_kms, REG_MDP5_PIPE_DECIMATION(pipe),
MDP5_PIPE_DECIMATION_VERT(vdecm) |
MDP5_PIPE_DECIMATION_HORZ(hdecm));
DBG("%s: complete flip", mdp5_plane->name);
- mdp5_smp_commit(mdp5_kms->smp, pipe);
+ if (mdp5_kms->smp)
+ mdp5_smp_commit(mdp5_kms->smp, pipe);
to_mdp5_plane_state(plane->state)->pending = false;
}
struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
enum mdp5_pipe pipe = mdp5_plane->pipe;
- if (!plane_enabled(plane->state)) {
+ if (!plane_enabled(plane->state) && mdp5_kms->smp) {
DBG("%s: free SMP", mdp5_plane->name);
mdp5_smp_release(mdp5_kms->smp, pipe);
}
struct mdp5_smp {
struct drm_device *dev;
- const struct mdp5_smp_block *cfg;
+ uint8_t reserved[MAX_CLIENTS]; /* fixed MMBs allocation per client */
int blk_cnt;
int blk_size;
struct mdp5_kms *mdp5_kms = get_kms(smp);
struct mdp5_client_smp_state *ps = &smp->client_state[cid];
int i, ret, avail, cur_nblks, cnt = smp->blk_cnt;
- int reserved;
+ uint8_t reserved;
unsigned long flags;
- reserved = smp->cfg->reserved[cid];
+ reserved = smp->reserved[cid];
spin_lock_irqsave(&smp->state_lock, flags);
}
smp->dev = dev;
- smp->cfg = cfg;
smp->blk_cnt = cfg->mmb_count;
smp->blk_size = cfg->mmb_size;
/* statically tied MMBs cannot be re-allocated: */
bitmap_copy(smp->state, cfg->reserved_state, smp->blk_cnt);
+ memcpy(smp->reserved, cfg->reserved, sizeof(smp->reserved));
spin_lock_init(&smp->state_lock);
return smp;
- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2576 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 36021 bytes, from 2015-07-09 22:10:24)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 26057 bytes, from 2015-08-14 21:47:57)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 344 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02)
- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07)
- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 29154 bytes, from 2015-08-10 21:25:43)
BG_PIXEL = 3,
};
+enum mdp_component_type {
+ COMP_0 = 0,
+ COMP_1_2 = 1,
+ COMP_3 = 2,
+ COMP_MAX = 3,
+};
+
enum mdp_bpc {
BPC1 = 0,
BPC5 = 1,
uint32_t mdp_get_formats(uint32_t *formats, uint32_t max_formats, bool rgb_only);
const struct msm_format *mdp_get_format(struct msm_kms *kms, uint32_t format);
+/* MDP capabilities */
+#define MDP_CAP_SMP BIT(0) /* Shared Memory Pool */
+#define MDP_CAP_DSC BIT(1) /* VESA Display Stream Compression */
+#define MDP_CAP_CDM BIT(2) /* Chroma Down Module (HDMI 2.0 YUV) */
+
/* MDP pipe capabilities */
#define MDP_PIPE_CAP_HFLIP BIT(0)
#define MDP_PIPE_CAP_VFLIP BIT(1)
#define MDP_PIPE_CAP_SCALE BIT(2)
#define MDP_PIPE_CAP_CSC BIT(3)
#define MDP_PIPE_CAP_DECIMATION BIT(4)
+#define MDP_PIPE_CAP_SW_PIX_EXT BIT(5)
static inline bool pipe_supports_yuv(uint32_t pipe_caps)
{
drm_atomic_helper_commit_modeset_disables(dev, state);
- drm_atomic_helper_commit_planes(dev, state);
+ drm_atomic_helper_commit_planes(dev, state, false);
drm_atomic_helper_commit_modeset_enables(dev, state);
static void msm_fb_output_poll_changed(struct drm_device *dev)
{
-#ifdef CONFIG_DRM_MSM_FBDEV
struct msm_drm_private *priv = dev->dev_private;
if (priv->fbdev)
drm_fb_helper_hotplug_event(priv->fbdev);
-#endif
}
static const struct drm_mode_config_funcs mode_config_funcs = {
#define reglog 0
#endif
-#ifdef CONFIG_DRM_MSM_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
static bool fbdev = true;
MODULE_PARM_DESC(fbdev, "Enable fbdev compat layer");
module_param(fbdev, bool, 0600);
drm_mode_config_reset(dev);
-#ifdef CONFIG_DRM_MSM_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
if (fbdev)
priv->fbdev = msm_fbdev_init(dev);
#endif
static void msm_lastclose(struct drm_device *dev)
{
-#ifdef CONFIG_DRM_MSM_FBDEV
struct msm_drm_private *priv = dev->dev_private;
if (priv->fbdev)
drm_fb_helper_restore_fbdev_mode_unlocked(priv->fbdev);
-#endif
}
static irqreturn_t msm_irq(int irq, void *arg)
kms->funcs->irq_uninstall(kms);
}
-static int msm_enable_vblank(struct drm_device *dev, int crtc_id)
+static int msm_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct msm_drm_private *priv = dev->dev_private;
struct msm_kms *kms = priv->kms;
if (!kms)
return -ENXIO;
- DBG("dev=%p, crtc=%d", dev, crtc_id);
- return vblank_ctrl_queue_work(priv, crtc_id, true);
+ DBG("dev=%p, crtc=%u", dev, pipe);
+ return vblank_ctrl_queue_work(priv, pipe, true);
}
-static void msm_disable_vblank(struct drm_device *dev, int crtc_id)
+static void msm_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct msm_drm_private *priv = dev->dev_private;
struct msm_kms *kms = priv->kms;
if (!kms)
return;
- DBG("dev=%p, crtc=%d", dev, crtc_id);
- vblank_ctrl_queue_work(priv, crtc_id, false);
+ DBG("dev=%p, crtc=%u", dev, pipe);
+ vblank_ctrl_queue_work(priv, pipe, false);
}
/*
}
static const struct drm_ioctl_desc msm_ioctls[] = {
- DRM_IOCTL_DEF_DRV(MSM_GET_PARAM, msm_ioctl_get_param, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(MSM_GEM_NEW, msm_ioctl_gem_new, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(MSM_GEM_INFO, msm_ioctl_gem_info, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_PREP, msm_ioctl_gem_cpu_prep, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_FINI, msm_ioctl_gem_cpu_fini, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(MSM_GEM_SUBMIT, msm_ioctl_gem_submit, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(MSM_WAIT_FENCE, msm_ioctl_wait_fence, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(MSM_GET_PARAM, msm_ioctl_get_param, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(MSM_GEM_NEW, msm_ioctl_gem_new, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(MSM_GEM_INFO, msm_ioctl_gem_info, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_PREP, msm_ioctl_gem_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_FINI, msm_ioctl_gem_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(MSM_GEM_SUBMIT, msm_ioctl_gem_submit, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(MSM_WAIT_FENCE, msm_ioctl_wait_fence, DRM_AUTH|DRM_RENDER_ALLOW),
};
static const struct vm_operations_struct vm_ops = {
.irq_preinstall = msm_irq_preinstall,
.irq_postinstall = msm_irq_postinstall,
.irq_uninstall = msm_irq_uninstall,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = msm_enable_vblank,
.disable_vblank = msm_disable_vblank,
.gem_free_object = msm_gem_free_object,
if (drm_device_is_unplugged(dev))
return -ENODEV;
- mutex_lock(&dev->struct_mutex);
-
ret = drm_gem_mmap_obj(drm_obj, drm_obj->size, vma);
-
- mutex_unlock(&dev->struct_mutex);
-
if (ret) {
pr_err("%s:drm_gem_mmap_obj fail\n", __func__);
return ret;
{
int ret;
- mutex_lock(&obj->dev->struct_mutex);
ret = drm_gem_mmap_obj(obj, obj->size, vma);
- mutex_unlock(&obj->dev->struct_mutex);
if (ret < 0)
return ret;
if (iommu) {
dev_info(drm->dev, "%s: using IOMMU\n", name);
gpu->mmu = msm_iommu_new(&pdev->dev, iommu);
+ if (IS_ERR(gpu->mmu)) {
+ ret = PTR_ERR(gpu->mmu);
+ dev_err(drm->dev, "failed to init iommu: %d\n", ret);
+ gpu->mmu = NULL;
+ iommu_domain_free(iommu);
+ goto fail;
+ }
+
} else {
dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
}
#include "hw.h"
#include "tvnv17.h"
-char *nv17_tv_norm_names[NUM_TV_NORMS] = {
+const char * const nv17_tv_norm_names[NUM_TV_NORMS] = {
[TV_NORM_PAL] = "PAL",
[TV_NORM_PAL_M] = "PAL-M",
[TV_NORM_PAL_N] = "PAL-N",
#define to_tv_enc(x) container_of(nouveau_encoder(x), \
struct nv17_tv_encoder, base)
-extern char *nv17_tv_norm_names[NUM_TV_NORMS];
+extern const char * const nv17_tv_norm_names[NUM_TV_NORMS];
extern struct nv17_tv_norm_params {
enum {
#include <linux/agp_backend.h>
#include <linux/reset.h>
#include <linux/iommu.h>
+#include <linux/of_device.h>
#include <asm/unaligned.h>
#include <core/mm.h>
struct nvkm_device_tegra {
+ const struct nvkm_device_tegra_func *func;
struct nvkm_device device;
struct platform_device *pdev;
int irq;
int gpu_speedo;
};
-int nvkm_device_tegra_new(struct platform_device *,
+struct nvkm_device_tegra_func {
+ /*
+ * If an IOMMU is used, indicates which address bit will trigger a
+ * IOMMU translation when set (when this bit is not set, IOMMU is
+ * bypassed). A value of 0 means an IOMMU is never used.
+ */
+ u8 iommu_bit;
+};
+
+int nvkm_device_tegra_new(const struct nvkm_device_tegra_func *,
+ struct platform_device *,
const char *cfg, const char *dbg,
bool detect, bool mmio, u64 subdev_mask,
struct nvkm_device **);
DCB_GPIO_VID5 = 0x74,
DCB_GPIO_VID6 = 0x75,
DCB_GPIO_VID7 = 0x76,
+ DCB_GPIO_VID_PWM = 0x81,
};
#define DCB_GPIO_LOG_DIR 0x02
};
u32 nvbios_pmuTe(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
-u32 nvbios_pmuTp(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
- struct nvbios_pmuT *);
struct nvbios_pmuE {
u8 type;
unsigned ramcfg_timing;
unsigned ramcfg_DLLoff;
unsigned ramcfg_RON;
+ unsigned ramcfg_FBVDDQ;
union {
struct {
unsigned ramcfg_00_03_01:1;
unsigned ramcfg_11_01_04:1;
unsigned ramcfg_11_01_08:1;
unsigned ramcfg_11_01_10:1;
- unsigned ramcfg_11_01_20:1;
unsigned ramcfg_11_01_40:1;
unsigned ramcfg_11_01_80:1;
unsigned ramcfg_11_02_03:2;
#ifndef __NVBIOS_VOLT_H__
#define __NVBIOS_VOLT_H__
+
+enum nvbios_volt_type {
+ NVBIOS_VOLT_GPIO = 0,
+ NVBIOS_VOLT_PWM,
+};
+
struct nvbios_volt {
- u8 vidmask;
+ enum nvbios_volt_type type;
u32 min;
u32 max;
u32 base;
+
+ /* GPIO mode */
+ u8 vidmask;
s16 step;
+
+ /* PWM mode */
+ u32 pwm_freq;
+ u32 pwm_range;
};
u16 nvbios_volt_table(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
void nvkm_hwsq_wr32(struct nvkm_hwsq *, u32 addr, u32 data);
void nvkm_hwsq_setf(struct nvkm_hwsq *, u8 flag, int data);
void nvkm_hwsq_wait(struct nvkm_hwsq *, u8 flag, u8 data);
+void nvkm_hwsq_wait_vblank(struct nvkm_hwsq *);
void nvkm_hwsq_nsec(struct nvkm_hwsq *, u32 nsec);
int nv04_bus_new(struct nvkm_device *, int, struct nvkm_bus **);
#include <core/subdev.h>
int gf100_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
+int gf117_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
int gk104_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
int gk20a_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
#endif
int nvkm_ltc_zbc_color_get(struct nvkm_ltc *, int index, const u32[4]);
int nvkm_ltc_zbc_depth_get(struct nvkm_ltc *, int index, const u32);
+void nvkm_ltc_invalidate(struct nvkm_ltc *);
+void nvkm_ltc_flush(struct nvkm_ltc *);
+
int gf100_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
int gk104_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
+int gk20a_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
int gm107_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
#endif
u32 nvkm_pci_rd32(struct nvkm_pci *, u16 addr);
void nvkm_pci_wr08(struct nvkm_pci *, u16 addr, u8 data);
void nvkm_pci_wr32(struct nvkm_pci *, u16 addr, u32 data);
+u32 nvkm_pci_mask(struct nvkm_pci *, u16 addr, u32 mask, u32 value);
void nvkm_pci_rom_shadow(struct nvkm_pci *, bool shadow);
int nv04_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
int nv40_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
+int nv46_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
int nv4c_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
-int nv50_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
+int g84_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
+int g94_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
int gf100_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
#endif
#define nvkm_usec(d,u,cond...) nvkm_nsec((d), (u) * 1000, ##cond)
#define nvkm_msec(d,m,cond...) nvkm_usec((d), (m) * 1000, ##cond)
+#define nvkm_wait_nsec(d,n,addr,mask,data) \
+ nvkm_nsec(d, n, \
+ if ((nvkm_rd32(d, (addr)) & (mask)) == (data)) \
+ break; \
+ )
+#define nvkm_wait_usec(d,u,addr,mask,data) \
+ nvkm_wait_nsec((d), (u) * 1000, (addr), (mask), (data))
+#define nvkm_wait_msec(d,m,addr,mask,data) \
+ nvkm_wait_usec((d), (m) * 1000, (addr), (mask), (data))
+
int nv04_timer_new(struct nvkm_device *, int, struct nvkm_timer **);
int nv40_timer_new(struct nvkm_device *, int, struct nvkm_timer **);
int nv41_timer_new(struct nvkm_device *, int, struct nvkm_timer **);
int nvkm_volt_set_id(struct nvkm_volt *, u8 id, int condition);
int nv40_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
+int gk104_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
int gk20a_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
#endif
#include <nvif/driver.h>
#include <nvif/ioctl.h>
#include <nvif/class.h>
+#include <nvif/unpack.h>
#include "nouveau_drm.h"
#include "nouveau_dma.h"
#include "nouveau_chan.h"
#include "nouveau_abi16.h"
-struct nouveau_abi16 *
-nouveau_abi16_get(struct drm_file *file_priv, struct drm_device *dev)
+static struct nouveau_abi16 *
+nouveau_abi16(struct drm_file *file_priv)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
- mutex_lock(&cli->mutex);
if (!cli->abi16) {
struct nouveau_abi16 *abi16;
cli->abi16 = abi16 = kzalloc(sizeof(*abi16), GFP_KERNEL);
* device (ie. the one that belongs to the fd it
* opened)
*/
- if (nvif_device_init(&cli->base.object,
- NOUVEAU_ABI16_DEVICE, NV_DEVICE,
+ if (nvif_device_init(&cli->base.object, 0, NV_DEVICE,
&args, sizeof(args),
&abi16->device) == 0)
return cli->abi16;
kfree(cli->abi16);
cli->abi16 = NULL;
}
-
- mutex_unlock(&cli->mutex);
}
return cli->abi16;
}
+struct nouveau_abi16 *
+nouveau_abi16_get(struct drm_file *file_priv)
+{
+ struct nouveau_cli *cli = nouveau_cli(file_priv);
+ mutex_lock(&cli->mutex);
+ if (nouveau_abi16(file_priv))
+ return cli->abi16;
+ mutex_unlock(&cli->mutex);
+ return NULL;
+}
+
int
nouveau_abi16_put(struct nouveau_abi16 *abi16, int ret)
{
/* destroy channel object, all children will be killed too */
if (chan->chan) {
- abi16->handles &= ~(1ULL << (chan->chan->user.handle & 0xffff));
nouveau_channel_idle(chan->chan);
nouveau_channel_del(&chan->chan);
}
struct drm_nouveau_channel_alloc *init = data;
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct nouveau_drm *drm = nouveau_drm(dev);
- struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+ struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
struct nouveau_abi16_chan *chan;
struct nvif_device *device;
int ret;
return nouveau_abi16_put(abi16, -EINVAL);
/* allocate "abi16 channel" data and make up a handle for it */
- init->channel = __ffs64(~abi16->handles);
- if (~abi16->handles == 0)
- return nouveau_abi16_put(abi16, -ENOSPC);
-
chan = kzalloc(sizeof(*chan), GFP_KERNEL);
if (!chan)
return nouveau_abi16_put(abi16, -ENOMEM);
INIT_LIST_HEAD(&chan->notifiers);
list_add(&chan->head, &abi16->channels);
- abi16->handles |= (1ULL << init->channel);
/* create channel object and initialise dma and fence management */
- ret = nouveau_channel_new(drm, device,
- NOUVEAU_ABI16_CHAN(init->channel),
- init->fb_ctxdma_handle,
+ ret = nouveau_channel_new(drm, device, init->fb_ctxdma_handle,
init->tt_ctxdma_handle, &chan->chan);
if (ret)
goto done;
+ init->channel = chan->chan->chid;
+
if (device->info.family >= NV_DEVICE_INFO_V0_TESLA)
init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM |
NOUVEAU_GEM_DOMAIN_GART;
struct nouveau_abi16_chan *chan;
list_for_each_entry(chan, &abi16->channels, head) {
- if (chan->chan->user.handle == NOUVEAU_ABI16_CHAN(channel))
+ if (chan->chan->chid == channel)
return chan;
}
return NULL;
}
+int
+nouveau_abi16_usif(struct drm_file *file_priv, void *data, u32 size)
+{
+ union {
+ struct nvif_ioctl_v0 v0;
+ } *args = data;
+ struct nouveau_abi16_chan *chan;
+ struct nouveau_abi16 *abi16;
+ int ret;
+
+ if (nvif_unpack(args->v0, 0, 0, true)) {
+ switch (args->v0.type) {
+ case NVIF_IOCTL_V0_NEW:
+ case NVIF_IOCTL_V0_MTHD:
+ case NVIF_IOCTL_V0_SCLASS:
+ break;
+ default:
+ return -EACCES;
+ }
+ } else
+ return ret;
+
+ if (!(abi16 = nouveau_abi16(file_priv)))
+ return -ENOMEM;
+
+ if (args->v0.token != ~0ULL) {
+ if (!(chan = nouveau_abi16_chan(abi16, args->v0.token)))
+ return -EINVAL;
+ args->v0.object = nvif_handle(&chan->chan->user);
+ args->v0.owner = NVIF_IOCTL_V0_OWNER_ANY;
+ return 0;
+ }
+
+ args->v0.object = nvif_handle(&abi16->device.object);
+ args->v0.owner = NVIF_IOCTL_V0_OWNER_ANY;
+ return 0;
+}
+
int
nouveau_abi16_ioctl_channel_free(ABI16_IOCTL_ARGS)
{
struct drm_nouveau_channel_free *req = data;
- struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+ struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
struct nouveau_abi16_chan *chan;
if (unlikely(!abi16))
nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS)
{
struct drm_nouveau_grobj_alloc *init = data;
- struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+ struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
struct nouveau_abi16_chan *chan;
struct nouveau_abi16_ntfy *ntfy;
struct nvif_client *client;
{
struct drm_nouveau_notifierobj_alloc *info = data;
struct nouveau_drm *drm = nouveau_drm(dev);
- struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+ struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
struct nouveau_abi16_chan *chan;
struct nouveau_abi16_ntfy *ntfy;
struct nvif_device *device = &abi16->device;
nouveau_abi16_ioctl_gpuobj_free(ABI16_IOCTL_ARGS)
{
struct drm_nouveau_gpuobj_free *fini = data;
- struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+ struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
struct nouveau_abi16_chan *chan;
struct nouveau_abi16_ntfy *ntfy;
int ret = -ENOENT;
u64 handles;
};
-struct nouveau_drm;
-struct nouveau_abi16 *nouveau_abi16_get(struct drm_file *, struct drm_device *);
+struct nouveau_abi16 *nouveau_abi16_get(struct drm_file *);
int nouveau_abi16_put(struct nouveau_abi16 *, int);
void nouveau_abi16_fini(struct nouveau_abi16 *);
s32 nouveau_abi16_swclass(struct nouveau_drm *);
+int nouveau_abi16_usif(struct drm_file *, void *data, u32 size);
#define NOUVEAU_GEM_DOMAIN_VRAM (1 << 1)
#define NOUVEAU_GEM_DOMAIN_GART (1 << 2)
return VGA_SWITCHEROO_DIS;
}
-static struct vga_switcheroo_handler nouveau_dsm_handler = {
+static const struct vga_switcheroo_handler nouveau_dsm_handler = {
.switchto = nouveau_dsm_switchto,
.power_state = nouveau_dsm_power_state,
.get_client_id = nouveau_dsm_get_client_id,
nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
uint32_t page_flags, struct page *dummy_read)
{
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
struct nouveau_drm *drm = nouveau_bdev(bdev);
if (drm->agp.bridge) {
/* System memory */
return 0;
case TTM_PL_TT:
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (drm->agp.bridge) {
mem->bus.offset = mem->start << PAGE_SHIFT;
mem->bus.base = drm->agp.base;
ttm->caching_state == tt_uncached)
return ttm_dma_populate(ttm_dma, dev->dev);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (drm->agp.bridge) {
return ttm_agp_tt_populate(ttm);
}
return;
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (drm->agp.bridge) {
ttm_agp_tt_unpopulate(ttm);
return;
}
if (ret) {
- NV_PRINTK(err, cli, "failed to idle channel "
- "0x%08x [%s]\n",
- chan->user.handle,
- nvxx_client(&cli->base)->name);
+ NV_PRINTK(err, cli, "failed to idle channel %d [%s]\n",
+ chan->chid, nvxx_client(&cli->base)->name);
return ret;
}
}
static int
nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device,
- u32 handle, u32 size, struct nouveau_channel **pchan)
+ u32 size, struct nouveau_channel **pchan)
{
struct nouveau_cli *cli = (void *)device->object.client;
struct nvkm_mmu *mmu = nvxx_mmu(device);
}
}
- ret = nvif_object_init(&device->object, NVDRM_PUSH |
- (handle & 0xffff), NV_DMA_FROM_MEMORY,
+ ret = nvif_object_init(&device->object, 0, NV_DMA_FROM_MEMORY,
&args, sizeof(args), &chan->push.ctxdma);
if (ret) {
nouveau_channel_del(pchan);
static int
nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device,
- u32 handle, u32 engine, struct nouveau_channel **pchan)
+ u32 engine, struct nouveau_channel **pchan)
{
static const u16 oclasses[] = { MAXWELL_CHANNEL_GPFIFO_A,
KEPLER_CHANNEL_GPFIFO_A,
int ret;
/* allocate dma push buffer */
- ret = nouveau_channel_prep(drm, device, handle, 0x12000, &chan);
+ ret = nouveau_channel_prep(drm, device, 0x12000, &chan);
*pchan = chan;
if (ret)
return ret;
size = sizeof(args.nv50);
}
- ret = nvif_object_init(&device->object, handle, *oclass++,
+ ret = nvif_object_init(&device->object, 0, *oclass++,
&args, size, &chan->user);
if (ret == 0) {
if (chan->user.oclass >= KEPLER_CHANNEL_GPFIFO_A)
static int
nouveau_channel_dma(struct nouveau_drm *drm, struct nvif_device *device,
- u32 handle, struct nouveau_channel **pchan)
+ struct nouveau_channel **pchan)
{
static const u16 oclasses[] = { NV40_CHANNEL_DMA,
NV17_CHANNEL_DMA,
int ret;
/* allocate dma push buffer */
- ret = nouveau_channel_prep(drm, device, handle, 0x10000, &chan);
+ ret = nouveau_channel_prep(drm, device, 0x10000, &chan);
*pchan = chan;
if (ret)
return ret;
args.offset = chan->push.vma.offset;
do {
- ret = nvif_object_init(&device->object, handle, *oclass++,
+ ret = nvif_object_init(&device->object, 0, *oclass++,
&args, sizeof(args), &chan->user);
if (ret == 0) {
chan->chid = args.chid;
int
nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device,
- u32 handle, u32 arg0, u32 arg1,
- struct nouveau_channel **pchan)
+ u32 arg0, u32 arg1, struct nouveau_channel **pchan)
{
struct nouveau_cli *cli = (void *)device->object.client;
bool super;
super = cli->base.super;
cli->base.super = true;
- ret = nouveau_channel_ind(drm, device, handle, arg0, pchan);
+ ret = nouveau_channel_ind(drm, device, arg0, pchan);
if (ret) {
NV_PRINTK(dbg, cli, "ib channel create, %d\n", ret);
- ret = nouveau_channel_dma(drm, device, handle, pchan);
+ ret = nouveau_channel_dma(drm, device, pchan);
if (ret) {
NV_PRINTK(dbg, cli, "dma channel create, %d\n", ret);
goto done;
int nouveau_channel_new(struct nouveau_drm *, struct nvif_device *,
- u32 handle, u32 arg0, u32 arg1,
- struct nouveau_channel **);
+ u32 arg0, u32 arg1, struct nouveau_channel **);
void nouveau_channel_del(struct nouveau_channel **);
int nouveau_channel_idle(struct nouveau_channel *);
}
int
-nouveau_display_vblank_enable(struct drm_device *dev, int head)
+nouveau_display_vblank_enable(struct drm_device *dev, unsigned int pipe)
{
struct drm_crtc *crtc;
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
- if (nv_crtc->index == head) {
+ if (nv_crtc->index == pipe) {
nvif_notify_get(&nv_crtc->vblank);
return 0;
}
}
void
-nouveau_display_vblank_disable(struct drm_device *dev, int head)
+nouveau_display_vblank_disable(struct drm_device *dev, unsigned int pipe)
{
struct drm_crtc *crtc;
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
- if (nv_crtc->index == head) {
+ if (nv_crtc->index == pipe) {
nvif_notify_put(&nv_crtc->vblank);
return;
}
.base.head = nouveau_crtc(crtc)->index,
};
struct nouveau_display *disp = nouveau_display(crtc->dev);
+ struct drm_vblank_crtc *vblank = &crtc->dev->vblank[drm_crtc_index(crtc)];
int ret, retry = 1;
do {
break;
}
- if (retry) ndelay(crtc->linedur_ns);
+ if (retry) ndelay(vblank->linedur_ns);
} while (retry--);
*hpos = args.scan.hline;
}
int
-nouveau_display_scanoutpos(struct drm_device *dev, int head, unsigned int flags,
- int *vpos, int *hpos, ktime_t *stime, ktime_t *etime)
+nouveau_display_scanoutpos(struct drm_device *dev, unsigned int pipe,
+ unsigned int flags, int *vpos, int *hpos,
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode)
{
struct drm_crtc *crtc;
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
- if (nouveau_crtc(crtc)->index == head) {
+ if (nouveau_crtc(crtc)->index == pipe) {
return nouveau_display_scanoutpos_head(crtc, vpos, hpos,
stime, etime);
}
}
int
-nouveau_display_vblstamp(struct drm_device *dev, int head, int *max_error,
- struct timeval *time, unsigned flags)
+nouveau_display_vblstamp(struct drm_device *dev, unsigned int pipe,
+ int *max_error, struct timeval *time, unsigned flags)
{
struct drm_crtc *crtc;
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
- if (nouveau_crtc(crtc)->index == head) {
+ if (nouveau_crtc(crtc)->index == pipe) {
return drm_calc_vbltimestamp_from_scanoutpos(dev,
- head, max_error, time, flags, crtc,
+ pipe, max_error, time, flags,
&crtc->hwmode);
}
}
if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
dev->mode_config.max_width = 4096;
dev->mode_config.max_height = 4096;
- } else {
+ } else
+ if (drm->device.info.family < NV_DEVICE_INFO_V0_FERMI) {
dev->mode_config.max_width = 8192;
dev->mode_config.max_height = 8192;
+ } else {
+ dev->mode_config.max_width = 16384;
+ dev->mode_config.max_height = 16384;
}
dev->mode_config.preferred_depth = 24;
int i;
for (i = 0, ret = -ENODEV; ret && i < ARRAY_SIZE(oclass); i++) {
- ret = nvif_object_init(&drm->device.object,
- NVDRM_DISPLAY, oclass[i],
- NULL, 0, &disp->disp);
+ ret = nvif_object_init(&drm->device.object, 0,
+ oclass[i], NULL, 0, &disp->disp);
}
if (ret == 0) {
void nouveau_display_fini(struct drm_device *dev);
int nouveau_display_suspend(struct drm_device *dev, bool runtime);
void nouveau_display_resume(struct drm_device *dev, bool runtime);
-int nouveau_display_vblank_enable(struct drm_device *, int);
-void nouveau_display_vblank_disable(struct drm_device *, int);
-int nouveau_display_scanoutpos(struct drm_device *, int, unsigned int,
- int *, int *, ktime_t *, ktime_t *);
-int nouveau_display_vblstamp(struct drm_device *, int, int *,
+int nouveau_display_vblank_enable(struct drm_device *, unsigned int);
+void nouveau_display_vblank_disable(struct drm_device *, unsigned int);
+int nouveau_display_scanoutpos(struct drm_device *, unsigned int,
+ unsigned int, int *, int *, ktime_t *,
+ ktime_t *, const struct drm_display_mode *);
+int nouveau_display_vblstamp(struct drm_device *, unsigned int, int *,
struct timeval *, unsigned);
int nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
}
if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
- ret = nouveau_channel_new(drm, &drm->device, NVDRM_CHAN + 1,
+ ret = nouveau_channel_new(drm, &drm->device,
KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE0|
KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE1,
0, &drm->cechan);
if (device->info.chipset >= 0xa3 &&
device->info.chipset != 0xaa &&
device->info.chipset != 0xac) {
- ret = nouveau_channel_new(drm, &drm->device, NVDRM_CHAN + 1,
+ ret = nouveau_channel_new(drm, &drm->device,
NvDmaFB, NvDmaTT, &drm->cechan);
if (ret)
NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
arg1 = NvDmaTT;
}
- ret = nouveau_channel_new(drm, &drm->device, NVDRM_CHAN, arg0, arg1,
- &drm->channel);
+ ret = nouveau_channel_new(drm, &drm->device, arg0, arg1, &drm->channel);
if (ret) {
NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
nouveau_accel_fini(drm);
nouveau_get_hdmi_dev(drm);
- ret = nvif_device_init(&drm->client.base.object,
- NVDRM_DEVICE, NV_DEVICE,
+ ret = nvif_device_init(&drm->client.base.object, 0, NV_DEVICE,
&(struct nv_device_v0) {
.device = ~0,
}, sizeof(struct nv_device_v0),
static const struct drm_ioctl_desc
nouveau_ioctls[] = {
- DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_abi16_ioctl_setparam, DRM_UNLOCKED|DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_abi16_ioctl_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_AUTH|DRM_RENDER_ALLOW),
};
long
.debugfs_cleanup = nouveau_debugfs_takedown,
#endif
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = nouveau_display_vblank_enable,
.disable_vblank = nouveau_display_vblank_disable,
.get_scanout_position = nouveau_display_scanoutpos,
};
struct drm_device *
-nouveau_platform_device_create(struct platform_device *pdev,
+nouveau_platform_device_create(const struct nvkm_device_tegra_func *func,
+ struct platform_device *pdev,
struct nvkm_device **pdevice)
{
struct drm_device *drm;
int err;
- err = nvkm_device_tegra_new(pdev, nouveau_config, nouveau_debug,
+ err = nvkm_device_tegra_new(func, pdev, nouveau_config, nouveau_debug,
true, true, ~0ULL, pdevice);
if (err)
goto err_free;
#define DRIVER_MAJOR 1
#define DRIVER_MINOR 3
-#define DRIVER_PATCHLEVEL 0
+#define DRIVER_PATCHLEVEL 1
/*
* 1.1.1:
* 1.3.0:
* - NVIF ABI modified, safe because only (current) users are test
* programs that get directly linked with NVKM.
+ * 1.3.1:
+ * - implemented limited ABI16/NVIF interop
*/
#include <nvif/client.h>
};
enum nouveau_drm_handle {
- NVDRM_CLIENT = 0xffffffff,
- NVDRM_DEVICE = 0xdddddddd,
- NVDRM_CONTROL = 0xdddddddc,
- NVDRM_DISPLAY = 0xd1500000,
- NVDRM_PUSH = 0xbbbb0000, /* |= client chid */
NVDRM_CHAN = 0xcccc0000, /* |= client chid */
NVDRM_NVSW = 0x55550000,
};
int nouveau_pmops_suspend(struct device *);
int nouveau_pmops_resume(struct device *);
+#include <nvkm/core/tegra.h>
+
struct drm_device *
-nouveau_platform_device_create(struct platform_device *, struct nvkm_device **);
+nouveau_platform_device_create(const struct nvkm_device_tegra_func *,
+ struct platform_device *, struct nvkm_device **);
void nouveau_drm_device_remove(struct drm_device *dev);
#define NV_PRINTK(l,c,f,a...) do { \
return 0;
}
+static int
+nouveau_fbcon_open(struct fb_info *info, int user)
+{
+ struct nouveau_fbdev *fbcon = info->par;
+ struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+ int ret = pm_runtime_get_sync(drm->dev->dev);
+ if (ret < 0 && ret != -EACCES)
+ return ret;
+ return 0;
+}
+
+static int
+nouveau_fbcon_release(struct fb_info *info, int user)
+{
+ struct nouveau_fbdev *fbcon = info->par;
+ struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+ pm_runtime_put(drm->dev->dev);
+ return 0;
+}
+
static struct fb_ops nouveau_fbcon_ops = {
.owner = THIS_MODULE,
+ .fb_open = nouveau_fbcon_open,
+ .fb_release = nouveau_fbcon_release,
.fb_check_var = drm_fb_helper_check_var,
.fb_set_par = drm_fb_helper_set_par,
.fb_fillrect = nouveau_fbcon_fillrect,
static struct fb_ops nouveau_fbcon_sw_ops = {
.owner = THIS_MODULE,
+ .fb_open = nouveau_fbcon_open,
+ .fb_release = nouveau_fbcon_release,
.fb_check_var = drm_fb_helper_check_var,
.fb_set_par = drm_fb_helper_set_par,
.fb_fillrect = drm_fb_helper_cfb_fillrect,
}
ret = pm_runtime_get_sync(dev);
- if (ret < 0 && ret != -EACCES)
+ if (ret < 0 && ret != -EACCES) {
+ kfree(vma);
goto out;
+ }
ret = nouveau_bo_vma_add(nvbo, cli->vm, vma);
if (ret)
struct nouveau_bo *nvbo = nouveau_gem_object(gem);
struct nvkm_vma *vma;
- if (nvbo->bo.mem.mem_type == TTM_PL_TT)
+ if (is_power_of_2(nvbo->valid_domains))
+ rep->domain = nvbo->valid_domains;
+ else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
rep->domain = NOUVEAU_GEM_DOMAIN_GART;
else
rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
-
rep->offset = nvbo->bo.offset;
if (cli->vm) {
vma = nouveau_bo_vma_find(nvbo, cli->vm);
nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
- struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+ struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct nouveau_abi16_chan *temp;
struct nouveau_drm *drm = nouveau_drm(dev);
return -ENOMEM;
list_for_each_entry(temp, &abi16->channels, head) {
- if (temp->chan->user.handle == (NVDRM_CHAN | req->channel)) {
+ if (temp->chan->chid == req->channel) {
chan = temp->chan;
break;
}
static int nouveau_platform_probe(struct platform_device *pdev)
{
+ const struct nvkm_device_tegra_func *func;
struct nvkm_device *device;
struct drm_device *drm;
int ret;
- drm = nouveau_platform_device_create(pdev, &device);
+ func = of_device_get_match_data(&pdev->dev);
+
+ drm = nouveau_platform_device_create(func, pdev, &device);
if (IS_ERR(drm))
return PTR_ERR(drm);
}
#if IS_ENABLED(CONFIG_OF)
+static const struct nvkm_device_tegra_func gk20a_platform_data = {
+ .iommu_bit = 34,
+};
+
static const struct of_device_id nouveau_platform_match[] = {
- { .compatible = "nvidia,gk20a" },
- { .compatible = "nvidia,gm20b" },
+ {
+ .compatible = "nvidia,gk20a",
+ .data = &gk20a_platform_data,
+ },
+ {
+ .compatible = "nvidia,gm20b",
+ .data = &gk20a_platform_data,
+ },
{ }
};
if (!sysfs)
return -ENOMEM;
- ret = nvif_object_init(&device->object, NVDRM_CONTROL,
- NVIF_IOCTL_NEW_V0_CONTROL, NULL, 0,
- &sysfs->ctrl);
+ ret = nvif_object_init(&device->object, 0, NVIF_IOCTL_NEW_V0_CONTROL,
+ NULL, 0, &sysfs->ctrl);
if (ret == 0)
device_create_file(nvxx_device(device)->dev, &dev_attr_pstate);
#include "nouveau_gem.h"
#include "drm_legacy.h"
+
+#include <core/tegra.h>
+
static int
nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
{
struct nvkm_device *device = nvxx_device(&drm->device);
struct nvkm_pci *pci = device->pci;
struct drm_device *dev = drm->dev;
- u32 bits;
+ u8 bits;
int ret;
if (pci && pci->agp.bridge) {
bits = nvxx_mmu(&drm->device)->dma_bits;
if (nvxx_device(&drm->device)->func->pci) {
if (drm->agp.bridge ||
- !pci_dma_supported(dev->pdev, DMA_BIT_MASK(bits)))
+ !dma_supported(dev->dev, DMA_BIT_MASK(bits)))
bits = 32;
+ } else if (device->func->tegra) {
+ struct nvkm_device_tegra *tegra = device->func->tegra(device);
- ret = pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(bits));
- if (ret)
- return ret;
+ /*
+ * If the platform can use a IOMMU, then the addressable DMA
+ * space is constrained by the IOMMU bit
+ */
+ if (tegra->func->iommu_bit)
+ bits = min(bits, tegra->func->iommu_bit);
- ret = pci_set_consistent_dma_mask(dev->pdev,
- DMA_BIT_MASK(bits));
- if (ret)
- pci_set_consistent_dma_mask(dev->pdev,
- DMA_BIT_MASK(32));
}
+ ret = dma_set_mask(dev->dev, DMA_BIT_MASK(bits));
+ if (ret)
+ return ret;
+
+ ret = dma_set_coherent_mask(dev->dev, DMA_BIT_MASK(bits));
+ if (ret)
+ dma_set_coherent_mask(dev->dev, DMA_BIT_MASK(32));
+
ret = nouveau_ttm_global_init(drm);
if (ret)
return ret;
#include "nouveau_drm.h"
#include "nouveau_usif.h"
+#include "nouveau_abi16.h"
#include <nvif/notify.h>
#include <nvif/unpack.h>
} else
goto done;
+ /* USIF slightly abuses some return-only ioctl members in order
+ * to provide interoperability with the older ABI16 objects
+ */
mutex_lock(&cli->mutex);
+ if (argv->v0.route) {
+ if (ret = -EINVAL, argv->v0.route == 0xff)
+ ret = nouveau_abi16_usif(filp, argv, argc);
+ if (ret) {
+ mutex_unlock(&cli->mutex);
+ goto done;
+ }
+ }
+
switch (argv->v0.type) {
case NVIF_IOCTL_V0_NEW:
- /* ... except if we're creating children */
- argv->v0.owner = NVIF_IOCTL_V0_OWNER_ANY;
ret = usif_object_new(filp, data, size, argv, argc);
break;
case NVIF_IOCTL_V0_NTFY_NEW:
const s32 *oclass, u8 head, void *data, u32 size,
struct nv50_chan *chan)
{
- const u32 handle = (oclass[0] << 16) | head;
struct nvif_sclass *sclass;
int ret, i, n;
while (oclass[0]) {
for (i = 0; i < n; i++) {
if (sclass[i].oclass == oclass[0]) {
- ret = nvif_object_init(disp, handle, oclass[0],
+ ret = nvif_object_init(disp, 0, oclass[0],
data, size, &chan->user);
if (ret == 0)
nvif_object_map(&chan->user);
if (!dmac->ptr)
return -ENOMEM;
- ret = nvif_object_init(&device->object, 0xd0000000,
- NV_DMA_FROM_MEMORY, &(struct nv_dma_v0) {
+ ret = nvif_object_init(&device->object, 0, NV_DMA_FROM_MEMORY,
+ &(struct nv_dma_v0) {
.target = NV_DMA_V0_TARGET_PCI_US,
.access = NV_DMA_V0_ACCESS_RD,
.start = dmac->handle + 0x0000,
.imem = nv40_instmem_new,
.mc = nv44_mc_new,
.mmu = nv44_mmu_new,
- .pci = nv4c_pci_new,
+ .pci = nv46_pci_new,
.therm = nv40_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = nv50_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv50_pci_new,
+ .pci = nv46_pci_new,
.therm = nv50_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = nv50_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv50_pci_new,
+ .pci = g84_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = nv50_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv50_pci_new,
+ .pci = g84_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = nv50_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv50_pci_new,
+ .pci = g84_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = nv50_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = nv50_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gt215_pmu_new,
.therm = gt215_therm_new,
.timer = nv41_timer_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gt215_pmu_new,
.therm = gt215_therm_new,
.timer = nv41_timer_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gt215_pmu_new,
.therm = gt215_therm_new,
.timer = nv41_timer_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.therm = g84_therm_new,
.timer = nv41_timer_new,
.volt = nv40_volt_new,
.mc = g98_mc_new,
.mmu = nv50_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gt215_pmu_new,
.therm = gt215_therm_new,
.timer = nv41_timer_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gf100_pmu_new,
.therm = gt215_therm_new,
.timer = nv41_timer_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gf100_pmu_new,
.therm = gt215_therm_new,
.timer = nv41_timer_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gf100_pmu_new,
.therm = gt215_therm_new,
.timer = nv41_timer_new,
.fuse = gf100_fuse_new,
.gpio = gf119_gpio_new,
.i2c = gf117_i2c_new,
- .ibus = gf100_ibus_new,
+ .ibus = gf117_ibus_new,
.imem = nv50_instmem_new,
.ltc = gf100_ltc_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
.ce[0] = gf100_ce_new,
.fuse = gf100_fuse_new,
.gpio = gf119_gpio_new,
.i2c = gf119_i2c_new,
- .ibus = gf100_ibus_new,
+ .ibus = gf117_ibus_new,
.imem = nv50_instmem_new,
.ltc = gf100_ltc_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gf119_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gk104_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
- .volt = nv40_volt_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[1] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gk104_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
- .volt = nv40_volt_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[1] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
- .pmu = gf119_pmu_new,
+ .pci = g94_pci_new,
+ .pmu = gk104_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
- .volt = nv40_volt_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[1] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gk110_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
- .volt = nv40_volt_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[1] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.mc = gf100_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gk110_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
- .volt = nv40_volt_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[1] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.mc = gk20a_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gk208_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
- .volt = nv40_volt_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[1] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.mc = gk20a_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gk208_pmu_new,
.therm = gf119_therm_new,
.timer = nv41_timer_new,
- .volt = nv40_volt_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[1] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.mc = gk20a_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gm107_pmu_new,
.therm = gm107_therm_new,
.timer = gk20a_timer_new,
+ .volt = gk104_volt_new,
.ce[0] = gk104_ce_new,
.ce[2] = gk104_ce_new,
.disp = gm107_disp_new,
.mc = gk20a_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gm107_pmu_new,
.timer = gk20a_timer_new,
+ .volt = gk104_volt_new,
.ce[0] = gm204_ce_new,
.ce[1] = gm204_ce_new,
.ce[2] = gm204_ce_new,
.mc = gk20a_mc_new,
.mmu = gf100_mmu_new,
.mxm = nv50_mxm_new,
- .pci = nv40_pci_new,
+ .pci = g94_pci_new,
.pmu = gm107_pmu_new,
.timer = gk20a_timer_new,
+ .volt = gk104_volt_new,
.ce[0] = gm204_ce_new,
.ce[1] = gm204_ce_new,
.ce[2] = gm204_ce_new,
{}
};
+static const struct nvkm_device_pci_vendor
+nvkm_device_pci_10de_0fcd[] = {
+ { 0x17aa, 0x3801, NULL, { .War00C800_0 = true } }, /* Lenovo Y510P */
+ {}
+};
+
static const struct nvkm_device_pci_vendor
nvkm_device_pci_10de_0fd2[] = {
{ 0x1028, 0x0595, "GeForce GT 640M LE" },
static const struct nvkm_device_pci_vendor
nvkm_device_pci_10de_1199[] = {
{ 0x1458, 0xd001, "GeForce GTX 760" },
+ { 0x1462, 0x1106, "GeForce GTX 780M", { .War00C800_0 = true } }, /* Medion Erazer X7827 */
{}
};
{ 0x0fc6, "GeForce GTX 650" },
{ 0x0fc8, "GeForce GT 740" },
{ 0x0fc9, "GeForce GT 730" },
- { 0x0fcd, "GeForce GT 755M" },
+ { 0x0fcd, "GeForce GT 755M", nvkm_device_pci_10de_0fcd },
{ 0x0fce, "GeForce GT 640M LE" },
{ 0x0fd1, "GeForce GT 650M" },
{ 0x0fd2, "GeForce GT 640M", nvkm_device_pci_10de_0fd2 },
unsigned long pgsize_bitmap;
int ret;
+ if (!tdev->func->iommu_bit)
+ return;
+
mutex_init(&tdev->iommu.mutex);
if (iommu_present(&platform_bus_type)) {
goto free_domain;
ret = nvkm_mm_init(&tdev->iommu.mm, 0,
- (1ULL << 40) >> tdev->iommu.pgshift, 1);
+ (1ULL << tdev->func->iommu_bit) >>
+ tdev->iommu.pgshift, 1);
if (ret)
goto detach_device;
}
};
int
-nvkm_device_tegra_new(struct platform_device *pdev,
+nvkm_device_tegra_new(const struct nvkm_device_tegra_func *func,
+ struct platform_device *pdev,
const char *cfg, const char *dbg,
bool detect, bool mmio, u64 subdev_mask,
struct nvkm_device **pdevice)
if (!(tdev = kzalloc(sizeof(*tdev), GFP_KERNEL)))
return -ENOMEM;
*pdevice = &tdev->device;
+ tdev->func = func;
tdev->pdev = pdev;
tdev->irq = -1;
}
#else
int
-nvkm_device_tegra_new(struct platform_device *pdev,
+nvkm_device_tegra_new(const struct nvkm_device_tegra_func *func,
+ struct platform_device *pdev,
const char *cfg, const char *dbg,
bool detect, bool mmio, u64 subdev_mask,
struct nvkm_device **pdevice)
return -EINVAL;
}
-static struct nvkm_object_func
+static const struct nvkm_object_func
nv04_disp_root = {
.mthd = nv04_disp_mthd,
.ntfy = nvkm_disp_ntfy,
{ 0x0d, "GPR_OUT_OF_BOUNDS" },
{ 0x0e, "MEM_OUT_OF_BOUNDS" },
{ 0x0f, "UNALIGNED_MEM_ACCESS" },
+ { 0x10, "INVALID_ADDR_SPACE" },
{ 0x11, "INVALID_PARAM" },
{}
};
{ -1, -1, FERMI_B, &gf100_fermi },
{ -1, -1, FERMI_C, &gf100_fermi },
{ -1, -1, FERMI_COMPUTE_A },
+ { -1, -1, FERMI_COMPUTE_B },
{}
}
};
{ -1, -1, FERMI_B, &gf100_fermi },
{ -1, -1, FERMI_C, &gf100_fermi },
{ -1, -1, FERMI_COMPUTE_A },
+ { -1, -1, FERMI_COMPUTE_B },
{}
}
};
{ -1, -1, FERMI_B, &gf100_fermi },
{ -1, -1, FERMI_C, &gf100_fermi },
{ -1, -1, FERMI_COMPUTE_A },
+ { -1, -1, FERMI_COMPUTE_B },
{}
}
};
return perfmon;
}
-static struct nvkm_object_func
+static const struct nvkm_object_func
nvkm_perfmon = {
.dtor = nvkm_perfmon_dtor,
.mthd = nvkm_perfmon_mthd,
index = NVKM_I2C_BUS_PRI;
if (init->outp && init->outp->i2c_upper_default)
index = NVKM_I2C_BUS_SEC;
+ } else
+ if (index == 0x80) {
+ index = NVKM_I2C_BUS_PRI;
+ } else
+ if (index == 0x81) {
+ index = NVKM_I2C_BUS_SEC;
}
bus = nvkm_i2c_bus_find(i2c, index);
return data;
}
-u32
-nvbios_pmuTp(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
- struct nvbios_pmuT *info)
-{
- u32 data = nvbios_pmuTe(bios, ver, hdr, cnt, len);
- memset(info, 0x00, sizeof(*info));
- switch (!!data * *ver) {
- default:
- break;
- }
- return data;
-}
-
u32
nvbios_pmuEe(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr)
{
void *(*init)(struct nvkm_bios *, const char *);
void (*fini)(void *);
u32 (*read)(void *, u32 offset, u32 length, struct nvkm_bios *);
+ u32 (*size)(void *);
bool rw;
+ bool ignore_checksum;
+ bool no_pcir;
};
int nvbios_extend(struct nvkm_bios *, u32 length);
p->ramcfg_DLLoff = (nvbios_rd08(bios, data + 0x03) & 0x04) >> 2;
p->ramcfg_00_03_08 = (nvbios_rd08(bios, data + 0x03) & 0x08) >> 3;
p->ramcfg_RON = (nvbios_rd08(bios, data + 0x03) & 0x10) >> 3;
+ p->ramcfg_FBVDDQ = (nvbios_rd08(bios, data + 0x03) & 0x80) >> 7;
p->ramcfg_00_04_02 = (nvbios_rd08(bios, data + 0x04) & 0x02) >> 1;
p->ramcfg_00_04_04 = (nvbios_rd08(bios, data + 0x04) & 0x04) >> 2;
p->ramcfg_00_04_20 = (nvbios_rd08(bios, data + 0x04) & 0x20) >> 5;
p->ramcfg_DLLoff = (nvbios_rd08(bios, data + 0x02) & 0x40) >> 6;
p->ramcfg_10_03_0f = (nvbios_rd08(bios, data + 0x03) & 0x0f) >> 0;
p->ramcfg_10_04_01 = (nvbios_rd08(bios, data + 0x04) & 0x01) >> 0;
+ p->ramcfg_FBVDDQ = (nvbios_rd08(bios, data + 0x04) & 0x08) >> 3;
p->ramcfg_10_05 = (nvbios_rd08(bios, data + 0x05) & 0xff) >> 0;
p->ramcfg_10_06 = (nvbios_rd08(bios, data + 0x06) & 0xff) >> 0;
p->ramcfg_10_07 = (nvbios_rd08(bios, data + 0x07) & 0xff) >> 0;
p->ramcfg_11_01_04 = (nvbios_rd08(bios, data + 0x01) & 0x04) >> 2;
p->ramcfg_11_01_08 = (nvbios_rd08(bios, data + 0x01) & 0x08) >> 3;
p->ramcfg_11_01_10 = (nvbios_rd08(bios, data + 0x01) & 0x10) >> 4;
- p->ramcfg_11_01_20 = (nvbios_rd08(bios, data + 0x01) & 0x20) >> 5;
+ p->ramcfg_DLLoff = (nvbios_rd08(bios, data + 0x01) & 0x20) >> 5;
p->ramcfg_11_01_40 = (nvbios_rd08(bios, data + 0x01) & 0x40) >> 6;
p->ramcfg_11_01_80 = (nvbios_rd08(bios, data + 0x01) & 0x80) >> 7;
p->ramcfg_11_02_03 = (nvbios_rd08(bios, data + 0x02) & 0x03) >> 0;
u32 read = mthd->func->read(data, start, limit - start, bios);
bios->size = start + read;
}
- return bios->size >= limit;
+ return bios->size >= upto;
}
static int
struct nvbios_image image;
int score = 1;
- if (!shadow_fetch(bios, mthd, offset + 0x1000)) {
- nvkm_debug(subdev, "%08x: header fetch failed\n", offset);
- return 0;
- }
+ if (mthd->func->no_pcir) {
+ image.base = 0;
+ image.type = 0;
+ image.size = mthd->func->size(mthd->data);
+ image.last = 1;
+ } else {
+ if (!shadow_fetch(bios, mthd, offset + 0x1000)) {
+ nvkm_debug(subdev, "%08x: header fetch failed\n",
+ offset);
+ return 0;
+ }
- if (!nvbios_image(bios, idx, &image)) {
- nvkm_debug(subdev, "image %d invalid\n", idx);
- return 0;
+ if (!nvbios_image(bios, idx, &image)) {
+ nvkm_debug(subdev, "image %d invalid\n", idx);
+ return 0;
+ }
}
nvkm_debug(subdev, "%08x: type %02x, %d bytes\n",
image.base, image.type, image.size);
switch (image.type) {
case 0x00:
- if (nvbios_checksum(&bios->data[image.base], image.size)) {
+ if (!mthd->func->ignore_checksum &&
+ nvbios_checksum(&bios->data[image.base], image.size)) {
nvkm_debug(subdev, "%08x: checksum failed\n",
image.base);
if (mthd->func->rw)
*
*/
#include "priv.h"
+
#include <core/pci.h>
#if defined(__powerpc__)
of_read(void *data, u32 offset, u32 length, struct nvkm_bios *bios)
{
struct priv *priv = data;
- if (offset + length <= priv->size) {
+ if (offset < priv->size) {
+ length = min_t(u32, length, priv->size - offset);
memcpy_fromio(bios->data + offset, priv->data + offset, length);
return length;
}
return 0;
}
+static u32
+of_size(void *data)
+{
+ struct priv *priv = data;
+ return priv->size;
+}
+
static void *
of_init(struct nvkm_bios *bios, const char *name)
{
- struct pci_dev *pdev = bios->subdev.device->func->pci(bios->subdev.device)->pdev;
+ struct nvkm_device *device = bios->subdev.device;
+ struct pci_dev *pdev = device->func->pci(device)->pdev;
struct device_node *dn;
struct priv *priv;
if (!(dn = pci_device_to_OF_node(pdev)))
.init = of_init,
.fini = (void(*)(void *))kfree,
.read = of_read,
+ .size = of_size,
.rw = false,
+ .ignore_checksum = true,
+ .no_pcir = true,
};
#else
const struct nvbios_source
memset(info, 0x00, sizeof(*info));
switch (!!volt * *ver) {
case 0x12:
+ info->type = NVBIOS_VOLT_GPIO;
info->vidmask = nvbios_rd08(bios, volt + 0x04);
break;
case 0x20:
+ info->type = NVBIOS_VOLT_GPIO;
info->vidmask = nvbios_rd08(bios, volt + 0x05);
break;
case 0x30:
+ info->type = NVBIOS_VOLT_GPIO;
info->vidmask = nvbios_rd08(bios, volt + 0x04);
break;
case 0x40:
+ info->type = NVBIOS_VOLT_GPIO;
info->base = nvbios_rd32(bios, volt + 0x04);
info->step = nvbios_rd16(bios, volt + 0x08);
info->vidmask = nvbios_rd08(bios, volt + 0x0b);
info->max = info->base;
break;
case 0x50:
- info->vidmask = nvbios_rd08(bios, volt + 0x06);
info->min = nvbios_rd32(bios, volt + 0x0a);
info->max = nvbios_rd32(bios, volt + 0x0e);
info->base = nvbios_rd32(bios, volt + 0x12) & 0x00ffffff;
- info->step = nvbios_rd16(bios, volt + 0x16);
+
+ /* offset 4 seems to be a flag byte */
+ if (nvbios_rd32(bios, volt + 0x4) & 1) {
+ info->type = NVBIOS_VOLT_PWM;
+ info->pwm_freq = nvbios_rd32(bios, volt + 0x5) / 1000;
+ info->pwm_range = nvbios_rd32(bios, volt + 0x16);
+ } else {
+ info->type = NVBIOS_VOLT_GPIO;
+ info->vidmask = nvbios_rd08(bios, volt + 0x06);
+ info->step = nvbios_rd16(bios, volt + 0x16);
+ }
break;
}
return volt;
hwsq_cmd(hwsq, 3, (u8[]){ 0x5f, flag, data });
}
+void
+nvkm_hwsq_wait_vblank(struct nvkm_hwsq *hwsq)
+{
+ struct nvkm_subdev *subdev = hwsq->subdev;
+ struct nvkm_device *device = subdev->device;
+ u32 heads, x, y, px = 0;
+ int i, head_sync;
+
+ heads = nvkm_rd32(device, 0x610050);
+ for (i = 0; i < 2; i++) {
+ /* Heuristic: sync to head with biggest resolution */
+ if (heads & (2 << (i << 3))) {
+ x = nvkm_rd32(device, 0x610b40 + (0x540 * i));
+ y = (x & 0xffff0000) >> 16;
+ x &= 0x0000ffff;
+ if ((x * y) > px) {
+ px = (x * y);
+ head_sync = i;
+ }
+ }
+ }
+
+ if (px == 0) {
+ nvkm_debug(subdev, "WAIT VBLANK !NO ACTIVE HEAD\n");
+ return;
+ }
+
+ nvkm_debug(subdev, "WAIT VBLANK HEAD%d\n", head_sync);
+ nvkm_hwsq_wait(hwsq, head_sync ? 0x3 : 0x1, 0x0);
+ nvkm_hwsq_wait(hwsq, head_sync ? 0x3 : 0x1, 0x1);
+}
+
void
nvkm_hwsq_nsec(struct nvkm_hwsq *hwsq, u32 nsec)
{
nvkm_hwsq_wait(ram->hwsq, flag, data);
}
+static inline void
+hwsq_wait_vblank(struct hwsq *ram)
+{
+ nvkm_hwsq_wait_vblank(ram->hwsq);
+}
+
static inline void
hwsq_nsec(struct hwsq *ram, u32 nsec)
{
g84_clk_new(struct nvkm_device *device, int index, struct nvkm_clk **pclk)
{
return nv50_clk_new_(&g84_clk, device, index,
- (device->chipset == 0xa0), pclk);
+ (device->chipset >= 0x94), pclk);
}
{ 5, 2 }, { 7, 4 }, { 8, 5 }, { 9, 6 }, { 10, 7 },
{ 11, 0 }, { 13 , 1 },
/* the below are mentioned in some, but not all, gddr3 docs */
- { 4, 1 }, { 6, 3 }, { 12, 1 },
+ { 4, 0 }, { 6, 3 }, { 12, 1 },
{ -1 }
};
WR = (ram->next->bios.timing[2] & 0x007f0000) >> 16;
/* XXX: Get these values from the VBIOS instead */
DLL = !(ram->mr[1] & 0x1);
- ODT = (ram->mr[1] & 0x004) >> 2 |
- (ram->mr[1] & 0x040) >> 5 |
- (ram->mr[1] & 0x200) >> 7;
RON = !(ram->mr[1] & 0x300) >> 8;
break;
default:
return -ENOSYS;
}
+ if (ram->next->bios.timing_ver == 0x20 ||
+ ram->next->bios.ramcfg_timing == 0xff) {
+ ODT = (ram->mr[1] & 0xc) >> 2;
+ }
+
hi = ram->mr[2] & 0x1;
CL = ramxlat(hi ? ramgddr3_cl_hi : ramgddr3_cl_lo, CL);
WR = ramxlat(ramgddr3_wr_lo, WR);
int WL, CL, WR, at[2], dt, ds;
int rq = ram->freq < 1000000; /* XXX */
+ xd = !ram->next->bios.ramcfg_DLLoff;
+
switch (ram->next->bios.ramcfg_ver) {
case 0x11:
pd = ram->next->bios.ramcfg_11_01_80;
lf = ram->next->bios.ramcfg_11_01_40;
- xd = !ram->next->bios.ramcfg_11_01_20;
vh = ram->next->bios.ramcfg_11_02_10;
vr = ram->next->bios.ramcfg_11_02_04;
vo = ram->next->bios.ramcfg_11_06;
* DDR3
******************************************************************************/
+static void
+nvkm_sddr3_dll_reset(struct gk104_ramfuc *fuc)
+{
+ ram_nuke(fuc, mr[0]);
+ ram_mask(fuc, mr[0], 0x100, 0x100);
+ ram_mask(fuc, mr[0], 0x100, 0x000);
+}
+
+static void
+nvkm_sddr3_dll_disable(struct gk104_ramfuc *fuc)
+{
+ u32 mr1_old = ram_rd32(fuc, mr[1]);
+
+ if (!(mr1_old & 0x1)) {
+ ram_mask(fuc, mr[1], 0x1, 0x1);
+ ram_nsec(fuc, 1000);
+ }
+}
+
static int
gk104_ram_calc_sddr3(struct gk104_ram *ram, u32 freq)
{
ram_mask(fuc, 0x10f808, 0x04000000, 0x04000000);
ram_wr32(fuc, 0x10f314, 0x00000001); /* PRECHARGE */
+
+ if (next->bios.ramcfg_DLLoff)
+ nvkm_sddr3_dll_disable(fuc);
+
ram_wr32(fuc, 0x10f210, 0x00000000); /* REFRESH_AUTO = 0 */
ram_wr32(fuc, 0x10f310, 0x00000001); /* REFRESH */
ram_mask(fuc, 0x10f200, 0x80000000, 0x80000000);
ram_wr32(fuc, 0x10f210, 0x80000000); /* REFRESH_AUTO = 1 */
ram_nsec(fuc, 1000);
- ram_nuke(fuc, mr[0]);
- ram_mask(fuc, mr[0], 0x100, 0x100);
- ram_mask(fuc, mr[0], 0x100, 0x000);
+ if (!next->bios.ramcfg_DLLoff) {
+ ram_mask(fuc, mr[1], 0x1, 0x0);
+ nvkm_sddr3_dll_reset(fuc);
+ }
- ram_mask(fuc, mr[2], 0xfff, ram->base.mr[2]);
+ ram_mask(fuc, mr[2], 0x00000fff, ram->base.mr[2]);
+ ram_mask(fuc, mr[1], 0xffffffff, ram->base.mr[1]);
ram_wr32(fuc, mr[0], ram->base.mr[0]);
ram_nsec(fuc, 1000);
- ram_nuke(fuc, mr[0]);
- ram_mask(fuc, mr[0], 0x100, 0x100);
- ram_mask(fuc, mr[0], 0x100, 0x000);
+ if (!next->bios.ramcfg_DLLoff) {
+ nvkm_sddr3_dll_reset(fuc);
+ ram_nsec(fuc, 1000);
+ }
if (vc == 0 && ram_have(fuc, gpio2E)) {
u32 temp = ram_mask(fuc, gpio2E, 0x3000, fuc->r_func2E[0]);
return -EINVAL;
}
+static int
+gk104_calc_pll_output(int fN, int M, int N, int P, int clk)
+{
+ return ((clk * N) + (((u16)(fN + 4096) * clk) >> 13)) / (M * P);
+}
+
+static int
+gk104_pll_calc_hiclk(int target_khz, int crystal,
+ int *N1, int *fN1, int *M1, int *P1,
+ int *N2, int *M2, int *P2)
+{
+ int best_clk = 0, best_err = target_khz, p_ref, n_ref;
+ bool upper = false;
+
+ *M1 = 1;
+ /* M has to be 1, otherwise it gets unstable */
+ *M2 = 1;
+ /* can be 1 or 2, sticking with 1 for simplicity */
+ *P2 = 1;
+
+ for (p_ref = 0x7; p_ref >= 0x5; --p_ref) {
+ for (n_ref = 0x25; n_ref <= 0x2b; ++n_ref) {
+ int cur_N, cur_clk, cur_err;
+
+ cur_clk = gk104_calc_pll_output(0, 1, n_ref, p_ref, crystal);
+ cur_N = target_khz / cur_clk;
+ cur_err = target_khz
+ - gk104_calc_pll_output(0xf000, 1, cur_N, 1, cur_clk);
+
+ /* we found a better combination */
+ if (cur_err < best_err) {
+ best_err = cur_err;
+ best_clk = cur_clk;
+ *N2 = cur_N;
+ *N1 = n_ref;
+ *P1 = p_ref;
+ upper = false;
+ }
+
+ cur_N += 1;
+ cur_err = gk104_calc_pll_output(0xf000, 1, cur_N, 1, cur_clk)
+ - target_khz;
+ if (cur_err < best_err) {
+ best_err = cur_err;
+ best_clk = cur_clk;
+ *N2 = cur_N;
+ *N1 = n_ref;
+ *P1 = p_ref;
+ upper = true;
+ }
+ }
+ }
+
+ /* adjust fN to get closer to the target clock */
+ *fN1 = (u16)((((best_err / *N2 * *P2) * (*P1 * *M1)) << 13) / crystal);
+ if (upper)
+ *fN1 = (u16)(1 - *fN1);
+
+ return gk104_calc_pll_output(*fN1, 1, *N1, *P1, crystal);
+}
+
static int
gk104_ram_calc_xits(struct gk104_ram *ram, struct nvkm_ram_data *next)
{
* kepler boards, no idea how/why they're chosen.
*/
refclk = next->freq;
- if (ram->mode == 2)
- refclk = fuc->mempll.refclk;
-
- /* calculate refpll coefficients */
- ret = gt215_pll_calc(subdev, &fuc->refpll, refclk, &ram->N1,
- &ram->fN1, &ram->M1, &ram->P1);
- fuc->mempll.refclk = ret;
- if (ret <= 0) {
- nvkm_error(subdev, "unable to calc refpll\n");
- return -EINVAL;
- }
-
- /* calculate mempll coefficients, if we're using it */
if (ram->mode == 2) {
- /* post-divider doesn't work... the reg takes the values but
- * appears to completely ignore it. there *is* a bit at
- * bit 28 that appears to divide the clock by 2 if set.
- */
- fuc->mempll.min_p = 1;
- fuc->mempll.max_p = 2;
-
- ret = gt215_pll_calc(subdev, &fuc->mempll, next->freq,
- &ram->N2, NULL, &ram->M2, &ram->P2);
+ ret = gk104_pll_calc_hiclk(next->freq, subdev->device->crystal,
+ &ram->N1, &ram->fN1, &ram->M1, &ram->P1,
+ &ram->N2, &ram->M2, &ram->P2);
+ fuc->mempll.refclk = ret;
+ if (ret <= 0) {
+ nvkm_error(subdev, "unable to calc plls\n");
+ return -EINVAL;
+ }
+ nvkm_debug(subdev, "sucessfully calced PLLs for clock %i kHz"
+ " (refclock: %i kHz)\n", next->freq, ret);
+ } else {
+ /* calculate refpll coefficients */
+ ret = gt215_pll_calc(subdev, &fuc->refpll, refclk, &ram->N1,
+ &ram->fN1, &ram->M1, &ram->P1);
+ fuc->mempll.refclk = ret;
if (ret <= 0) {
- nvkm_error(subdev, "unable to calc mempll\n");
+ nvkm_error(subdev, "unable to calc refpll\n");
return -EINVAL;
}
}
break;
case NVKM_RAM_TYPE_DDR3:
ram->fuc.r_mr[0] = ramfuc_reg(0x10f300);
+ ram->fuc.r_mr[1] = ramfuc_reg(0x10f304);
ram->fuc.r_mr[2] = ramfuc_reg(0x10f320);
break;
default:
#include <subdev/clk/gt215.h>
#include <subdev/gpio.h>
-/* XXX: Remove when memx gains GPIO support */
-extern int nv50_gpio_location(int line, u32 *reg, u32 *shift);
-
struct gt215_ramfuc {
struct ramfuc base;
struct ramfuc_reg r_0x001610;
struct ramfuc_reg r_0x111400;
struct ramfuc_reg r_0x611200;
struct ramfuc_reg r_mr[4];
- struct ramfuc_reg r_gpioFBVREF;
+ struct ramfuc_reg r_gpio[4];
};
struct gt215_ltrain {
}
static void
-gt215_ram_fbvref(struct gt215_ramfuc *fuc, u32 val)
+gt215_ram_gpio(struct gt215_ramfuc *fuc, u8 tag, u32 val)
{
struct nvkm_gpio *gpio = fuc->base.fb->subdev.device->gpio;
struct dcb_gpio_func func;
u32 reg, sh, gpio_val;
int ret;
- if (nvkm_gpio_get(gpio, 0, 0x2e, DCB_GPIO_UNUSED) != val) {
- ret = nvkm_gpio_find(gpio, 0, 0x2e, DCB_GPIO_UNUSED, &func);
+ if (nvkm_gpio_get(gpio, 0, tag, DCB_GPIO_UNUSED) != val) {
+ ret = nvkm_gpio_find(gpio, 0, tag, DCB_GPIO_UNUSED, &func);
if (ret)
return;
- nv50_gpio_location(func.line, ®, &sh);
- gpio_val = ram_rd32(fuc, gpioFBVREF);
+ reg = func.line >> 3;
+ sh = (func.line & 0x7) << 2;
+ gpio_val = ram_rd32(fuc, gpio[reg]);
if (gpio_val & (8 << sh))
val = !val;
+ if (!(func.log[1] & 1))
+ val = !val;
- ram_mask(fuc, gpioFBVREF, (0x3 << sh), ((val | 0x2) << sh));
+ ram_mask(fuc, gpio[reg], (0x3 << sh), ((val | 0x2) << sh));
ram_nsec(fuc, 20000);
}
}
struct nvkm_device *device = subdev->device;
struct nvkm_bios *bios = device->bios;
struct gt215_clk_info mclk;
+ struct nvkm_gpio *gpio = device->gpio;
struct nvkm_ram_data *next;
u8 ver, hdr, cnt, len, strap;
u32 data;
break;
}
- if (fuc->r_gpioFBVREF.addr && next->bios.timing_10_ODT)
- gt215_ram_fbvref(fuc, 0);
+ if (next->bios.timing_10_ODT)
+ gt215_ram_gpio(fuc, 0x2e, 1);
/* Brace RAM for impact */
ram_wr32(fuc, 0x1002d4, 0x00000001);
if (device->chipset == 0xa3 && freq <= 500000)
ram_mask(fuc, 0x100700, 0x00000006, 0x00000006);
+ /* Alter FBVDD/Q, apparently must be done with PLL disabled, thus
+ * set it to bypass */
+ if (nvkm_gpio_get(gpio, 0, 0x18, DCB_GPIO_UNUSED) ==
+ next->bios.ramcfg_FBVDDQ) {
+ data = ram_rd32(fuc, 0x004000) & 0x9;
+
+ if (data == 0x1)
+ ram_mask(fuc, 0x004000, 0x8, 0x8);
+ if (data & 0x1)
+ ram_mask(fuc, 0x004000, 0x1, 0x0);
+
+ gt215_ram_gpio(fuc, 0x18, !next->bios.ramcfg_FBVDDQ);
+
+ if (data & 0x1)
+ ram_mask(fuc, 0x004000, 0x1, 0x1);
+ }
+
/* Fiddle with clocks */
/* There's 4 scenario's
* pll->pll: first switch to a 324MHz clock, set up new PLL, switch
unk71c = ram_rd32(fuc, 0x10071c) & ~0x00000100;
r111100 = ram_rd32(fuc, 0x111100) & ~0x3a800000;
- if (next->bios.ramcfg_10_02_04) {
- switch (ram->base.type) {
- case NVKM_RAM_TYPE_DDR3:
- if (device->chipset != 0xa8)
- r111100 |= 0x00000004;
- /* no break */
- case NVKM_RAM_TYPE_DDR2:
- r111100 |= 0x08000000;
- break;
- default:
- break;
- }
- } else {
- switch (ram->base.type) {
- case NVKM_RAM_TYPE_DDR2:
- r111100 |= 0x1a800000;
+ /* NVA8 seems to skip various bits related to ramcfg_10_02_04 */
+ if (device->chipset == 0xa8) {
+ r111100 |= 0x08000000;
+ if (!next->bios.ramcfg_10_02_04)
unk714 |= 0x00000010;
- break;
- case NVKM_RAM_TYPE_DDR3:
- if (device->chipset == 0xa8) {
- r111100 |= 0x08000000;
- } else {
- r111100 &= ~0x00000004;
+ } else {
+ if (next->bios.ramcfg_10_02_04) {
+ switch (ram->base.type) {
+ case NVKM_RAM_TYPE_DDR2:
+ case NVKM_RAM_TYPE_DDR3:
+ r111100 &= ~0x00000020;
+ if (next->bios.ramcfg_10_02_10)
+ r111100 |= 0x08000004;
+ else
+ r111100 |= 0x00000024;
+ break;
+ default:
+ break;
+ }
+ } else {
+ switch (ram->base.type) {
+ case NVKM_RAM_TYPE_DDR2:
+ case NVKM_RAM_TYPE_DDR3:
+ r111100 &= ~0x00000024;
r111100 |= 0x12800000;
+
+ if (next->bios.ramcfg_10_02_10)
+ r111100 |= 0x08000000;
+ unk714 |= 0x00000010;
+ break;
+ case NVKM_RAM_TYPE_GDDR3:
+ r111100 |= 0x30000000;
+ unk714 |= 0x00000020;
+ break;
+ default:
+ break;
}
- unk714 |= 0x00000010;
- break;
- case NVKM_RAM_TYPE_GDDR3:
- r111100 |= 0x30000000;
- unk714 |= 0x00000020;
- break;
- default:
- break;
}
}
ram_mask(fuc, 0x100718, 0xffffffff, unk718);
ram_mask(fuc, 0x111100, 0xffffffff, r111100);
- if (fuc->r_gpioFBVREF.addr && !next->bios.timing_10_ODT)
- gt215_ram_fbvref(fuc, 1);
+ if (!next->bios.timing_10_ODT)
+ gt215_ram_gpio(fuc, 0x2e, 0);
/* Reset DLL */
if (!next->bios.ramcfg_DLLoff)
int
gt215_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
{
- struct nvkm_gpio *gpio = fb->subdev.device->gpio;
- struct dcb_gpio_func func;
struct gt215_ram *ram;
- u32 reg, shift;
int ret, i;
if (!(ram = kzalloc(sizeof(*ram), GFP_KERNEL)))
ram->fuc.r_mr[2] = ramfuc_reg(0x1002e0);
ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);
}
-
- ret = nvkm_gpio_find(gpio, 0, 0x2e, DCB_GPIO_UNUSED, &func);
- if (ret == 0) {
- nv50_gpio_location(func.line, ®, &shift);
- ram->fuc.r_gpioFBVREF = ramfuc_reg(reg);
- }
+ ram->fuc.r_gpio[0] = ramfuc_reg(0x00e104);
+ ram->fuc.r_gpio[1] = ramfuc_reg(0x00e108);
+ ram->fuc.r_gpio[2] = ramfuc_reg(0x00e120);
+ ram->fuc.r_gpio[3] = ramfuc_reg(0x00e124);
return 0;
}
#include <subdev/bios/rammap.h>
#include <subdev/bios/timing.h>
#include <subdev/clk/pll.h>
+#include <subdev/gpio.h>
struct nv50_ramseq {
struct hwsq base;
struct hwsq_reg r_0x611200;
struct hwsq_reg r_timing[9];
struct hwsq_reg r_mr[4];
+ struct hwsq_reg r_gpio[4];
};
struct nv50_ram {
nvkm_debug(subdev, " 240: %08x\n", timing[8]);
return 0;
}
+
+static int
+nv50_ram_timing_read(struct nv50_ram *ram, u32 *timing)
+{
+ unsigned int i;
+ struct nvbios_ramcfg *cfg = &ram->base.target.bios;
+ struct nvkm_subdev *subdev = &ram->base.fb->subdev;
+ struct nvkm_device *device = subdev->device;
+
+ for (i = 0; i <= 8; i++)
+ timing[i] = nvkm_rd32(device, 0x100220 + (i * 4));
+
+ /* Derive the bare minimum for the MR calculation to succeed */
+ cfg->timing_ver = 0x10;
+ T(CL) = (timing[3] & 0xff) + 1;
+
+ switch (ram->base.type) {
+ case NVKM_RAM_TYPE_DDR2:
+ T(CWL) = T(CL) - 1;
+ break;
+ case NVKM_RAM_TYPE_GDDR3:
+ T(CWL) = ((timing[2] & 0xff000000) >> 24) + 1;
+ break;
+ default:
+ return -ENOSYS;
+ break;
+ }
+
+ T(WR) = ((timing[1] >> 24) & 0xff) - 1 - T(CWL);
+
+ return 0;
+}
#undef T
static void
ram_nsec(hwsq, 24000);
}
+static void
+nv50_ram_gpio(struct nv50_ramseq *hwsq, u8 tag, u32 val)
+{
+ struct nvkm_gpio *gpio = hwsq->base.subdev->device->gpio;
+ struct dcb_gpio_func func;
+ u32 reg, sh, gpio_val;
+ int ret;
+
+ if (nvkm_gpio_get(gpio, 0, tag, DCB_GPIO_UNUSED) != val) {
+ ret = nvkm_gpio_find(gpio, 0, tag, DCB_GPIO_UNUSED, &func);
+ if (ret)
+ return;
+
+ reg = func.line >> 3;
+ sh = (func.line & 0x7) << 2;
+ gpio_val = ram_rd32(hwsq, gpio[reg]);
+
+ if (gpio_val & (8 << sh))
+ val = !val;
+ if (!(func.log[1] & 1))
+ val = !val;
+
+ ram_mask(hwsq, gpio[reg], (0x3 << sh), ((val | 0x2) << sh));
+ ram_nsec(hwsq, 20000);
+ }
+}
+
static int
nv50_ram_calc(struct nvkm_ram *base, u32 freq)
{
strap, data, ver, hdr);
return -EINVAL;
}
+ nv50_ram_timing_calc(ram, timing);
+ } else {
+ nv50_ram_timing_read(ram, timing);
}
- nv50_ram_timing_calc(ram, timing);
-
ret = ram_init(hwsq, subdev);
if (ret)
return ret;
break;
}
- if (ret)
+ if (ret) {
+ nvkm_error(subdev, "Could not calculate MR\n");
return ret;
+ }
+
+ if (subdev->device->chipset <= 0x96 && !next->bios.ramcfg_00_03_02)
+ ram_mask(hwsq, 0x100710, 0x00000200, 0x00000000);
/* Always disable this bit during reclock */
ram_mask(hwsq, 0x100200, 0x00000800, 0x00000000);
- ram_wait(hwsq, 0x01, 0x00); /* wait for !vblank */
- ram_wait(hwsq, 0x01, 0x01); /* wait for vblank */
+ ram_wait_vblank(hwsq);
ram_wr32(hwsq, 0x611200, 0x00003300);
ram_wr32(hwsq, 0x002504, 0x00000001); /* block fifo */
ram_nsec(hwsq, 8000);
ram_wait(hwsq, 0x00, 0x01); /* wait for fb disabled */
ram_nsec(hwsq, 2000);
+ if (next->bios.timing_10_ODT)
+ nv50_ram_gpio(hwsq, 0x2e, 1);
+
ram_wr32(hwsq, 0x1002d4, 0x00000001); /* precharge */
ram_wr32(hwsq, 0x1002d0, 0x00000001); /* refresh */
ram_wr32(hwsq, 0x1002d0, 0x00000001); /* refresh */
next->bios.rammap_00_16_40 << 14);
ram_mask(hwsq, 0x00400c, 0x0000ffff, (N1 << 8) | M1);
ram_mask(hwsq, 0x004008, 0x91ff0000, r004008);
- if (subdev->device->chipset >= 0x96)
+
+ /* XXX: GDDR3 only? */
+ if (subdev->device->chipset >= 0x92)
ram_wr32(hwsq, 0x100da0, r100da0);
+
+ nv50_ram_gpio(hwsq, 0x18, !next->bios.ramcfg_FBVDDQ);
ram_nsec(hwsq, 64000); /*XXX*/
ram_nsec(hwsq, 32000); /*XXX*/
ram_mask(hwsq, 0x100200, 0x00001000, !next->bios.ramcfg_00_04_02 << 12);
/* XXX: A lot of this could be "chipset"/"ram type" specific stuff */
- unk710 = ram_rd32(hwsq, 0x100710) & ~0x00000101;
+ unk710 = ram_rd32(hwsq, 0x100710) & ~0x00000100;
unk714 = ram_rd32(hwsq, 0x100714) & ~0xf0000020;
unk718 = ram_rd32(hwsq, 0x100718) & ~0x00000100;
unk71c = ram_rd32(hwsq, 0x10071c) & ~0x00000100;
+ if (subdev->device->chipset <= 0x96) {
+ unk710 &= ~0x0000006e;
+ unk714 &= ~0x00000100;
+
+ if (!next->bios.ramcfg_00_03_08)
+ unk710 |= 0x00000060;
+ if (!next->bios.ramcfg_FBVDDQ)
+ unk714 |= 0x00000100;
+ if ( next->bios.ramcfg_00_04_04)
+ unk710 |= 0x0000000e;
+ } else {
+ unk710 &= ~0x00000001;
+
+ if (!next->bios.ramcfg_00_03_08)
+ unk710 |= 0x00000001;
+ }
if ( next->bios.ramcfg_00_03_01)
unk71c |= 0x00000100;
if ( next->bios.ramcfg_00_03_02)
unk710 |= 0x00000100;
- if (!next->bios.ramcfg_00_03_08) {
- unk710 |= 0x1;
- unk714 |= 0x20;
- }
+ if (!next->bios.ramcfg_00_03_08)
+ unk714 |= 0x00000020;
if ( next->bios.ramcfg_00_04_04)
unk714 |= 0x70000000;
if ( next->bios.ramcfg_00_04_20)
ram_mask(hwsq, 0x100718, 0xffffffff, unk718);
ram_mask(hwsq, 0x100710, 0xffffffff, unk710);
+ /* XXX: G94 does not even test these regs in trace. Harmless we do it,
+ * but why is it omitted? */
if (next->bios.rammap_00_16_20) {
ram_wr32(hwsq, 0x1005a0, next->bios.ramcfg_00_07 << 16 |
next->bios.ramcfg_00_06 << 8 |
}
ram_mask(hwsq, mr[1], 0xffffffff, ram->base.mr[1]);
+ if (!next->bios.timing_10_ODT)
+ nv50_ram_gpio(hwsq, 0x2e, 0);
+
/* Reset DLL */
if (!next->bios.ramcfg_DLLoff)
nvkm_sddr2_dll_reset(hwsq);
ram_mask(hwsq, 0x004008, 0x00004000, 0x00000000);
if (next->bios.ramcfg_00_03_02)
ram_mask(hwsq, 0x10021c, 0x00010000, 0x00010000);
+ if (subdev->device->chipset <= 0x96 && next->bios.ramcfg_00_03_02)
+ ram_mask(hwsq, 0x100710, 0x00000200, 0x00000200);
return 0;
}
ram->hwsq.r_mr[3] = hwsq_reg(0x1002e4);
}
+ ram->hwsq.r_gpio[0] = hwsq_reg(0x00e104);
+ ram->hwsq.r_gpio[1] = hwsq_reg(0x00e108);
+ ram->hwsq.r_gpio[2] = hwsq_reg(0x00e120);
+ ram->hwsq.r_gpio[3] = hwsq_reg(0x00e124);
+
return 0;
}
#define ram_mask(s,r,m,d) hwsq_mask(&(s)->base, &(s)->r_##r, (m), (d))
#define ram_setf(s,f,d) hwsq_setf(&(s)->base, (f), (d))
#define ram_wait(s,f,d) hwsq_wait(&(s)->base, (f), (d))
+#define ram_wait_vblank(s) hwsq_wait_vblank(&(s)->base)
#define ram_nsec(s,n) hwsq_nsec(&(s)->base, (n))
#endif
return -ENOSYS;
}
+ if (ram->next->bios.timing_ver == 0x20 ||
+ ram->next->bios.ramcfg_timing == 0xff) {
+ ODT = (ram->mr[1] & 0x004) >> 2 |
+ (ram->mr[1] & 0x040) >> 5;
+ }
+
CL = ramxlat(ramddr2_cl, CL);
WR = ramxlat(ramddr2_wr, WR);
if (CL < 0 || WR < 0)
{
int CWL, CL, WR, DLL = 0, ODT = 0;
+ DLL = !ram->next->bios.ramcfg_DLLoff;
+
switch (ram->next->bios.timing_ver) {
case 0x10:
if (ram->next->bios.timing_hdr < 0x17) {
CWL = ram->next->bios.timing_10_CWL;
CL = ram->next->bios.timing_10_CL;
WR = ram->next->bios.timing_10_WR;
- DLL = !ram->next->bios.ramcfg_DLLoff;
ODT = ram->next->bios.timing_10_ODT;
break;
case 0x20:
CL = (ram->next->bios.timing[1] & 0x0000001f) >> 0;
WR = (ram->next->bios.timing[2] & 0x007f0000) >> 16;
/* XXX: Get these values from the VBIOS instead */
- DLL = !(ram->mr[1] & 0x1);
ODT = (ram->mr[1] & 0x004) >> 2 |
(ram->mr[1] & 0x040) >> 5 |
(ram->mr[1] & 0x200) >> 7;
}
}
-int
+static int
nv50_gpio_location(int line, u32 *reg, u32 *shift)
{
const u32 nv50_gpio_reg[4] = { 0xe104, 0xe108, 0xe280, 0xe284 };
nvkm-y += nvkm/subdev/ibus/gf100.o
+nvkm-y += nvkm/subdev/ibus/gf117.o
nvkm-y += nvkm/subdev/ibus/gk104.o
nvkm-y += nvkm/subdev/ibus/gk20a.o
*
* Authors: Ben Skeggs
*/
-#include <subdev/ibus.h>
+#include "priv.h"
static void
gf100_ibus_intr_hub(struct nvkm_subdev *ibus, int i)
nvkm_mask(device, 0x128128 + (i * 0x0400), 0x00000200, 0x00000000);
}
-static void
+void
gf100_ibus_intr(struct nvkm_subdev *ibus)
{
struct nvkm_device *device = ibus->device;
}
}
+static int
+gf100_ibus_init(struct nvkm_subdev *ibus)
+{
+ struct nvkm_device *device = ibus->device;
+ nvkm_mask(device, 0x122310, 0x0003ffff, 0x00000800);
+ nvkm_wr32(device, 0x12232c, 0x00100064);
+ nvkm_wr32(device, 0x122330, 0x00100064);
+ nvkm_wr32(device, 0x122334, 0x00100064);
+ nvkm_mask(device, 0x122348, 0x0003ffff, 0x00000100);
+ return 0;
+}
+
static const struct nvkm_subdev_func
gf100_ibus = {
+ .init = gf100_ibus_init,
.intr = gf100_ibus_intr,
};
--- /dev/null
+/*
+ * Copyright 2015 Samuel Pitosiet
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Samuel Pitoiset
+ */
+#include "priv.h"
+
+static int
+gf117_ibus_init(struct nvkm_subdev *ibus)
+{
+ struct nvkm_device *device = ibus->device;
+ nvkm_mask(device, 0x122310, 0x0003ffff, 0x00000800);
+ nvkm_mask(device, 0x122348, 0x0003ffff, 0x00000100);
+ nvkm_mask(device, 0x1223b0, 0x0003ffff, 0x00000fff);
+ return 0;
+}
+
+static const struct nvkm_subdev_func
+gf117_ibus = {
+ .init = gf117_ibus_init,
+ .intr = gf100_ibus_intr,
+};
+
+int
+gf117_ibus_new(struct nvkm_device *device, int index,
+ struct nvkm_subdev **pibus)
+{
+ struct nvkm_subdev *ibus;
+ if (!(ibus = *pibus = kzalloc(sizeof(*ibus), GFP_KERNEL)))
+ return -ENOMEM;
+ nvkm_subdev_ctor(&gf117_ibus, device, index, 0, ibus);
+ return 0;
+}
--- /dev/null
+#ifndef __NVKM_IBUS_PRIV_H__
+#define __NVKM_IBUS_PRIV_H__
+
+#include <subdev/ibus.h>
+
+void gf100_ibus_intr(struct nvkm_subdev *);
+#endif
/*
* GK20A does not have dedicated video memory, and to accurately represent this
* fact Nouveau will not create a RAM device for it. Therefore its instmem
- * implementation must be done directly on top of system memory, while providing
- * coherent read and write operations.
+ * implementation must be done directly on top of system memory, while
+ * preserving coherency for read and write operations.
*
* Instmem can be allocated through two means:
- * 1) If an IOMMU mapping has been probed, the IOMMU API is used to make memory
+ * 1) If an IOMMU unit has been probed, the IOMMU API is used to make memory
* pages contiguous to the GPU. This is the preferred way.
- * 2) If no IOMMU mapping is probed, the DMA API is used to allocate physically
+ * 2) If no IOMMU unit is probed, the DMA API is used to allocate physically
* contiguous memory.
*
- * In both cases CPU read and writes are performed using PRAMIN (i.e. using the
- * GPU path) to ensure these operations are coherent for the GPU. This allows us
- * to use more "relaxed" allocation parameters when using the DMA API, since we
- * never need a kernel mapping.
+ * In both cases CPU read and writes are performed by creating a write-combined
+ * mapping. The GPU L2 cache must thus be flushed/invalidated when required. To
+ * be conservative we do this every time we acquire or release an instobj, but
+ * ideally L2 management should be handled at a higher level.
+ *
+ * To improve performance, CPU mappings are not removed upon instobj release.
+ * Instead they are placed into a LRU list to be recycled when the mapped space
+ * goes beyond a certain threshold. At the moment this limit is 1MB.
*/
-#define gk20a_instmem(p) container_of((p), struct gk20a_instmem, base)
#include "priv.h"
#include <core/memory.h>
#include <core/mm.h>
#include <core/tegra.h>
#include <subdev/fb.h>
-
-#define gk20a_instobj(p) container_of((p), struct gk20a_instobj, memory)
+#include <subdev/ltc.h>
struct gk20a_instobj {
struct nvkm_memory memory;
- struct gk20a_instmem *imem;
struct nvkm_mem mem;
+ struct gk20a_instmem *imem;
+
+ /* CPU mapping */
+ u32 *vaddr;
+ struct list_head vaddr_node;
};
+#define gk20a_instobj(p) container_of((p), struct gk20a_instobj, memory)
/*
* Used for objects allocated using the DMA API
struct gk20a_instobj_dma {
struct gk20a_instobj base;
- void *cpuaddr;
+ u32 *cpuaddr;
dma_addr_t handle;
struct nvkm_mm_node r;
};
+#define gk20a_instobj_dma(p) \
+ container_of(gk20a_instobj(p), struct gk20a_instobj_dma, base)
/*
* Used for objects flattened using the IOMMU API
struct gk20a_instobj_iommu {
struct gk20a_instobj base;
- /* array of base.mem->size pages */
+ /* will point to the higher half of pages */
+ dma_addr_t *dma_addrs;
+ /* array of base.mem->size pages (+ dma_addr_ts) */
struct page *pages[];
};
+#define gk20a_instobj_iommu(p) \
+ container_of(gk20a_instobj(p), struct gk20a_instobj_iommu, base)
struct gk20a_instmem {
struct nvkm_instmem base;
- unsigned long lock_flags;
+
+ /* protects vaddr_* and gk20a_instobj::vaddr* */
spinlock_t lock;
- u64 addr;
+
+ /* CPU mappings LRU */
+ unsigned int vaddr_use;
+ unsigned int vaddr_max;
+ struct list_head vaddr_lru;
/* Only used if IOMMU if present */
struct mutex *mm_mutex;
struct nvkm_mm *mm;
struct iommu_domain *domain;
unsigned long iommu_pgshift;
+ u16 iommu_bit;
/* Only used by DMA API */
struct dma_attrs attrs;
+
+ void __iomem * (*cpu_map)(struct nvkm_memory *);
};
+#define gk20a_instmem(p) container_of((p), struct gk20a_instmem, base)
static enum nvkm_memory_target
gk20a_instobj_target(struct nvkm_memory *memory)
gk20a_instobj_addr(struct nvkm_memory *memory)
{
return gk20a_instobj(memory)->mem.offset;
-
}
static u64
return (u64)gk20a_instobj(memory)->mem.size << 12;
}
+static void __iomem *
+gk20a_instobj_cpu_map_dma(struct nvkm_memory *memory)
+{
+ struct gk20a_instobj_dma *node = gk20a_instobj_dma(memory);
+ struct device *dev = node->base.imem->base.subdev.device->dev;
+ int npages = nvkm_memory_size(memory) >> 12;
+ struct page *pages[npages];
+ int i;
+
+ /* phys_to_page does not exist on all platforms... */
+ pages[0] = pfn_to_page(dma_to_phys(dev, node->handle) >> PAGE_SHIFT);
+ for (i = 1; i < npages; i++)
+ pages[i] = pages[0] + i;
+
+ return vmap(pages, npages, VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+}
+
+static void __iomem *
+gk20a_instobj_cpu_map_iommu(struct nvkm_memory *memory)
+{
+ struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory);
+ int npages = nvkm_memory_size(memory) >> 12;
+
+ return vmap(node->pages, npages, VM_MAP,
+ pgprot_writecombine(PAGE_KERNEL));
+}
+
+/*
+ * Must be called while holding gk20a_instmem_lock
+ */
+static void
+gk20a_instmem_vaddr_gc(struct gk20a_instmem *imem, const u64 size)
+{
+ while (imem->vaddr_use + size > imem->vaddr_max) {
+ struct gk20a_instobj *obj;
+
+ /* no candidate that can be unmapped, abort... */
+ if (list_empty(&imem->vaddr_lru))
+ break;
+
+ obj = list_first_entry(&imem->vaddr_lru, struct gk20a_instobj,
+ vaddr_node);
+ list_del(&obj->vaddr_node);
+ vunmap(obj->vaddr);
+ obj->vaddr = NULL;
+ imem->vaddr_use -= nvkm_memory_size(&obj->memory);
+ nvkm_debug(&imem->base.subdev, "(GC) vaddr used: %x/%x\n",
+ imem->vaddr_use, imem->vaddr_max);
+
+ }
+}
+
static void __iomem *
gk20a_instobj_acquire(struct nvkm_memory *memory)
{
- struct gk20a_instmem *imem = gk20a_instobj(memory)->imem;
+ struct gk20a_instobj *node = gk20a_instobj(memory);
+ struct gk20a_instmem *imem = node->imem;
+ struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;
+ const u64 size = nvkm_memory_size(memory);
unsigned long flags;
+
+ nvkm_ltc_flush(ltc);
+
spin_lock_irqsave(&imem->lock, flags);
- imem->lock_flags = flags;
- return NULL;
+
+ if (node->vaddr) {
+ /* remove us from the LRU list since we cannot be unmapped */
+ list_del(&node->vaddr_node);
+
+ goto out;
+ }
+
+ /* try to free some address space if we reached the limit */
+ gk20a_instmem_vaddr_gc(imem, size);
+
+ node->vaddr = imem->cpu_map(memory);
+
+ if (!node->vaddr) {
+ nvkm_error(&imem->base.subdev, "cannot map instobj - "
+ "this is not going to end well...\n");
+ goto out;
+ }
+
+ imem->vaddr_use += size;
+ nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n",
+ imem->vaddr_use, imem->vaddr_max);
+
+out:
+ spin_unlock_irqrestore(&imem->lock, flags);
+
+ return node->vaddr;
}
static void
gk20a_instobj_release(struct nvkm_memory *memory)
{
- struct gk20a_instmem *imem = gk20a_instobj(memory)->imem;
- spin_unlock_irqrestore(&imem->lock, imem->lock_flags);
-}
+ struct gk20a_instobj *node = gk20a_instobj(memory);
+ struct gk20a_instmem *imem = node->imem;
+ struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;
+ unsigned long flags;
-/*
- * Use PRAMIN to read/write data and avoid coherency issues.
- * PRAMIN uses the GPU path and ensures data will always be coherent.
- *
- * A dynamic mapping based solution would be desirable in the future, but
- * the issue remains of how to maintain coherency efficiently. On ARM it is
- * not easy (if possible at all?) to create uncached temporary mappings.
- */
+ spin_lock_irqsave(&imem->lock, flags);
+
+ /* add ourselves to the LRU list so our CPU mapping can be freed */
+ list_add_tail(&node->vaddr_node, &imem->vaddr_lru);
+
+ spin_unlock_irqrestore(&imem->lock, flags);
+
+ wmb();
+ nvkm_ltc_invalidate(ltc);
+}
static u32
gk20a_instobj_rd32(struct nvkm_memory *memory, u64 offset)
{
struct gk20a_instobj *node = gk20a_instobj(memory);
- struct gk20a_instmem *imem = node->imem;
- struct nvkm_device *device = imem->base.subdev.device;
- u64 base = (node->mem.offset + offset) & 0xffffff00000ULL;
- u64 addr = (node->mem.offset + offset) & 0x000000fffffULL;
- u32 data;
-
- if (unlikely(imem->addr != base)) {
- nvkm_wr32(device, 0x001700, base >> 16);
- imem->addr = base;
- }
- data = nvkm_rd32(device, 0x700000 + addr);
- return data;
+
+ return node->vaddr[offset / 4];
}
static void
gk20a_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
{
struct gk20a_instobj *node = gk20a_instobj(memory);
- struct gk20a_instmem *imem = node->imem;
- struct nvkm_device *device = imem->base.subdev.device;
- u64 base = (node->mem.offset + offset) & 0xffffff00000ULL;
- u64 addr = (node->mem.offset + offset) & 0x000000fffffULL;
- if (unlikely(imem->addr != base)) {
- nvkm_wr32(device, 0x001700, base >> 16);
- imem->addr = base;
- }
- nvkm_wr32(device, 0x700000 + addr, data);
+ node->vaddr[offset / 4] = data;
}
static void
gk20a_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset)
{
struct gk20a_instobj *node = gk20a_instobj(memory);
+
nvkm_vm_map_at(vma, offset, &node->mem);
}
+/*
+ * Clear the CPU mapping of an instobj if it exists
+ */
static void
-gk20a_instobj_dtor_dma(struct gk20a_instobj *_node)
+gk20a_instobj_dtor(struct gk20a_instobj *node)
+{
+ struct gk20a_instmem *imem = node->imem;
+ struct gk20a_instobj *obj;
+ unsigned long flags;
+
+ spin_lock_irqsave(&imem->lock, flags);
+
+ if (!node->vaddr)
+ goto out;
+
+ list_for_each_entry(obj, &imem->vaddr_lru, vaddr_node) {
+ if (obj == node) {
+ list_del(&obj->vaddr_node);
+ break;
+ }
+ }
+ vunmap(node->vaddr);
+ node->vaddr = NULL;
+ imem->vaddr_use -= nvkm_memory_size(&node->memory);
+ nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n",
+ imem->vaddr_use, imem->vaddr_max);
+
+out:
+ spin_unlock_irqrestore(&imem->lock, flags);
+}
+
+static void *
+gk20a_instobj_dtor_dma(struct nvkm_memory *memory)
{
- struct gk20a_instobj_dma *node = (void *)_node;
- struct gk20a_instmem *imem = _node->imem;
+ struct gk20a_instobj_dma *node = gk20a_instobj_dma(memory);
+ struct gk20a_instmem *imem = node->base.imem;
struct device *dev = imem->base.subdev.device->dev;
+ gk20a_instobj_dtor(&node->base);
+
if (unlikely(!node->cpuaddr))
- return;
+ goto out;
- dma_free_attrs(dev, _node->mem.size << PAGE_SHIFT, node->cpuaddr,
+ dma_free_attrs(dev, node->base.mem.size << PAGE_SHIFT, node->cpuaddr,
node->handle, &imem->attrs);
+
+out:
+ return node;
}
-static void
-gk20a_instobj_dtor_iommu(struct gk20a_instobj *_node)
+static void *
+gk20a_instobj_dtor_iommu(struct nvkm_memory *memory)
{
- struct gk20a_instobj_iommu *node = (void *)_node;
- struct gk20a_instmem *imem = _node->imem;
+ struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory);
+ struct gk20a_instmem *imem = node->base.imem;
+ struct device *dev = imem->base.subdev.device->dev;
struct nvkm_mm_node *r;
int i;
- if (unlikely(list_empty(&_node->mem.regions)))
- return;
+ gk20a_instobj_dtor(&node->base);
- r = list_first_entry(&_node->mem.regions, struct nvkm_mm_node,
+ if (unlikely(list_empty(&node->base.mem.regions)))
+ goto out;
+
+ r = list_first_entry(&node->base.mem.regions, struct nvkm_mm_node,
rl_entry);
- /* clear bit 34 to unmap pages */
- r->offset &= ~BIT(34 - imem->iommu_pgshift);
+ /* clear IOMMU bit to unmap pages */
+ r->offset &= ~BIT(imem->iommu_bit - imem->iommu_pgshift);
/* Unmap pages from GPU address space and free them */
- for (i = 0; i < _node->mem.size; i++) {
+ for (i = 0; i < node->base.mem.size; i++) {
iommu_unmap(imem->domain,
(r->offset + i) << imem->iommu_pgshift, PAGE_SIZE);
+ dma_unmap_page(dev, node->dma_addrs[i], PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
__free_page(node->pages[i]);
}
mutex_lock(imem->mm_mutex);
nvkm_mm_free(imem->mm, &r);
mutex_unlock(imem->mm_mutex);
-}
-
-static void *
-gk20a_instobj_dtor(struct nvkm_memory *memory)
-{
- struct gk20a_instobj *node = gk20a_instobj(memory);
- struct gk20a_instmem *imem = node->imem;
-
- if (imem->domain)
- gk20a_instobj_dtor_iommu(node);
- else
- gk20a_instobj_dtor_dma(node);
+out:
return node;
}
static const struct nvkm_memory_func
-gk20a_instobj_func = {
- .dtor = gk20a_instobj_dtor,
+gk20a_instobj_func_dma = {
+ .dtor = gk20a_instobj_dtor_dma,
+ .target = gk20a_instobj_target,
+ .addr = gk20a_instobj_addr,
+ .size = gk20a_instobj_size,
+ .acquire = gk20a_instobj_acquire,
+ .release = gk20a_instobj_release,
+ .rd32 = gk20a_instobj_rd32,
+ .wr32 = gk20a_instobj_wr32,
+ .map = gk20a_instobj_map,
+};
+
+static const struct nvkm_memory_func
+gk20a_instobj_func_iommu = {
+ .dtor = gk20a_instobj_dtor_iommu,
.target = gk20a_instobj_target,
.addr = gk20a_instobj_addr,
.size = gk20a_instobj_size,
return -ENOMEM;
*_node = &node->base;
+ nvkm_memory_ctor(&gk20a_instobj_func_dma, &node->base.memory);
+
node->cpuaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT,
&node->handle, GFP_KERNEL,
&imem->attrs);
{
struct gk20a_instobj_iommu *node;
struct nvkm_subdev *subdev = &imem->base.subdev;
+ struct device *dev = subdev->device->dev;
struct nvkm_mm_node *r;
int ret;
int i;
- if (!(node = kzalloc(sizeof(*node) +
- sizeof( node->pages[0]) * npages, GFP_KERNEL)))
+ /*
+ * despite their variable size, instmem allocations are small enough
+ * (< 1 page) to be handled by kzalloc
+ */
+ if (!(node = kzalloc(sizeof(*node) + ((sizeof(node->pages[0]) +
+ sizeof(*node->dma_addrs)) * npages), GFP_KERNEL)))
return -ENOMEM;
*_node = &node->base;
+ node->dma_addrs = (void *)(node->pages + npages);
+
+ nvkm_memory_ctor(&gk20a_instobj_func_iommu, &node->base.memory);
/* Allocate backing memory */
for (i = 0; i < npages; i++) {
struct page *p = alloc_page(GFP_KERNEL);
+ dma_addr_t dma_adr;
if (p == NULL) {
ret = -ENOMEM;
goto free_pages;
}
node->pages[i] = p;
+ dma_adr = dma_map_page(dev, p, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, dma_adr)) {
+ nvkm_error(subdev, "DMA mapping error!\n");
+ ret = -ENOMEM;
+ goto free_pages;
+ }
+ node->dma_addrs[i] = dma_adr;
}
mutex_lock(imem->mm_mutex);
align >> imem->iommu_pgshift, &r);
mutex_unlock(imem->mm_mutex);
if (ret) {
- nvkm_error(subdev, "virtual space is full!\n");
+ nvkm_error(subdev, "IOMMU space is full!\n");
goto free_pages;
}
/* Map into GPU address space */
for (i = 0; i < npages; i++) {
- struct page *p = node->pages[i];
u32 offset = (r->offset + i) << imem->iommu_pgshift;
- ret = iommu_map(imem->domain, offset, page_to_phys(p),
+ ret = iommu_map(imem->domain, offset, node->dma_addrs[i],
PAGE_SIZE, IOMMU_READ | IOMMU_WRITE);
if (ret < 0) {
nvkm_error(subdev, "IOMMU mapping failure: %d\n", ret);
}
}
- /* Bit 34 tells that an address is to be resolved through the IOMMU */
- r->offset |= BIT(34 - imem->iommu_pgshift);
+ /* IOMMU bit tells that an address is to be resolved through the IOMMU */
+ r->offset |= BIT(imem->iommu_bit - imem->iommu_pgshift);
node->base.mem.offset = ((u64)r->offset) << imem->iommu_pgshift;
mutex_unlock(imem->mm_mutex);
free_pages:
- for (i = 0; i < npages && node->pages[i] != NULL; i++)
+ for (i = 0; i < npages && node->pages[i] != NULL; i++) {
+ dma_addr_t dma_addr = node->dma_addrs[i];
+ if (dma_addr)
+ dma_unmap_page(dev, dma_addr, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
__free_page(node->pages[i]);
+ }
return ret;
}
struct nvkm_memory **pmemory)
{
struct gk20a_instmem *imem = gk20a_instmem(base);
- struct gk20a_instobj *node = NULL;
struct nvkm_subdev *subdev = &imem->base.subdev;
+ struct gk20a_instobj *node = NULL;
int ret;
nvkm_debug(subdev, "%s (%s): size: %x align: %x\n", __func__,
if (ret)
return ret;
- nvkm_memory_ctor(&gk20a_instobj_func, &node->memory);
node->imem = imem;
/* present memory for being mapped using small pages */
return 0;
}
-static void
-gk20a_instmem_fini(struct nvkm_instmem *base)
+static void *
+gk20a_instmem_dtor(struct nvkm_instmem *base)
{
- gk20a_instmem(base)->addr = ~0ULL;
+ struct gk20a_instmem *imem = gk20a_instmem(base);
+
+ /* perform some sanity checks... */
+ if (!list_empty(&imem->vaddr_lru))
+ nvkm_warn(&base->subdev, "instobj LRU not empty!\n");
+
+ if (imem->vaddr_use != 0)
+ nvkm_warn(&base->subdev, "instobj vmap area not empty! "
+ "0x%x bytes still mapped\n", imem->vaddr_use);
+
+ return imem;
}
static const struct nvkm_instmem_func
gk20a_instmem = {
- .fini = gk20a_instmem_fini,
+ .dtor = gk20a_instmem_dtor,
.memory_new = gk20a_instobj_new,
.persistent = true,
.zero = false,
spin_lock_init(&imem->lock);
*pimem = &imem->base;
+ /* do not allow more than 1MB of CPU-mapped instmem */
+ imem->vaddr_use = 0;
+ imem->vaddr_max = 0x100000;
+ INIT_LIST_HEAD(&imem->vaddr_lru);
+
if (tdev->iommu.domain) {
- imem->domain = tdev->iommu.domain;
+ imem->mm_mutex = &tdev->iommu.mutex;
imem->mm = &tdev->iommu.mm;
+ imem->domain = tdev->iommu.domain;
imem->iommu_pgshift = tdev->iommu.pgshift;
- imem->mm_mutex = &tdev->iommu.mutex;
+ imem->cpu_map = gk20a_instobj_cpu_map_iommu;
+ imem->iommu_bit = tdev->func->iommu_bit;
nvkm_info(&imem->base.subdev, "using IOMMU\n");
} else {
init_dma_attrs(&imem->attrs);
- /*
- * We will access instmem through PRAMIN and thus do not need a
- * consistent CPU pointer or kernel mapping
- */
+ /* We will access the memory through our own mapping */
dma_set_attr(DMA_ATTR_NON_CONSISTENT, &imem->attrs);
dma_set_attr(DMA_ATTR_WEAK_ORDERING, &imem->attrs);
dma_set_attr(DMA_ATTR_WRITE_COMBINE, &imem->attrs);
dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &imem->attrs);
+ imem->cpu_map = gk20a_instobj_cpu_map_dma;
nvkm_info(&imem->base.subdev, "using DMA API\n");
}
return index;
}
+void
+nvkm_ltc_invalidate(struct nvkm_ltc *ltc)
+{
+ if (ltc->func->invalidate)
+ ltc->func->invalidate(ltc);
+}
+
+void
+nvkm_ltc_flush(struct nvkm_ltc *ltc)
+{
+ if (ltc->func->flush)
+ ltc->func->flush(ltc);
+}
+
static void
nvkm_ltc_intr(struct nvkm_subdev *subdev)
{
}
}
+void
+gf100_ltc_invalidate(struct nvkm_ltc *ltc)
+{
+ struct nvkm_device *device = ltc->subdev.device;
+ s64 taken;
+
+ nvkm_wr32(device, 0x70004, 0x00000001);
+ taken = nvkm_wait_msec(device, 2, 0x70004, 0x00000003, 0x00000000);
+ if (taken < 0)
+ nvkm_warn(<c->subdev, "LTC invalidate timeout\n");
+
+ if (taken > 0)
+ nvkm_debug(<c->subdev, "LTC invalidate took %lld ns\n", taken);
+}
+
+void
+gf100_ltc_flush(struct nvkm_ltc *ltc)
+{
+ struct nvkm_device *device = ltc->subdev.device;
+ s64 taken;
+
+ nvkm_wr32(device, 0x70010, 0x00000001);
+ taken = nvkm_wait_msec(device, 2, 0x70010, 0x00000003, 0x00000000);
+ if (taken < 0)
+ nvkm_warn(<c->subdev, "LTC flush timeout\n");
+
+ if (taken > 0)
+ nvkm_debug(<c->subdev, "LTC flush took %lld ns\n", taken);
+}
+
/* TODO: Figure out tag memory details and drop the over-cautious allocation.
*/
int
.zbc = 16,
.zbc_clear_color = gf100_ltc_zbc_clear_color,
.zbc_clear_depth = gf100_ltc_zbc_clear_depth,
+ .invalidate = gf100_ltc_invalidate,
+ .flush = gf100_ltc_flush,
};
int
.zbc = 16,
.zbc_clear_color = gf100_ltc_zbc_clear_color,
.zbc_clear_depth = gf100_ltc_zbc_clear_depth,
+ .invalidate = gf100_ltc_invalidate,
+ .flush = gf100_ltc_flush,
};
int
.zbc = 16,
.zbc_clear_color = gm107_ltc_zbc_clear_color,
.zbc_clear_depth = gm107_ltc_zbc_clear_depth,
+ .invalidate = gf100_ltc_invalidate,
+ .flush = gf100_ltc_flush,
};
int
int zbc;
void (*zbc_clear_color)(struct nvkm_ltc *, int, const u32[4]);
void (*zbc_clear_depth)(struct nvkm_ltc *, int, const u32);
+
+ void (*invalidate)(struct nvkm_ltc *);
+ void (*flush)(struct nvkm_ltc *);
};
int gf100_ltc_oneinit(struct nvkm_ltc *);
void gf100_ltc_cbc_wait(struct nvkm_ltc *);
void gf100_ltc_zbc_clear_color(struct nvkm_ltc *, int, const u32[4]);
void gf100_ltc_zbc_clear_depth(struct nvkm_ltc *, int, const u32);
+void gf100_ltc_invalidate(struct nvkm_ltc *);
+void gf100_ltc_flush(struct nvkm_ltc *);
#endif
nvkm-y += nvkm/subdev/pci/base.o
nvkm-y += nvkm/subdev/pci/nv04.o
nvkm-y += nvkm/subdev/pci/nv40.o
+nvkm-y += nvkm/subdev/pci/nv46.o
nvkm-y += nvkm/subdev/pci/nv4c.o
-nvkm-y += nvkm/subdev/pci/nv50.o
+nvkm-y += nvkm/subdev/pci/g84.o
+nvkm-y += nvkm/subdev/pci/g94.o
nvkm-y += nvkm/subdev/pci/gf100.o
nvkm_device_agp_quirks[] = {
/* VIA Apollo PRO133x / GeForce FX 5600 Ultra - fdo#20341 */
{ PCI_VENDOR_ID_VIA, 0x0691, PCI_VENDOR_ID_NVIDIA, 0x0311, 2 },
+ /* SiS 761 does not support AGP cards, use PCI mode */
+ { PCI_VENDOR_ID_SI, 0x0761, PCI_ANY_ID, PCI_ANY_ID, 0 },
{},
};
while (quirk->hostbridge_vendor) {
if (info.device->vendor == quirk->hostbridge_vendor &&
info.device->device == quirk->hostbridge_device &&
- pci->pdev->vendor == quirk->chip_vendor &&
- pci->pdev->device == quirk->chip_device) {
+ (quirk->chip_vendor == (u16)PCI_ANY_ID ||
+ pci->pdev->vendor == quirk->chip_vendor) &&
+ (quirk->chip_device == (u16)PCI_ANY_ID ||
+ pci->pdev->device == quirk->chip_device)) {
nvkm_info(subdev, "forcing default agp mode to %dX, "
"use NvAGP=<mode> to override\n",
quirk->mode);
pci->func->wr32(pci, addr, data);
}
+u32
+nvkm_pci_mask(struct nvkm_pci *pci, u16 addr, u32 mask, u32 value)
+{
+ u32 data = pci->func->rd32(pci, addr);
+ pci->func->wr32(pci, addr, (data & ~mask) | value);
+ return data;
+}
+
void
nvkm_pci_rom_shadow(struct nvkm_pci *pci, bool shadow)
{
return ret;
}
+ if (pci->func->init)
+ pci->func->init(pci);
+
ret = request_irq(pdev->irq, nvkm_pci_intr, IRQF_SHARED, "nvkm", pci);
if (ret)
return ret;
--- /dev/null
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+#include <core/pci.h>
+
+void
+g84_pci_init(struct nvkm_pci *pci)
+{
+ /* The following only concerns PCIe cards. */
+ if (!pci_is_pcie(pci->pdev))
+ return;
+
+ /* Tag field is 8-bit long, regardless of EXT_TAG.
+ * However, if EXT_TAG is disabled, only the lower 5 bits of the tag
+ * field should be used, limiting the number of request to 32.
+ *
+ * Apparently, 0x041c stores some limit on the number of requests
+ * possible, so if EXT_TAG is disabled, limit that requests number to
+ * 32
+ *
+ * Fixes fdo#86537
+ */
+ if (nvkm_pci_rd32(pci, 0x007c) & 0x00000020)
+ nvkm_pci_mask(pci, 0x0080, 0x00000100, 0x00000100);
+ else
+ nvkm_pci_mask(pci, 0x041c, 0x00000060, 0x00000000);
+}
+
+static const struct nvkm_pci_func
+g84_pci_func = {
+ .init = g84_pci_init,
+ .rd32 = nv40_pci_rd32,
+ .wr08 = nv40_pci_wr08,
+ .wr32 = nv40_pci_wr32,
+ .msi_rearm = nv46_pci_msi_rearm,
+};
+
+int
+g84_pci_new(struct nvkm_device *device, int index, struct nvkm_pci **ppci)
+{
+ return nvkm_pci_new_(&g84_pci_func, device, index, ppci);
+}
--- /dev/null
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+static const struct nvkm_pci_func
+g94_pci_func = {
+ .init = g84_pci_init,
+ .rd32 = nv40_pci_rd32,
+ .wr08 = nv40_pci_wr08,
+ .wr32 = nv40_pci_wr32,
+ .msi_rearm = nv40_pci_msi_rearm,
+};
+
+int
+g94_pci_new(struct nvkm_device *device, int index, struct nvkm_pci **ppci)
+{
+ return nvkm_pci_new_(&g94_pci_func, device, index, ppci);
+}
static const struct nvkm_pci_func
gf100_pci_func = {
+ .init = g84_pci_init,
.rd32 = nv40_pci_rd32,
.wr08 = nv40_pci_wr08,
.wr32 = nv40_pci_wr32,
nvkm_wr32(device, 0x088000 + addr, data);
}
-static void
+void
nv40_pci_msi_rearm(struct nvkm_pci *pci)
{
nvkm_pci_wr08(pci, 0x0068, 0xff);
--- /dev/null
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+#include <core/pci.h>
+
+/* MSI re-arm through the PRI appears to be broken on NV46/NV50/G84/G86/G92,
+ * so we access it via alternate PCI config space mechanisms.
+ */
+void
+nv46_pci_msi_rearm(struct nvkm_pci *pci)
+{
+ struct nvkm_device *device = pci->subdev.device;
+ struct pci_dev *pdev = device->func->pci(device)->pdev;
+ pci_write_config_byte(pdev, 0x68, 0xff);
+}
+
+static const struct nvkm_pci_func
+nv46_pci_func = {
+ .rd32 = nv40_pci_rd32,
+ .wr08 = nv40_pci_wr08,
+ .wr32 = nv40_pci_wr32,
+ .msi_rearm = nv46_pci_msi_rearm,
+};
+
+int
+nv46_pci_new(struct nvkm_device *device, int index, struct nvkm_pci **ppci)
+{
+ return nvkm_pci_new_(&nv46_pci_func, device, index, ppci);
+}
+++ /dev/null
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "priv.h"
-
-#include <core/pci.h>
-
-/* MSI re-arm through the PRI appears to be broken on the original G80,
- * so we access it via alternate PCI config space mechanisms.
- */
-static void
-nv50_pci_msi_rearm(struct nvkm_pci *pci)
-{
- struct nvkm_device *device = pci->subdev.device;
- struct pci_dev *pdev = device->func->pci(device)->pdev;
- pci_write_config_byte(pdev, 0x68, 0xff);
-}
-
-static const struct nvkm_pci_func
-nv50_pci_func = {
- .rd32 = nv40_pci_rd32,
- .wr08 = nv40_pci_wr08,
- .wr32 = nv40_pci_wr32,
- .msi_rearm = nv50_pci_msi_rearm,
-};
-
-int
-nv50_pci_new(struct nvkm_device *device, int index, struct nvkm_pci **ppci)
-{
- return nvkm_pci_new_(&nv50_pci_func, device, index, ppci);
-}
int index, struct nvkm_pci **);
struct nvkm_pci_func {
+ void (*init)(struct nvkm_pci *);
u32 (*rd32)(struct nvkm_pci *, u16 addr);
void (*wr08)(struct nvkm_pci *, u16 addr, u8 data);
void (*wr32)(struct nvkm_pci *, u16 addr, u32 data);
u32 nv40_pci_rd32(struct nvkm_pci *, u16);
void nv40_pci_wr08(struct nvkm_pci *, u16, u8);
void nv40_pci_wr32(struct nvkm_pci *, u16, u32);
+void nv40_pci_msi_rearm(struct nvkm_pci *);
+
+void nv46_pci_msi_rearm(struct nvkm_pci *);
+
+void g84_pci_init(struct nvkm_pci *pci);
#endif
void
nvkm_pmu_pgob(struct nvkm_pmu *pmu, bool enable)
{
- if (pmu->func->pgob)
+ if (pmu && pmu->func->pgob)
pmu->func->pgob(pmu, enable);
}
#include "fuc/gf119.fuc4.h"
#include <core/option.h>
+#include <subdev/fuse.h>
#include <subdev/timer.h>
static void
{
struct nvkm_device *device = pmu->subdev.device;
+ if (!(nvkm_fuse_read(device->fuse, 0x31c) & 0x00000001))
+ return;
+
nvkm_mask(device, 0x000200, 0x00001000, 0x00000000);
nvkm_rd32(device, 0x000200);
nvkm_mask(device, 0x000200, 0x08000000, 0x08000000);
nvkm-y += nvkm/subdev/volt/base.o
nvkm-y += nvkm/subdev/volt/gpio.o
nvkm-y += nvkm/subdev/volt/nv40.o
+nvkm-y += nvkm/subdev/volt/gk104.o
nvkm-y += nvkm/subdev/volt/gk20a.o
int
nvkm_volt_get(struct nvkm_volt *volt)
{
- int ret = volt->func->vid_get(volt), i;
+ int ret, i;
+
+ if (volt->func->volt_get)
+ return volt->func->volt_get(volt);
+
+ ret = volt->func->vid_get(volt);
if (ret >= 0) {
for (i = 0; i < volt->vid_nr; i++) {
if (volt->vid[i].vid == ret)
{
struct nvkm_subdev *subdev = &volt->subdev;
int i, ret = -EINVAL;
+
+ if (volt->func->volt_set)
+ return volt->func->volt_set(volt, uv);
+
for (i = 0; i < volt->vid_nr; i++) {
if (volt->vid[i].uv == uv) {
ret = volt->func->vid_set(volt, volt->vid[i].vid);
--- /dev/null
+/*
+ * Copyright 2015 Martin Peres
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+#include "priv.h"
+
+#include <subdev/volt.h>
+#include <subdev/gpio.h>
+#include <subdev/bios.h>
+#include <subdev/bios/volt.h>
+
+#define gk104_volt(p) container_of((p), struct gk104_volt, base)
+struct gk104_volt {
+ struct nvkm_volt base;
+ struct nvbios_volt bios;
+};
+
+int
+gk104_volt_get(struct nvkm_volt *base)
+{
+ struct nvbios_volt *bios = &gk104_volt(base)->bios;
+ struct nvkm_device *device = base->subdev.device;
+ u32 div, duty;
+
+ div = nvkm_rd32(device, 0x20340);
+ duty = nvkm_rd32(device, 0x20344);
+
+ return bios->base + bios->pwm_range * duty / div;
+}
+
+int
+gk104_volt_set(struct nvkm_volt *base, u32 uv)
+{
+ struct nvbios_volt *bios = &gk104_volt(base)->bios;
+ struct nvkm_device *device = base->subdev.device;
+ u32 div, duty;
+
+ /* the blob uses this crystal frequency, let's use it too. */
+ div = 27648000 / bios->pwm_freq;
+ duty = (uv - bios->base) * div / bios->pwm_range;
+
+ nvkm_wr32(device, 0x20340, div);
+ nvkm_wr32(device, 0x20344, 0x8000000 | duty);
+
+ return 0;
+}
+
+static const struct nvkm_volt_func
+gk104_volt_pwm = {
+ .volt_get = gk104_volt_get,
+ .volt_set = gk104_volt_set,
+}, gk104_volt_gpio = {
+ .vid_get = nvkm_voltgpio_get,
+ .vid_set = nvkm_voltgpio_set,
+};
+
+int
+gk104_volt_new(struct nvkm_device *device, int index, struct nvkm_volt **pvolt)
+{
+ const struct nvkm_volt_func *volt_func = &gk104_volt_gpio;
+ struct dcb_gpio_func gpio;
+ struct nvbios_volt bios;
+ struct gk104_volt *volt;
+ u8 ver, hdr, cnt, len;
+ const char *mode;
+
+ if (!nvbios_volt_parse(device->bios, &ver, &hdr, &cnt, &len, &bios))
+ return 0;
+
+ if (!nvkm_gpio_find(device->gpio, 0, DCB_GPIO_VID_PWM, 0xff, &gpio) &&
+ bios.type == NVBIOS_VOLT_PWM) {
+ volt_func = &gk104_volt_pwm;
+ }
+
+ if (!(volt = kzalloc(sizeof(*volt), GFP_KERNEL)))
+ return -ENOMEM;
+ nvkm_volt_ctor(volt_func, device, index, &volt->base);
+ *pvolt = &volt->base;
+ volt->bios = bios;
+
+ /* now that we have a subdev, we can show an error if we found through
+ * the voltage table that we were supposed to use the PWN mode but we
+ * did not find the right GPIO for it.
+ */
+ if (bios.type == NVBIOS_VOLT_PWM && volt_func != &gk104_volt_pwm) {
+ nvkm_error(&volt->base.subdev,
+ "Type mismatch between the voltage table type and "
+ "the GPIO table. Fallback to GPIO mode.\n");
+ }
+
+ if (volt_func == &gk104_volt_gpio) {
+ nvkm_voltgpio_init(&volt->base);
+ mode = "GPIO";
+ } else
+ mode = "PWM";
+
+ nvkm_debug(&volt->base.subdev, "Using %s mode\n", mode);
+
+ return 0;
+}
int index, struct nvkm_volt **);
struct nvkm_volt_func {
+ int (*volt_get)(struct nvkm_volt *);
+ int (*volt_set)(struct nvkm_volt *, u32 uv);
int (*vid_get)(struct nvkm_volt *);
int (*vid_set)(struct nvkm_volt *, u8 vid);
int (*set_id)(struct nvkm_volt *, u8 id, int condition);
int nvkm_voltgpio_init(struct nvkm_volt *);
int nvkm_voltgpio_get(struct nvkm_volt *);
int nvkm_voltgpio_set(struct nvkm_volt *, u8);
+
+int nvkm_voltpwm_init(struct nvkm_volt *volt);
+int nvkm_voltpwm_get(struct nvkm_volt *volt);
+int nvkm_voltpwm_set(struct nvkm_volt *volt, u32 uv);
#endif
dispc_mgr_go(omap_crtc->channel);
omap_irq_register(crtc->dev, &omap_crtc->vblank_irq);
}
-
- crtc->invert_dimensions = !!(crtc->primary->state->rotation &
- (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270)));
}
static int omap_crtc_atomic_set_property(struct drm_crtc *crtc,
dispc_runtime_get();
drm_atomic_helper_commit_modeset_disables(dev, old_state);
- drm_atomic_helper_commit_planes(dev, old_state);
+ drm_atomic_helper_commit_planes(dev, old_state, false);
drm_atomic_helper_commit_modeset_enables(dev, old_state);
omap_atomic_wait_for_completion(dev, old_state);
}
static const struct drm_ioctl_desc ioctls[DRM_COMMAND_END - DRM_COMMAND_BASE] = {
- DRM_IOCTL_DEF_DRV(OMAP_GET_PARAM, ioctl_get_param, DRM_UNLOCKED|DRM_AUTH),
- DRM_IOCTL_DEF_DRV(OMAP_SET_PARAM, ioctl_set_param, DRM_UNLOCKED|DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(OMAP_GEM_NEW, ioctl_gem_new, DRM_UNLOCKED|DRM_AUTH),
- DRM_IOCTL_DEF_DRV(OMAP_GEM_CPU_PREP, ioctl_gem_cpu_prep, DRM_UNLOCKED|DRM_AUTH),
- DRM_IOCTL_DEF_DRV(OMAP_GEM_CPU_FINI, ioctl_gem_cpu_fini, DRM_UNLOCKED|DRM_AUTH),
- DRM_IOCTL_DEF_DRV(OMAP_GEM_INFO, ioctl_gem_info, DRM_UNLOCKED|DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(OMAP_GET_PARAM, ioctl_get_param, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(OMAP_SET_PARAM, ioctl_set_param, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(OMAP_GEM_NEW, ioctl_gem_new, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(OMAP_GEM_CPU_PREP, ioctl_gem_cpu_prep, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(OMAP_GEM_CPU_FINI, ioctl_gem_cpu_fini, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(OMAP_GEM_INFO, ioctl_gem_info, DRM_AUTH),
};
/*
{
int i;
- /* we don't support vga-switcheroo.. so just make sure the fbdev
+ /* we don't support vga_switcheroo.. so just make sure the fbdev
* mode is active
*/
struct omap_drm_private *priv = dev->dev_private;
.preclose = dev_preclose,
.postclose = dev_postclose,
.set_busid = drm_platform_set_busid,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = omap_irq_enable_vblank,
.disable_vblank = omap_irq_disable_vblank,
#ifdef CONFIG_DEBUG_FS
int omap_gem_resume(struct device *dev);
#endif
-int omap_irq_enable_vblank(struct drm_device *dev, int crtc_id);
-void omap_irq_disable_vblank(struct drm_device *dev, int crtc_id);
+int omap_irq_enable_vblank(struct drm_device *dev, unsigned int pipe);
+void omap_irq_disable_vblank(struct drm_device *dev, unsigned int pipe);
void __omap_irq_register(struct drm_device *dev, struct omap_drm_irq *irq);
void __omap_irq_unregister(struct drm_device *dev, struct omap_drm_irq *irq);
void omap_irq_register(struct drm_device *dev, struct omap_drm_irq *irq);
uint32_t w = win->src_w;
uint32_t h = win->src_h;
- switch (win->rotation & 0xf) {
+ switch (win->rotation & DRM_ROTATE_MASK) {
default:
dev_err(fb->dev->dev, "invalid rotation: %02x",
(uint32_t)win->rotation);
info->rotation_type = OMAP_DSS_ROT_TILER;
info->screen_width = omap_gem_tiled_stride(plane->bo, orient);
} else {
- switch (win->rotation & 0xf) {
+ switch (win->rotation & DRM_ROTATE_MASK) {
case 0:
case BIT(DRM_ROTATE_0):
/* OK */
struct vm_area_struct *vma)
{
struct drm_gem_object *obj = buffer->priv;
- struct drm_device *dev = obj->dev;
int ret = 0;
if (WARN_ON(!obj->filp))
return -EINVAL;
- mutex_lock(&dev->struct_mutex);
ret = drm_gem_mmap_obj(obj, omap_gem_mmap_size(obj), vma);
- mutex_unlock(&dev->struct_mutex);
if (ret < 0)
return ret;
/**
* enable_vblank - enable vblank interrupt events
* @dev: DRM device
- * @crtc: which irq to enable
+ * @pipe: which irq to enable
*
* Enable vblank interrupts for @crtc. If the device doesn't have
* a hardware vblank counter, this routine should be a no-op, since
* Zero on success, appropriate errno if the given @crtc's vblank
* interrupt cannot be enabled.
*/
-int omap_irq_enable_vblank(struct drm_device *dev, int crtc_id)
+int omap_irq_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct omap_drm_private *priv = dev->dev_private;
- struct drm_crtc *crtc = priv->crtcs[crtc_id];
+ struct drm_crtc *crtc = priv->crtcs[pipe];
unsigned long flags;
- DBG("dev=%p, crtc=%d", dev, crtc_id);
+ DBG("dev=%p, crtc=%u", dev, pipe);
spin_lock_irqsave(&list_lock, flags);
priv->vblank_mask |= pipe2vbl(crtc);
/**
* disable_vblank - disable vblank interrupt events
* @dev: DRM device
- * @crtc: which irq to enable
+ * @pipe: which irq to enable
*
* Disable vblank interrupts for @crtc. If the device doesn't have
* a hardware vblank counter, this routine should be a no-op, since
* interrupts will have to stay on to keep the count accurate.
*/
-void omap_irq_disable_vblank(struct drm_device *dev, int crtc_id)
+void omap_irq_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct omap_drm_private *priv = dev->dev_private;
- struct drm_crtc *crtc = priv->crtcs[crtc_id];
+ struct drm_crtc *crtc = priv->crtcs[pipe];
unsigned long flags;
- DBG("dev=%p, crtc=%d", dev, crtc_id);
+ DBG("dev=%p, crtc=%u", dev, pipe);
spin_lock_irqsave(&list_lock, flags);
priv->vblank_mask &= ~pipe2vbl(crtc);
}
static int omap_plane_prepare_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
- return omap_framebuffer_pin(fb);
+ if (!new_state->fb)
+ return 0;
+
+ return omap_framebuffer_pin(new_state->fb);
}
static void omap_plane_cleanup_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *old_state)
{
- omap_framebuffer_unpin(fb);
+ if (old_state->fb)
+ omap_framebuffer_unpin(old_state->fb);
}
static void omap_plane_atomic_update(struct drm_plane *plane,
win.src_x = state->src_x >> 16;
win.src_y = state->src_y >> 16;
- switch (state->rotation & 0xf) {
+ switch (state->rotation & DRM_ROTATE_MASK) {
case BIT(DRM_ROTATE_90):
case BIT(DRM_ROTATE_270):
win.src_w = state->src_h >> 16;
bo->is_primary = true;
ret = qxl_bo_reserve(bo, false);
+ if (ret)
+ return ret;
+ ret = qxl_bo_pin(bo, bo->type, NULL);
+ qxl_bo_unreserve(bo);
if (ret)
return ret;
}
drm_vblank_put(dev, qcrtc->index);
- qxl_bo_unreserve(bo);
+ ret = qxl_bo_reserve(bo, false);
+ if (!ret) {
+ qxl_bo_unpin(bo);
+ qxl_bo_unreserve(bo);
+ }
return 0;
}
adjusted_mode->hdisplay,
adjusted_mode->vdisplay);
- if (qcrtc->index == 0)
+ if (bo->is_primary == false)
recreate_primary = true;
if (bo->surf.stride * bo->surf.height > qdev->vram_size) {
drm_connector_to_qxl_output(connector);
struct drm_device *ddev = connector->dev;
struct qxl_device *qdev = ddev->dev_private;
- int connected;
+ bool connected = false;
/* The first monitor is always connected */
- connected = (output->index == 0) ||
- (qdev->client_monitors_config &&
- qdev->client_monitors_config->count > output->index &&
- qxl_head_enabled(&qdev->client_monitors_config->heads[output->index]));
+ if (!qdev->client_monitors_config) {
+ if (output->index == 0)
+ connected = true;
+ } else
+ connected = qdev->client_monitors_config->count > output->index &&
+ qxl_head_enabled(&qdev->client_monitors_config->heads[output->index]);
DRM_DEBUG("#%d connected: %d\n", output->index, connected);
if (!connected)
return qxl_drm_resume(drm_dev, false);
}
-static u32 qxl_noop_get_vblank_counter(struct drm_device *dev, int crtc)
+static u32 qxl_noop_get_vblank_counter(struct drm_device *dev,
+ unsigned int pipe)
{
return 0;
}
-static int qxl_noop_enable_vblank(struct drm_device *dev, int crtc)
+static int qxl_noop_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
return 0;
}
-static void qxl_noop_disable_vblank(struct drm_device *dev, int crtc)
+static void qxl_noop_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
}
spin_lock_irqsave(&qfbdev->dirty.lock, flags);
- if (qfbdev->dirty.y1 < y)
- y = qfbdev->dirty.y1;
- if (qfbdev->dirty.y2 > y2)
- y2 = qfbdev->dirty.y2;
- if (qfbdev->dirty.x1 < x)
- x = qfbdev->dirty.x1;
- if (qfbdev->dirty.x2 > x2)
- x2 = qfbdev->dirty.x2;
+ if ((qfbdev->dirty.y2 - qfbdev->dirty.y1) &&
+ (qfbdev->dirty.x2 - qfbdev->dirty.x1)) {
+ if (qfbdev->dirty.y1 < y)
+ y = qfbdev->dirty.y1;
+ if (qfbdev->dirty.y2 > y2)
+ y2 = qfbdev->dirty.y2;
+ if (qfbdev->dirty.x1 < x)
+ x = qfbdev->dirty.x1;
+ if (qfbdev->dirty.x2 > x2)
+ x2 = qfbdev->dirty.x2;
+ }
qfbdev->dirty.x1 = x;
qfbdev->dirty.x2 = x2;
}
const struct drm_ioctl_desc qxl_ioctls[] = {
- DRM_IOCTL_DEF_DRV(QXL_ALLOC, qxl_alloc_ioctl, DRM_AUTH|DRM_UNLOCKED),
+ DRM_IOCTL_DEF_DRV(QXL_ALLOC, qxl_alloc_ioctl, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(QXL_MAP, qxl_map_ioctl, DRM_AUTH|DRM_UNLOCKED),
+ DRM_IOCTL_DEF_DRV(QXL_MAP, qxl_map_ioctl, DRM_AUTH),
DRM_IOCTL_DEF_DRV(QXL_EXECBUFFER, qxl_execbuffer_ioctl,
- DRM_AUTH|DRM_UNLOCKED),
+ DRM_AUTH),
DRM_IOCTL_DEF_DRV(QXL_UPDATE_AREA, qxl_update_area_ioctl,
- DRM_AUTH|DRM_UNLOCKED),
+ DRM_AUTH),
DRM_IOCTL_DEF_DRV(QXL_GETPARAM, qxl_getparam_ioctl,
- DRM_AUTH|DRM_UNLOCKED),
+ DRM_AUTH),
DRM_IOCTL_DEF_DRV(QXL_CLIENTCAP, qxl_clientcap_ioctl,
- DRM_AUTH|DRM_UNLOCKED),
+ DRM_AUTH),
DRM_IOCTL_DEF_DRV(QXL_ALLOC_SURF, qxl_alloc_surf_ioctl,
- DRM_AUTH|DRM_UNLOCKED),
+ DRM_AUTH),
};
int qxl_max_ioctls = ARRAY_SIZE(qxl_ioctls);
idr_ret = qxl_release_alloc(qdev, QXL_RELEASE_SURFACE_CMD, release);
if (idr_ret < 0)
return idr_ret;
- bo = qxl_bo_ref(to_qxl_bo(entry->tv.bo));
+ bo = to_qxl_bo(entry->tv.bo);
(*release)->release_offset = create_rel->release_offset + 64;
info = qxl_release_map(qdev, *release);
info->id = idr_ret;
qxl_release_unmap(qdev, *release, info);
-
- qxl_bo_unref(&bo);
return 0;
}
/* The manual (p. 2) says this address is in "VM space". This
* means it's an offset from the start of AGP space.
*/
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (!dev_priv->is_pci)
ring_start = dev_priv->cce_ring->offset - dev->agp->base;
else
(drm_r128_sarea_t *) ((u8 *) dev_priv->sarea->handle +
init->sarea_priv_offset);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (!dev_priv->is_pci) {
drm_legacy_ioremap_wc(dev_priv->cce_ring, dev);
drm_legacy_ioremap_wc(dev_priv->ring_rptr, dev);
(void *)(unsigned long)dev->agp_buffer_map->offset;
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (!dev_priv->is_pci)
dev_priv->cce_buffers_offset = dev->agp->base;
else
dev_priv->sarea_priv->last_dispatch = 0;
R128_WRITE(R128_LAST_DISPATCH_REG, dev_priv->sarea_priv->last_dispatch);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->is_pci) {
#endif
dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
return -ENOMEM;
}
R128_WRITE(R128_PCI_GART_PAGE, dev_priv->gart_info.bus_addr);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
}
#endif
if (dev->dev_private) {
drm_r128_private_t *dev_priv = dev->dev_private;
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (!dev_priv->is_pci) {
if (dev_priv->cce_ring != NULL)
drm_legacy_ioremapfree(dev_priv->cce_ring, dev);
extern int r128_do_cce_idle(drm_r128_private_t *dev_priv);
extern int r128_do_cleanup_cce(struct drm_device *dev);
-extern int r128_enable_vblank(struct drm_device *dev, int crtc);
-extern void r128_disable_vblank(struct drm_device *dev, int crtc);
-extern u32 r128_get_vblank_counter(struct drm_device *dev, int crtc);
+extern int r128_enable_vblank(struct drm_device *dev, unsigned int pipe);
+extern void r128_disable_vblank(struct drm_device *dev, unsigned int pipe);
+extern u32 r128_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
extern irqreturn_t r128_driver_irq_handler(int irq, void *arg);
extern void r128_driver_irq_preinstall(struct drm_device *dev);
extern int r128_driver_irq_postinstall(struct drm_device *dev);
#include <drm/r128_drm.h>
#include "r128_drv.h"
-u32 r128_get_vblank_counter(struct drm_device *dev, int crtc)
+u32 r128_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
const drm_r128_private_t *dev_priv = dev->dev_private;
- if (crtc != 0)
+ if (pipe != 0)
return 0;
return atomic_read(&dev_priv->vbl_received);
return IRQ_NONE;
}
-int r128_enable_vblank(struct drm_device *dev, int crtc)
+int r128_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
drm_r128_private_t *dev_priv = dev->dev_private;
- if (crtc != 0) {
- DRM_ERROR("%s: bad crtc %d\n", __func__, crtc);
+ if (pipe != 0) {
+ DRM_ERROR("%s: bad crtc %u\n", __func__, pipe);
return -EINVAL;
}
return 0;
}
-void r128_disable_vblank(struct drm_device *dev, int crtc)
+void r128_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
- if (crtc != 0)
- DRM_ERROR("%s: bad crtc %d\n", __func__, crtc);
+ if (pipe != 0)
+ DRM_ERROR("%s: bad crtc %u\n", __func__, pipe);
/*
* FIXME: implement proper interrupt disable by using the vblank
switch (msg->request & ~DP_AUX_I2C_MOT) {
case DP_AUX_NATIVE_WRITE:
case DP_AUX_I2C_WRITE:
+ case DP_AUX_I2C_WRITE_STATUS_UPDATE:
/* The atom implementation only supports writes with a max payload of
* 12 bytes since it uses 4 bits for the total count (header + payload)
* in the parameter space. The atom interface supports 16 byte
} else
atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
- args.ucAction = ATOM_LCD_BLON;
- atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+ struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
+
+ atombios_set_backlight_level(radeon_encoder, dig->backlight_level);
}
break;
case DRM_MODE_DPMS_STANDBY:
atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0);
}
if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
- atombios_dig_transmitter_setup(encoder,
- ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0);
+ atombios_set_backlight_level(radeon_encoder, dig->backlight_level);
if (ext_encoder)
atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE);
break;
* evergreen cards need to use the 3D engine to blit data which requires
* quite a bit of hw state setup. Rather than pull the whole 3D driver
* (which normally generates the 3D state) into the DRM, we opt to use
- * statically generated state tables. The regsiter state and shaders
+ * statically generated state tables. The register state and shaders
* were hand generated to support blitting functionality. See the 3D
* driver or documentation for descriptions of the registers and
* shader instructions.
* @crtc_id: crtc to cleanup pageflip on
* @crtc_base: new address of the crtc (GPU MC address)
*
- * Does the actual pageflip (evergreen+).
- * During vblank we take the crtc lock and wait for the update_pending
- * bit to go high, when it does, we release the lock, and allow the
- * double buffered update to take place.
- * Returns the current update pending status.
+ * Triggers the actual pageflip by updating the primary
+ * surface base address (evergreen+).
*/
void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
{
struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
- u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
- int i;
-
- /* Lock the graphics update lock */
- tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
- WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
/* update the scanout addresses */
- WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
- upper_32_bits(crtc_base));
- WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
- (u32)crtc_base);
-
WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
upper_32_bits(crtc_base));
WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
(u32)crtc_base);
-
- /* Wait for update_pending to go high. */
- for (i = 0; i < rdev->usec_timeout; i++) {
- if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
- break;
- udelay(1);
- }
- DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
-
- /* Unlock the lock, so double-buffering can take place inside vblank */
- tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
- WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
+ /* post the write */
+ RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
}
/**
* evergreen cards need to use the 3D engine to blit data which requires
* quite a bit of hw state setup. Rather than pull the whole 3D driver
* (which normally generates the 3D state) into the DRM, we opt to use
- * statically generated state tables. The regsiter state and shaders
+ * statically generated state tables. The register state and shaders
* were hand generated to support blitting functionality. See the 3D
* driver or documentation for descriptions of the registers and
* shader instructions.
#define MAX(a,b) (((a)>(b))?(a):(b))
#define MIN(a,b) (((a)<(b))?(a):(b))
+#define REG_SAFE_BM_SIZE ARRAY_SIZE(evergreen_reg_safe_bm)
+
int r600_dma_cs_next_reloc(struct radeon_cs_parser *p,
struct radeon_bo_list **cs_reloc);
struct evergreen_cs_track {
u32 htile_surface;
struct radeon_bo *htile_bo;
unsigned long indirect_draw_buffer_size;
+ const unsigned *reg_safe_bm;
};
static u32 evergreen_cs_get_aray_mode(u32 tiling_flags)
* command stream.
*/
if (!surf.mode) {
- volatile u32 *ib = p->ib.ptr;
+ uint32_t *ib = p->ib.ptr;
unsigned long tmp, nby, bsize, size, min = 0;
/* find the height the ddx wants */
}
/**
- * evergreen_cs_check_reg() - check if register is authorized or not
+ * evergreen_cs_handle_reg() - process registers that need special handling.
* @parser: parser structure holding parsing context
* @reg: register we are testing
* @idx: index into the cs buffer
- *
- * This function will test against evergreen_reg_safe_bm and return 0
- * if register is safe. If register is not flag as safe this function
- * will test it against a list of register needind special handling.
*/
-static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
+static int evergreen_cs_handle_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
{
struct evergreen_cs_track *track = (struct evergreen_cs_track *)p->track;
struct radeon_bo_list *reloc;
- u32 last_reg;
- u32 m, i, tmp, *ib;
+ u32 tmp, *ib;
int r;
- if (p->rdev->family >= CHIP_CAYMAN)
- last_reg = ARRAY_SIZE(cayman_reg_safe_bm);
- else
- last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
-
- i = (reg >> 7);
- if (i >= last_reg) {
- dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
- return -EINVAL;
- }
- m = 1 << ((reg >> 2) & 31);
- if (p->rdev->family >= CHIP_CAYMAN) {
- if (!(cayman_reg_safe_bm[i] & m))
- return 0;
- } else {
- if (!(evergreen_reg_safe_bm[i] & m))
- return 0;
- }
ib = p->ib.ptr;
switch (reg) {
/* force following reg to 0 in an attempt to disable out buffer
return 0;
}
-static bool evergreen_is_safe_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
+/**
+ * evergreen_is_safe_reg() - check if register is authorized or not
+ * @parser: parser structure holding parsing context
+ * @reg: register we are testing
+ *
+ * This function will test against reg_safe_bm and return true
+ * if register is safe or false otherwise.
+ */
+static inline bool evergreen_is_safe_reg(struct radeon_cs_parser *p, u32 reg)
{
- u32 last_reg, m, i;
-
- if (p->rdev->family >= CHIP_CAYMAN)
- last_reg = ARRAY_SIZE(cayman_reg_safe_bm);
- else
- last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
+ struct evergreen_cs_track *track = p->track;
+ u32 m, i;
i = (reg >> 7);
- if (i >= last_reg) {
- dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
+ if (unlikely(i >= REG_SAFE_BM_SIZE)) {
return false;
}
m = 1 << ((reg >> 2) & 31);
- if (p->rdev->family >= CHIP_CAYMAN) {
- if (!(cayman_reg_safe_bm[i] & m))
- return true;
- } else {
- if (!(evergreen_reg_safe_bm[i] & m))
- return true;
- }
- dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
+ if (!(track->reg_safe_bm[i] & m))
+ return true;
+
return false;
}
{
struct radeon_bo_list *reloc;
struct evergreen_cs_track *track;
- volatile u32 *ib;
+ uint32_t *ib;
unsigned idx;
unsigned i;
unsigned start_reg, end_reg, reg;
DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
return -EINVAL;
}
- for (i = 0; i < pkt->count; i++) {
- reg = start_reg + (4 * i);
- r = evergreen_cs_check_reg(p, reg, idx+1+i);
+ for (reg = start_reg, idx++; reg <= end_reg; reg += 4, idx++) {
+ if (evergreen_is_safe_reg(p, reg))
+ continue;
+ r = evergreen_cs_handle_reg(p, reg, idx);
if (r)
return r;
}
DRM_ERROR("bad PACKET3_SET_CONTEXT_REG\n");
return -EINVAL;
}
- for (i = 0; i < pkt->count; i++) {
- reg = start_reg + (4 * i);
- r = evergreen_cs_check_reg(p, reg, idx+1+i);
+ for (reg = start_reg, idx++; reg <= end_reg; reg += 4, idx++) {
+ if (evergreen_is_safe_reg(p, reg))
+ continue;
+ r = evergreen_cs_handle_reg(p, reg, idx);
if (r)
return r;
}
} else {
/* SRC is a reg. */
reg = radeon_get_ib_value(p, idx+1) << 2;
- if (!evergreen_is_safe_reg(p, reg, idx+1))
+ if (!evergreen_is_safe_reg(p, reg)) {
+ dev_warn(p->dev, "forbidden register 0x%08x at %d\n",
+ reg, idx + 1);
return -EINVAL;
+ }
}
if (idx_value & 0x2) {
u64 offset;
} else {
/* DST is a reg. */
reg = radeon_get_ib_value(p, idx+3) << 2;
- if (!evergreen_is_safe_reg(p, reg, idx+3))
+ if (!evergreen_is_safe_reg(p, reg)) {
+ dev_warn(p->dev, "forbidden register 0x%08x at %d\n",
+ reg, idx + 3);
return -EINVAL;
+ }
}
break;
case PACKET3_NOP:
if (track == NULL)
return -ENOMEM;
evergreen_cs_track_init(track);
- if (p->rdev->family >= CHIP_CAYMAN)
+ if (p->rdev->family >= CHIP_CAYMAN) {
tmp = p->rdev->config.cayman.tile_config;
- else
+ track->reg_safe_bm = cayman_reg_safe_bm;
+ } else {
tmp = p->rdev->config.evergreen.tile_config;
-
+ track->reg_safe_bm = evergreen_reg_safe_bm;
+ }
+ BUILD_BUG_ON(ARRAY_SIZE(cayman_reg_safe_bm) != REG_SAFE_BM_SIZE);
+ BUILD_BUG_ON(ARRAY_SIZE(evergreen_reg_safe_bm) != REG_SAFE_BM_SIZE);
switch (tmp & 0xf) {
case 0:
track->npipes = 1;
struct radeon_cs_chunk *ib_chunk = p->chunk_ib;
struct radeon_bo_list *src_reloc, *dst_reloc, *dst2_reloc;
u32 header, cmd, count, sub_cmd;
- volatile u32 *ib = p->ib.ptr;
+ uint32_t *ib = p->ib.ptr;
u32 idx;
u64 src_offset, dst_offset, dst2_offset;
int r;
* R6xx+ cards need to use the 3D engine to blit data which requires
* quite a bit of hw state setup. Rather than pull the whole 3D driver
* (which normally generates the 3D state) into the DRM, we opt to use
- * statically generated state tables. The regsiter state and shaders
+ * statically generated state tables. The register state and shaders
* were hand generated to support blitting functionality. See the 3D
* driver or documentation for descriptions of the registers and
* shader instructions.
SET_RING_HEAD(dev_priv, 0);
dev_priv->ring.tail = 0;
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
rptr_addr = dev_priv->ring_rptr->offset
- dev->agp->base +
dev_priv->ring.size_l2qw);
#endif
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
/* XXX */
radeon_write_agp_base(dev_priv, dev->agp->base);
if (dev->irq_enabled)
drm_irq_uninstall(dev);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
if (dev_priv->cp_ring != NULL) {
drm_legacy_ioremapfree(dev_priv->cp_ring, dev);
}
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
/* XXX */
if (dev_priv->flags & RADEON_IS_AGP) {
drm_legacy_ioremap_wc(dev_priv->cp_ring, dev);
* location in the card and on the bus, though we have to
* align it down.
*/
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
/* XXX */
if (dev_priv->flags & RADEON_IS_AGP) {
base = dev->agp->base;
base, dev_priv->gart_vm_start);
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
/* XXX */
if (dev_priv->flags & RADEON_IS_AGP)
dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
/* XXX turn off pcie gart */
} else
#include <linux/acpi.h>
#include <linux/slab.h>
#include <linux/power_supply.h>
-#include <linux/vga_switcheroo.h>
#include <acpi/video.h>
#include <drm/drmP.h>
#include <drm/drm_crtc_helper.h>
#include "radeon.h"
#include <drm/radeon_drm.h>
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
struct radeon_agpmode_quirk {
u32 hostbridge_vendor;
int radeon_agp_init(struct radeon_device *rdev)
{
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
struct radeon_agpmode_quirk *p = radeon_agpmode_quirk_list;
struct drm_agp_mode mode;
struct drm_agp_info info;
void radeon_agp_resume(struct radeon_device *rdev)
{
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
int r;
if (rdev->flags & RADEON_IS_AGP) {
r = radeon_agp_init(rdev);
void radeon_agp_fini(struct radeon_device *rdev)
{
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (rdev->ddev->agp && rdev->ddev->agp->acquired) {
drm_agp_release(rdev->ddev);
}
#include <drm/drm_crtc_helper.h>
#include <drm/radeon_drm.h>
#include <linux/vgaarb.h>
-#include <linux/vga_switcheroo.h>
#include "radeon_reg.h"
#include "radeon.h"
#include "radeon_asic.h"
return VGA_SWITCHEROO_DIS;
}
-static struct vga_switcheroo_handler radeon_atpx_handler = {
+static const struct vga_switcheroo_handler radeon_atpx_handler = {
.switchto = radeon_atpx_switchto,
.power_state = radeon_atpx_power_state,
.init = radeon_atpx_init,
if (has_atpx && vga_count == 2) {
acpi_get_name(radeon_atpx_priv.atpx.handle, ACPI_FULL_PATHNAME, &buffer);
- printk(KERN_INFO "VGA switcheroo: detected switching method %s handle\n",
+ printk(KERN_INFO "vga_switcheroo: detected switching method %s handle\n",
acpi_method_name);
radeon_atpx_priv.atpx_detected = true;
return true;
#include "radeon.h"
#include "atom.h"
-#include <linux/vga_switcheroo.h>
#include <linux/slab.h>
#include <linux/acpi.h>
/*
((dev_priv->gart_vm_start - 1) & 0xffff0000)
| (dev_priv->fb_location >> 16));
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
radeon_write_agp_base(dev_priv, dev->agp->base);
SET_RING_HEAD(dev_priv, cur_read_ptr);
dev_priv->ring.tail = cur_read_ptr;
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
dev_priv->ring_rptr->offset
}
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
drm_legacy_ioremap_wc(dev_priv->cp_ring, dev);
drm_legacy_ioremap_wc(dev_priv->ring_rptr, dev);
* location in the card and on the bus, though we have to
* align it down.
*/
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
base = dev->agp->base;
/* Check if valid */
RADEON_READ(RADEON_CONFIG_APER_SIZE);
}
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP)
dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
- dev->agp->base
dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
/* Turn off PCI GART */
radeon_set_pcigart(dev_priv, 0);
if (dev->irq_enabled)
drm_irq_uninstall(dev);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
if (dev_priv->cp_ring != NULL) {
drm_legacy_ioremapfree(dev_priv->cp_ring, dev);
DRM_DEBUG("Starting radeon_do_resume_cp()\n");
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (dev_priv->flags & RADEON_IS_AGP) {
/* Turn off PCI GART */
radeon_set_pcigart(dev_priv, 0);
* radeon_switcheroo_set_state - set switcheroo state
*
* @pdev: pci dev pointer
- * @state: vga switcheroo state
+ * @state: vga_switcheroo state
*
* Callback for the switcheroo driver. Suspends or resumes the
* the asics before or after it is powered up using ACPI methods.
drm_kms_helper_poll_disable(dev);
+ drm_modeset_lock_all(dev);
/* turn off display hw */
list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
}
+ drm_modeset_unlock_all(dev);
/* unpin the front buffers and cursors */
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
if (fbcon) {
drm_helper_resume_force_mode(dev);
/* turn on display hw */
+ drm_modeset_lock_all(dev);
list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
}
+ drm_modeset_unlock_all(dev);
}
drm_kms_helper_poll_enable(dev);
*/
if (update_pending &&
(DRM_SCANOUTPOS_VALID & radeon_get_crtc_scanoutpos(rdev->ddev, crtc_id, 0,
- &vpos, &hpos, NULL, NULL)) &&
+ &vpos, &hpos, NULL, NULL,
+ &rdev->mode_info.crtcs[crtc_id]->base.hwmode)) &&
((vpos >= (99 * rdev->mode_info.crtcs[crtc_id]->base.hwmode.crtc_vdisplay)/100) ||
(vpos < 0 && !ASIC_IS_AVIVO(rdev)))) {
/* crtc didn't flip in this target vblank interval,
radeon_fbdev_init(rdev);
drm_kms_helper_poll_init(rdev->ddev);
- if (rdev->pm.dpm_enabled) {
- /* do dpm late init */
- ret = radeon_pm_late_init(rdev);
- if (ret) {
- rdev->pm.dpm_enabled = false;
- DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n");
- }
- /* set the dpm state for PX since there won't be
- * a modeset to call this.
- */
- radeon_pm_compute_clocks(rdev);
- }
+ /* do pm late init */
+ ret = radeon_pm_late_init(rdev);
return 0;
}
* unknown small number of scanlines wrt. real scanout position.
*
*/
-int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc, unsigned int flags,
- int *vpos, int *hpos, ktime_t *stime, ktime_t *etime)
+int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
+ unsigned int flags, int *vpos, int *hpos,
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode)
{
u32 stat_crtc = 0, vbl = 0, position = 0;
int vbl_start, vbl_end, vtotal, ret = 0;
*stime = ktime_get();
if (ASIC_IS_DCE4(rdev)) {
- if (crtc == 0) {
+ if (pipe == 0) {
vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
EVERGREEN_CRTC0_REGISTER_OFFSET);
position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
EVERGREEN_CRTC0_REGISTER_OFFSET);
ret |= DRM_SCANOUTPOS_VALID;
}
- if (crtc == 1) {
+ if (pipe == 1) {
vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
EVERGREEN_CRTC1_REGISTER_OFFSET);
position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
EVERGREEN_CRTC1_REGISTER_OFFSET);
ret |= DRM_SCANOUTPOS_VALID;
}
- if (crtc == 2) {
+ if (pipe == 2) {
vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
EVERGREEN_CRTC2_REGISTER_OFFSET);
position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
EVERGREEN_CRTC2_REGISTER_OFFSET);
ret |= DRM_SCANOUTPOS_VALID;
}
- if (crtc == 3) {
+ if (pipe == 3) {
vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
EVERGREEN_CRTC3_REGISTER_OFFSET);
position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
EVERGREEN_CRTC3_REGISTER_OFFSET);
ret |= DRM_SCANOUTPOS_VALID;
}
- if (crtc == 4) {
+ if (pipe == 4) {
vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
EVERGREEN_CRTC4_REGISTER_OFFSET);
position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
EVERGREEN_CRTC4_REGISTER_OFFSET);
ret |= DRM_SCANOUTPOS_VALID;
}
- if (crtc == 5) {
+ if (pipe == 5) {
vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
EVERGREEN_CRTC5_REGISTER_OFFSET);
position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
ret |= DRM_SCANOUTPOS_VALID;
}
} else if (ASIC_IS_AVIVO(rdev)) {
- if (crtc == 0) {
+ if (pipe == 0) {
vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END);
position = RREG32(AVIVO_D1CRTC_STATUS_POSITION);
ret |= DRM_SCANOUTPOS_VALID;
}
- if (crtc == 1) {
+ if (pipe == 1) {
vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END);
position = RREG32(AVIVO_D2CRTC_STATUS_POSITION);
ret |= DRM_SCANOUTPOS_VALID;
}
} else {
/* Pre-AVIVO: Different encoding of scanout pos and vblank interval. */
- if (crtc == 0) {
+ if (pipe == 0) {
/* Assume vbl_end == 0, get vbl_start from
* upper 16 bits.
*/
ret |= DRM_SCANOUTPOS_VALID;
}
- if (crtc == 1) {
+ if (pipe == 1) {
vbl = (RREG32(RADEON_CRTC2_V_TOTAL_DISP) &
RADEON_CRTC_V_DISP) >> RADEON_CRTC_V_DISP_SHIFT;
position = (RREG32(RADEON_CRTC2_VLINE_CRNT_VLINE) >> 16) & RADEON_CRTC_V_TOTAL;
}
else {
/* No: Fake something reasonable which gives at least ok results. */
- vbl_start = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vdisplay;
+ vbl_start = mode->crtc_vdisplay;
vbl_end = 0;
}
/* Inside "upper part" of vblank area? Apply corrective offset if so: */
if (in_vbl && (*vpos >= vbl_start)) {
- vtotal = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vtotal;
+ vtotal = mode->crtc_vtotal;
*vpos = *vpos - vtotal;
}
* We only do this if DRM_CALLED_FROM_VBLIRQ.
*/
if ((flags & DRM_CALLED_FROM_VBLIRQ) && !in_vbl) {
- vbl_start = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vdisplay;
- vtotal = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vtotal;
+ vbl_start = mode->crtc_vdisplay;
+ vtotal = mode->crtc_vtotal;
if (vbl_start - *vpos < vtotal / 100) {
*vpos -= vtotal;
{
struct radeon_connector *master = container_of(mgr, struct radeon_connector, mst_mgr);
struct drm_device *dev = master->base.dev;
- struct radeon_device *rdev = dev->dev_private;
struct radeon_connector *radeon_connector;
struct drm_connector *connector;
drm_object_attach_property(&connector->base, dev->mode_config.path_property, 0);
drm_mode_connector_set_path_property(connector, pathprop);
+ return connector;
+}
+
+static void radeon_dp_register_mst_connector(struct drm_connector *connector)
+{
+ struct drm_device *dev = connector->dev;
+ struct radeon_device *rdev = dev->dev_private;
+
drm_modeset_lock_all(dev);
radeon_fb_add_connector(rdev, connector);
drm_modeset_unlock_all(dev);
drm_connector_register(connector);
- return connector;
}
static void radeon_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
struct drm_dp_mst_topology_cbs mst_cbs = {
.add_connector = radeon_dp_add_mst_connector,
+ .register_connector = radeon_dp_register_mst_connector,
.destroy_connector = radeon_dp_destroy_mst_connector,
.hotplug = radeon_dp_mst_hotplug,
};
struct drm_file *file_priv);
int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon);
int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon);
-u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc);
-int radeon_enable_vblank_kms(struct drm_device *dev, int crtc);
-void radeon_disable_vblank_kms(struct drm_device *dev, int crtc);
-int radeon_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
+u32 radeon_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe);
+int radeon_enable_vblank_kms(struct drm_device *dev, unsigned int pipe);
+void radeon_disable_vblank_kms(struct drm_device *dev, unsigned int pipe);
+int radeon_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int pipe,
int *max_error,
struct timeval *vblank_time,
unsigned flags);
struct dma_buf *radeon_gem_prime_export(struct drm_device *dev,
struct drm_gem_object *gobj,
int flags);
-extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
- unsigned int flags,
- int *vpos, int *hpos, ktime_t *stime,
- ktime_t *etime);
+extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc,
+ unsigned int flags, int *vpos, int *hpos,
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode);
extern bool radeon_is_px(struct drm_device *dev);
extern const struct drm_ioctl_desc radeon_ioctls_kms[];
extern int radeon_max_kms_ioctl;
extern int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv);
extern void radeon_do_release(struct drm_device * dev);
-extern u32 radeon_get_vblank_counter(struct drm_device *dev, int crtc);
-extern int radeon_enable_vblank(struct drm_device *dev, int crtc);
-extern void radeon_disable_vblank(struct drm_device *dev, int crtc);
+extern u32 radeon_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
+extern int radeon_enable_vblank(struct drm_device *dev, unsigned int pipe);
+extern void radeon_disable_vblank(struct drm_device *dev, unsigned int pipe);
extern irqreturn_t radeon_driver_irq_handler(int irq, void *arg);
extern void radeon_driver_irq_preinstall(struct drm_device * dev);
extern int radeon_driver_irq_postinstall(struct drm_device *dev);
struct radeon_device *rdev;
};
-/**
- * radeon_fb_helper_set_par - Hide cursor on CRTCs used by fbdev.
- *
- * @info: fbdev info
- *
- * This function hides the cursor on all CRTCs used by fbdev.
- */
-static int radeon_fb_helper_set_par(struct fb_info *info)
-{
- int ret;
-
- ret = drm_fb_helper_set_par(info);
-
- /* XXX: with universal plane support fbdev will automatically disable
- * all non-primary planes (including the cursor)
- */
- if (ret == 0) {
- struct drm_fb_helper *fb_helper = info->par;
- int i;
-
- for (i = 0; i < fb_helper->crtc_count; i++) {
- struct drm_crtc *crtc = fb_helper->crtc_info[i].mode_set.crtc;
-
- radeon_crtc_cursor_set2(crtc, NULL, 0, 0, 0, 0, 0);
- }
- }
-
- return ret;
-}
-
static struct fb_ops radeonfb_ops = {
.owner = THIS_MODULE,
.fb_check_var = drm_fb_helper_check_var,
- .fb_set_par = radeon_fb_helper_set_par,
+ .fb_set_par = drm_fb_helper_set_par,
.fb_fillrect = drm_fb_helper_cfb_fillrect,
.fb_copyarea = drm_fb_helper_cfb_copyarea,
.fb_imageblit = drm_fb_helper_cfb_imageblit,
{
drm_fb_helper_remove_one_connector(&rdev->mode_info.rfbdev->helper, connector);
}
+
+void radeon_fbdev_restore_mode(struct radeon_device *rdev)
+{
+ struct radeon_fbdev *rfbdev = rdev->mode_info.rfbdev;
+ struct drm_fb_helper *fb_helper;
+ int ret;
+
+ if (!rfbdev)
+ return;
+
+ fb_helper = &rfbdev->helper;
+
+ ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper);
+ if (ret)
+ DRM_DEBUG("failed to restore crtc mode\n");
+}
RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg);
}
-int radeon_enable_vblank(struct drm_device *dev, int crtc)
+int radeon_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
drm_radeon_private_t *dev_priv = dev->dev_private;
if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
- switch (crtc) {
+ switch (pipe) {
case 0:
r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 1);
break;
r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 1);
break;
default:
- DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
- crtc);
+ DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
+ pipe);
return -EINVAL;
}
} else {
- switch (crtc) {
+ switch (pipe) {
case 0:
radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 1);
break;
radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 1);
break;
default:
- DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
- crtc);
+ DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
+ pipe);
return -EINVAL;
}
}
return 0;
}
-void radeon_disable_vblank(struct drm_device *dev, int crtc)
+void radeon_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
drm_radeon_private_t *dev_priv = dev->dev_private;
if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
- switch (crtc) {
+ switch (pipe) {
case 0:
r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 0);
break;
r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 0);
break;
default:
- DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
- crtc);
+ DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
+ pipe);
break;
}
} else {
- switch (crtc) {
+ switch (pipe) {
case 0:
radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 0);
break;
radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 0);
break;
default:
- DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
- crtc);
+ DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
+ pipe);
break;
}
}
return ret;
}
-u32 radeon_get_vblank_counter(struct drm_device *dev, int crtc)
+u32 radeon_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
drm_radeon_private_t *dev_priv = dev->dev_private;
return -EINVAL;
}
- if (crtc < 0 || crtc > 1) {
- DRM_ERROR("Invalid crtc %d\n", crtc);
+ if (pipe > 1) {
+ DRM_ERROR("Invalid crtc %u\n", pipe);
return -EINVAL;
}
if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
- if (crtc == 0)
+ if (pipe == 0)
return RADEON_READ(R500_D1CRTC_FRAME_COUNT);
else
return RADEON_READ(R500_D2CRTC_FRAME_COUNT);
} else {
- if (crtc == 0)
+ if (pipe == 0)
return RADEON_READ(RADEON_CRTC_CRNT_FRAME);
else
return RADEON_READ(RADEON_CRTC2_CRNT_FRAME);
struct drm_file *applier,
uint32_t *value)
{
- mutex_lock(&dev->struct_mutex);
+ struct radeon_device *rdev = dev->dev_private;
+
+ mutex_lock(&rdev->gem.mutex);
if (*value == 1) {
/* wants rights */
if (!*owner)
*owner = NULL;
}
*value = *owner == applier ? 1 : 0;
- mutex_unlock(&dev->struct_mutex);
+ mutex_unlock(&rdev->gem.mutex);
}
/*
* Outdated mess for old drm with Xorg being in charge (void function now).
*/
/**
- * radeon_driver_firstopen_kms - drm callback for last close
+ * radeon_driver_lastclose_kms - drm callback for last close
*
* @dev: drm dev pointer
*
- * Switch vga switcheroo state after last close (all asics).
+ * Switch vga_switcheroo state after last close (all asics).
*/
void radeon_driver_lastclose_kms(struct drm_device *dev)
{
+ struct radeon_device *rdev = dev->dev_private;
+
+ radeon_fbdev_restore_mode(rdev);
vga_switcheroo_process_delayed_switch();
}
struct drm_file *file_priv)
{
struct radeon_device *rdev = dev->dev_private;
+
+ mutex_lock(&rdev->gem.mutex);
if (rdev->hyperz_filp == file_priv)
rdev->hyperz_filp = NULL;
if (rdev->cmask_filp == file_priv)
rdev->cmask_filp = NULL;
+ mutex_unlock(&rdev->gem.mutex);
+
radeon_uvd_free_handles(rdev, file_priv);
radeon_vce_free_handles(rdev, file_priv);
}
/* Helper routine in DRM core does all the work: */
return drm_calc_vbltimestamp_from_scanoutpos(dev, crtc, max_error,
vblank_time, flags,
- drmcrtc, &drmcrtc->hwmode);
+ &drmcrtc->hwmode);
}
-#define KMS_INVALID_IOCTL(name) \
-static int name(struct drm_device *dev, void *data, struct drm_file \
- *file_priv) \
-{ \
- DRM_ERROR("invalid ioctl with kms %s\n", __func__); \
- return -EINVAL; \
-}
-
-/*
- * All these ioctls are invalid in kms world.
- */
-KMS_INVALID_IOCTL(radeon_cp_init_kms)
-KMS_INVALID_IOCTL(radeon_cp_start_kms)
-KMS_INVALID_IOCTL(radeon_cp_stop_kms)
-KMS_INVALID_IOCTL(radeon_cp_reset_kms)
-KMS_INVALID_IOCTL(radeon_cp_idle_kms)
-KMS_INVALID_IOCTL(radeon_cp_resume_kms)
-KMS_INVALID_IOCTL(radeon_engine_reset_kms)
-KMS_INVALID_IOCTL(radeon_fullscreen_kms)
-KMS_INVALID_IOCTL(radeon_cp_swap_kms)
-KMS_INVALID_IOCTL(radeon_cp_clear_kms)
-KMS_INVALID_IOCTL(radeon_cp_vertex_kms)
-KMS_INVALID_IOCTL(radeon_cp_indices_kms)
-KMS_INVALID_IOCTL(radeon_cp_texture_kms)
-KMS_INVALID_IOCTL(radeon_cp_stipple_kms)
-KMS_INVALID_IOCTL(radeon_cp_indirect_kms)
-KMS_INVALID_IOCTL(radeon_cp_vertex2_kms)
-KMS_INVALID_IOCTL(radeon_cp_cmdbuf_kms)
-KMS_INVALID_IOCTL(radeon_cp_getparam_kms)
-KMS_INVALID_IOCTL(radeon_cp_flip_kms)
-KMS_INVALID_IOCTL(radeon_mem_alloc_kms)
-KMS_INVALID_IOCTL(radeon_mem_free_kms)
-KMS_INVALID_IOCTL(radeon_mem_init_heap_kms)
-KMS_INVALID_IOCTL(radeon_irq_emit_kms)
-KMS_INVALID_IOCTL(radeon_irq_wait_kms)
-KMS_INVALID_IOCTL(radeon_cp_setparam_kms)
-KMS_INVALID_IOCTL(radeon_surface_alloc_kms)
-KMS_INVALID_IOCTL(radeon_surface_free_kms)
-
-
const struct drm_ioctl_desc radeon_ioctls_kms[] = {
- DRM_IOCTL_DEF_DRV(RADEON_CP_INIT, radeon_cp_init_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(RADEON_CP_START, radeon_cp_start_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(RADEON_CP_STOP, radeon_cp_stop_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(RADEON_CP_RESET, radeon_cp_reset_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(RADEON_CP_IDLE, radeon_cp_idle_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_CP_RESUME, radeon_cp_resume_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_RESET, radeon_engine_reset_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_FULLSCREEN, radeon_fullscreen_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_SWAP, radeon_cp_swap_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_CLEAR, radeon_cp_clear_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_VERTEX, radeon_cp_vertex_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_INDICES, radeon_cp_indices_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_TEXTURE, radeon_cp_texture_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_STIPPLE, radeon_cp_stipple_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_INDIRECT, radeon_cp_indirect_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(RADEON_VERTEX2, radeon_cp_vertex2_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_CMDBUF, radeon_cp_cmdbuf_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_GETPARAM, radeon_cp_getparam_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_FLIP, radeon_cp_flip_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_ALLOC, radeon_mem_alloc_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_FREE, radeon_mem_free_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_INIT_HEAP, radeon_mem_init_heap_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
- DRM_IOCTL_DEF_DRV(RADEON_IRQ_EMIT, radeon_irq_emit_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_IRQ_WAIT, radeon_irq_wait_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_SETPARAM, radeon_cp_setparam_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_SURF_ALLOC, radeon_surface_alloc_kms, DRM_AUTH),
- DRM_IOCTL_DEF_DRV(RADEON_SURF_FREE, radeon_surface_free_kms, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_CP_INIT, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(RADEON_CP_START, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(RADEON_CP_STOP, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(RADEON_CP_RESET, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(RADEON_CP_IDLE, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_CP_RESUME, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_RESET, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_FULLSCREEN, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_SWAP, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_CLEAR, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_VERTEX, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_INDICES, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_TEXTURE, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_STIPPLE, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_INDIRECT, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(RADEON_VERTEX2, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_CMDBUF, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_GETPARAM, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_FLIP, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_ALLOC, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_FREE, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_INIT_HEAP, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+ DRM_IOCTL_DEF_DRV(RADEON_IRQ_EMIT, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_IRQ_WAIT, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_SETPARAM, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_SURF_ALLOC, drm_invalid_op, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_SURF_FREE, drm_invalid_op, DRM_AUTH),
/* KMS */
- DRM_IOCTL_DEF_DRV(RADEON_GEM_INFO, radeon_gem_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_CREATE, radeon_gem_create_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_MMAP, radeon_gem_mmap_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_DOMAIN, radeon_gem_set_domain_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_PREAD, radeon_gem_pread_ioctl, DRM_AUTH|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_PWRITE, radeon_gem_pwrite_ioctl, DRM_AUTH|DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_WAIT_IDLE, radeon_gem_wait_idle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_CS, radeon_cs_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_INFO, radeon_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_TILING, radeon_gem_set_tiling_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_GET_TILING, radeon_gem_get_tiling_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_BUSY, radeon_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_VA, radeon_gem_va_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_OP, radeon_gem_op_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RADEON_GEM_USERPTR, radeon_gem_userptr_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_INFO, radeon_gem_info_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_CREATE, radeon_gem_create_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_MMAP, radeon_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_DOMAIN, radeon_gem_set_domain_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_PREAD, radeon_gem_pread_ioctl, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_PWRITE, radeon_gem_pwrite_ioctl, DRM_AUTH),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_WAIT_IDLE, radeon_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_CS, radeon_cs_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_INFO, radeon_info_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_TILING, radeon_gem_set_tiling_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_GET_TILING, radeon_gem_get_tiling_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_BUSY, radeon_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_VA, radeon_gem_va_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_OP, radeon_gem_op_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(RADEON_GEM_USERPTR, radeon_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
};
int radeon_max_kms_ioctl = ARRAY_SIZE(radeon_ioctls_kms);
int x, int y);
extern void radeon_cursor_reset(struct drm_crtc *crtc);
-extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
- unsigned int flags,
- int *vpos, int *hpos, ktime_t *stime,
- ktime_t *etime);
+extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
+ unsigned int flags, int *vpos, int *hpos,
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode);
extern bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev);
extern struct edid *
void radeon_fbdev_fini(struct radeon_device *rdev);
void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state);
bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj);
+void radeon_fbdev_restore_mode(struct radeon_device *rdev);
void radeon_fb_output_poll_changed(struct radeon_device *rdev);
INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler);
if (rdev->pm.num_power_states > 1) {
- /* where's the best place to put these? */
- ret = device_create_file(rdev->dev, &dev_attr_power_profile);
- if (ret)
- DRM_ERROR("failed to create device file for power profile\n");
- ret = device_create_file(rdev->dev, &dev_attr_power_method);
- if (ret)
- DRM_ERROR("failed to create device file for power method\n");
-
if (radeon_debugfs_pm_init(rdev)) {
DRM_ERROR("Failed to register debugfs file for PM!\n");
}
goto dpm_failed;
rdev->pm.dpm_enabled = true;
- ret = device_create_file(rdev->dev, &dev_attr_power_dpm_state);
- if (ret)
- DRM_ERROR("failed to create device file for dpm state\n");
- ret = device_create_file(rdev->dev, &dev_attr_power_dpm_force_performance_level);
- if (ret)
- DRM_ERROR("failed to create device file for dpm state\n");
- /* XXX: these are noops for dpm but are here for backwards compat */
- ret = device_create_file(rdev->dev, &dev_attr_power_profile);
- if (ret)
- DRM_ERROR("failed to create device file for power profile\n");
- ret = device_create_file(rdev->dev, &dev_attr_power_method);
- if (ret)
- DRM_ERROR("failed to create device file for power method\n");
-
if (radeon_debugfs_pm_init(rdev)) {
DRM_ERROR("Failed to register debugfs file for dpm!\n");
}
int ret = 0;
if (rdev->pm.pm_method == PM_METHOD_DPM) {
- mutex_lock(&rdev->pm.mutex);
- ret = radeon_dpm_late_enable(rdev);
- mutex_unlock(&rdev->pm.mutex);
+ if (rdev->pm.dpm_enabled) {
+ ret = device_create_file(rdev->dev, &dev_attr_power_dpm_state);
+ if (ret)
+ DRM_ERROR("failed to create device file for dpm state\n");
+ ret = device_create_file(rdev->dev, &dev_attr_power_dpm_force_performance_level);
+ if (ret)
+ DRM_ERROR("failed to create device file for dpm state\n");
+ /* XXX: these are noops for dpm but are here for backwards compat */
+ ret = device_create_file(rdev->dev, &dev_attr_power_profile);
+ if (ret)
+ DRM_ERROR("failed to create device file for power profile\n");
+ ret = device_create_file(rdev->dev, &dev_attr_power_method);
+ if (ret)
+ DRM_ERROR("failed to create device file for power method\n");
+
+ mutex_lock(&rdev->pm.mutex);
+ ret = radeon_dpm_late_enable(rdev);
+ mutex_unlock(&rdev->pm.mutex);
+ if (ret) {
+ rdev->pm.dpm_enabled = false;
+ DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n");
+ } else {
+ /* set the dpm state for PX since there won't be
+ * a modeset to call this.
+ */
+ radeon_pm_compute_clocks(rdev);
+ }
+ }
+ } else {
+ if (rdev->pm.num_power_states > 1) {
+ /* where's the best place to put these? */
+ ret = device_create_file(rdev->dev, &dev_attr_power_profile);
+ if (ret)
+ DRM_ERROR("failed to create device file for power profile\n");
+ ret = device_create_file(rdev->dev, &dev_attr_power_method);
+ if (ret)
+ DRM_ERROR("failed to create device file for power method\n");
+ }
}
return ret;
}
*/
for (crtc = 0; (crtc < rdev->num_crtc) && in_vbl; crtc++) {
if (rdev->pm.active_crtcs & (1 << crtc)) {
- vbl_status = radeon_get_crtc_scanoutpos(rdev->ddev, crtc, 0, &vpos, &hpos, NULL, NULL);
+ vbl_status = radeon_get_crtc_scanoutpos(rdev->ddev, crtc, 0,
+ &vpos, &hpos, NULL, NULL,
+ &rdev->mode_info.crtcs[crtc]->base.hwmode);
if ((vbl_status & DRM_SCANOUTPOS_VALID) &&
!(vbl_status & DRM_SCANOUTPOS_IN_VBLANK))
in_vbl = false;
man->available_caching = TTM_PL_MASK_CACHING;
man->default_caching = TTM_PL_FLAG_CACHED;
man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (rdev->flags & RADEON_IS_AGP) {
if (!rdev->ddev->agp) {
DRM_ERROR("AGP is not enabled for memory type %u\n",
/* system memory */
return 0;
case TTM_PL_TT:
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (rdev->flags & RADEON_IS_AGP) {
/* RADEON_IS_AGP is set only if AGP is active */
mem->bus.offset = mem->start << PAGE_SHIFT;
struct radeon_ttm_tt *gtt;
rdev = radeon_get_rdev(bdev);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_create(bdev, rdev->ddev->agp->bridge,
size, page_flags, dummy_read_page);
}
rdev = radeon_get_rdev(ttm->bdev);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
return;
rdev = radeon_get_rdev(ttm->bdev);
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
if (rdev->flags & RADEON_IS_AGP) {
ttm_agp_tt_unpopulate(ttm);
return;
{ PCI_VENDOR_ID_ATI, 0x6810, 0x1462, 0x3036, 0, 120000 },
{ PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0xe271, 0, 120000 },
{ PCI_VENDOR_ID_ATI, 0x6810, 0x174b, 0xe271, 85000, 90000 },
+ { PCI_VENDOR_ID_ATI, 0x6811, 0x1762, 0x2015, 0, 120000 },
+ { PCI_VENDOR_ID_ATI, 0x6811, 0x1043, 0x2015, 0, 120000 },
{ 0, 0, 0, 0 },
};
config DRM_RCAR_DU
tristate "DRM Support for R-Car Display Unit"
- depends on DRM && ARM && HAVE_DMA_ATTRS
+ depends on DRM && ARM && HAVE_DMA_ATTRS && OF
depends on ARCH_SHMOBILE || COMPILE_TEST
select DRM_KMS_HELPER
select DRM_KMS_CMA_HELPER
.num_lvds = 2,
};
+/* M2-W (r8a7791) and M2-N (r8a7793) are identical */
static const struct rcar_du_device_info rcar_du_r8a7791_info = {
.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
| RCAR_DU_FEATURE_EXT_CTRL_REGS,
.num_crtcs = 2,
.routes = {
- /* R8A7791 has one RGB output, one LVDS output and one
+ /* R8A779[13] has one RGB output, one LVDS output and one
* (currently unsupported) TCON output.
*/
[RCAR_DU_OUTPUT_DPAD0] = {
- .possible_crtcs = BIT(1),
+ .possible_crtcs = BIT(1) | BIT(0),
.encoder_type = DRM_MODE_ENCODER_NONE,
.port = 0,
},
.num_lvds = 1,
};
-static const struct platform_device_id rcar_du_id_table[] = {
- { "rcar-du-r8a7779", (kernel_ulong_t)&rcar_du_r8a7779_info },
- { "rcar-du-r8a7790", (kernel_ulong_t)&rcar_du_r8a7790_info },
- { "rcar-du-r8a7791", (kernel_ulong_t)&rcar_du_r8a7791_info },
- { }
+static const struct rcar_du_device_info rcar_du_r8a7794_info = {
+ .features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
+ | RCAR_DU_FEATURE_EXT_CTRL_REGS,
+ .num_crtcs = 2,
+ .routes = {
+ /* R8A7794 has two RGB outputs and one (currently unsupported)
+ * TCON output.
+ */
+ [RCAR_DU_OUTPUT_DPAD0] = {
+ .possible_crtcs = BIT(0),
+ .encoder_type = DRM_MODE_ENCODER_NONE,
+ .port = 0,
+ },
+ [RCAR_DU_OUTPUT_DPAD1] = {
+ .possible_crtcs = BIT(1),
+ .encoder_type = DRM_MODE_ENCODER_NONE,
+ .port = 1,
+ },
+ },
+ .num_lvds = 0,
};
-MODULE_DEVICE_TABLE(platform, rcar_du_id_table);
-
static const struct of_device_id rcar_du_of_table[] = {
{ .compatible = "renesas,du-r8a7779", .data = &rcar_du_r8a7779_info },
{ .compatible = "renesas,du-r8a7790", .data = &rcar_du_r8a7790_info },
{ .compatible = "renesas,du-r8a7791", .data = &rcar_du_r8a7791_info },
+ { .compatible = "renesas,du-r8a7793", .data = &rcar_du_r8a7791_info },
+ { .compatible = "renesas,du-r8a7794", .data = &rcar_du_r8a7794_info },
{ }
};
init_waitqueue_head(&rcdu->commit.wait);
rcdu->dev = &pdev->dev;
- rcdu->info = np ? of_match_device(rcar_du_of_table, rcdu->dev)->data
- : (void *)platform_get_device_id(pdev)->driver_data;
+ rcdu->info = of_match_device(rcar_du_of_table, rcdu->dev)->data;
rcdu->ddev = dev;
dev->dev_private = rcdu;
drm_fbdev_cma_restore_mode(rcdu->fbdev);
}
-static int rcar_du_enable_vblank(struct drm_device *dev, int crtc)
+static int rcar_du_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct rcar_du_device *rcdu = dev->dev_private;
- rcar_du_crtc_enable_vblank(&rcdu->crtcs[crtc], true);
+ rcar_du_crtc_enable_vblank(&rcdu->crtcs[pipe], true);
return 0;
}
-static void rcar_du_disable_vblank(struct drm_device *dev, int crtc)
+static void rcar_du_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct rcar_du_device *rcdu = dev->dev_private;
- rcar_du_crtc_enable_vblank(&rcdu->crtcs[crtc], false);
+ rcar_du_crtc_enable_vblank(&rcdu->crtcs[pipe], false);
}
static const struct file_operations rcar_du_fops = {
.preclose = rcar_du_preclose,
.lastclose = rcar_du_lastclose,
.set_busid = drm_platform_set_busid,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = rcar_du_enable_vblank,
.disable_vblank = rcar_du_disable_vblank,
.gem_free_object = drm_gem_cma_free_object,
.pm = &rcar_du_pm_ops,
.of_match_table = rcar_du_of_table,
},
- .id_table = rcar_du_id_table,
};
module_platform_driver(rcar_du_platform_driver);
u32 defr8 = DEFR8_CODE | DEFR8_DEFE8;
/* The DEFR8 register for the first group also controls RGB output
- * routing to DPAD0
+ * routing to DPAD0 for DU instances that support it.
*/
- if (rgrp->index == 0)
+ if (rgrp->dev->info->routes[RCAR_DU_OUTPUT_DPAD0].possible_crtcs > 1 &&
+ rgrp->index == 0)
defr8 |= DEFR8_DRGBS_DU(rgrp->dev->dpad0_source);
rcar_du_group_write(rgrp, DEFR8, defr8);
/* Apply the atomic update. */
drm_atomic_helper_commit_modeset_disables(dev, old_state);
drm_atomic_helper_commit_modeset_enables(dev, old_state);
- drm_atomic_helper_commit_planes(dev, old_state);
+ drm_atomic_helper_commit_planes(dev, old_state, false);
drm_atomic_helper_wait_for_vblanks(dev, old_state);
.atomic_update = rcar_du_plane_atomic_update,
};
-static void rcar_du_plane_reset(struct drm_plane *plane)
-{
- struct rcar_du_plane_state *state;
-
- if (plane->state && plane->state->fb)
- drm_framebuffer_unreference(plane->state->fb);
-
- kfree(plane->state);
- plane->state = NULL;
-
- state = kzalloc(sizeof(*state), GFP_KERNEL);
- if (state == NULL)
- return;
-
- state->hwindex = -1;
- state->alpha = 255;
- state->colorkey = RCAR_DU_COLORKEY_NONE;
- state->zpos = plane->type == DRM_PLANE_TYPE_PRIMARY ? 0 : 1;
-
- plane->state = &state->state;
- plane->state->plane = plane;
-}
-
static struct drm_plane_state *
rcar_du_plane_atomic_duplicate_state(struct drm_plane *plane)
{
kfree(to_rcar_plane_state(state));
}
+static void rcar_du_plane_reset(struct drm_plane *plane)
+{
+ struct rcar_du_plane_state *state;
+
+ if (plane->state) {
+ rcar_du_plane_atomic_destroy_state(plane, plane->state);
+ plane->state = NULL;
+ }
+
+ state = kzalloc(sizeof(*state), GFP_KERNEL);
+ if (state == NULL)
+ return;
+
+ state->hwindex = -1;
+ state->alpha = 255;
+ state->colorkey = RCAR_DU_COLORKEY_NONE;
+ state->zpos = plane->type == DRM_PLANE_TYPE_PRIMARY ? 0 : 1;
+
+ plane->state = &state->state;
+ plane->state->plane = plane;
+}
+
static int rcar_du_plane_atomic_set_property(struct drm_plane *plane,
struct drm_plane_state *state,
struct drm_property *property,
#include <drm/drmP.h>
#include <drm/drm_crtc_helper.h>
#include <drm/drm_fb_helper.h>
+#include <drm/drm_of.h>
#include <linux/dma-mapping.h>
#include <linux/pm_runtime.h>
#include <linux/module.h>
return NULL;
}
-static int rockchip_drm_crtc_enable_vblank(struct drm_device *dev, int pipe)
+static int rockchip_drm_crtc_enable_vblank(struct drm_device *dev,
+ unsigned int pipe)
{
struct rockchip_drm_private *priv = dev->dev_private;
struct drm_crtc *crtc = rockchip_crtc_from_pipe(dev, pipe);
return 0;
}
-static void rockchip_drm_crtc_disable_vblank(struct drm_device *dev, int pipe)
+static void rockchip_drm_crtc_disable_vblank(struct drm_device *dev,
+ unsigned int pipe)
{
struct rockchip_drm_private *priv = dev->dev_private;
struct drm_crtc *crtc = rockchip_crtc_from_pipe(dev, pipe);
.load = rockchip_drm_load,
.unload = rockchip_drm_unload,
.lastclose = rockchip_drm_lastclose,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = rockchip_drm_crtc_enable_vblank,
.disable_vblank = rockchip_drm_crtc_disable_vblank,
.gem_vm_ops = &rockchip_drm_vm_ops,
return dev->of_node == np;
}
-static void rockchip_add_endpoints(struct device *dev,
- struct component_match **match,
- struct device_node *port)
-{
- struct device_node *ep, *remote;
-
- for_each_child_of_node(port, ep) {
- remote = of_graph_get_remote_port_parent(ep);
- if (!remote || !of_device_is_available(remote)) {
- of_node_put(remote);
- continue;
- } else if (!of_device_is_available(remote->parent)) {
- dev_warn(dev, "parent device of %s is not available\n",
- remote->full_name);
- of_node_put(remote);
- continue;
- }
-
- component_match_add(dev, match, compare_of, remote);
- of_node_put(remote);
- }
-}
-
static int rockchip_drm_bind(struct device *dev)
{
struct drm_device *drm;
static int rockchip_drm_platform_probe(struct platform_device *pdev)
{
- struct device *dev = &pdev->dev;
- struct component_match *match = NULL;
- struct device_node *np = dev->of_node;
- struct device_node *port;
- int i;
-
- if (!np)
- return -ENODEV;
- /*
- * Bind the crtc ports first, so that
- * drm_of_find_possible_crtcs called from encoder .bind callbacks
- * works as expected.
- */
- for (i = 0;; i++) {
- port = of_parse_phandle(np, "ports", i);
- if (!port)
- break;
-
- if (!of_device_is_available(port->parent)) {
- of_node_put(port);
- continue;
- }
-
- component_match_add(dev, &match, compare_of, port->parent);
- of_node_put(port);
- }
+ int ret = drm_of_component_probe(&pdev->dev, compare_of,
+ &rockchip_drm_ops);
- if (i == 0) {
- dev_err(dev, "missing 'ports' property\n");
+ /* keep compatibility with old code that was returning -ENODEV */
+ if (ret == -EINVAL)
return -ENODEV;
- }
- if (!match) {
- dev_err(dev, "No available vop found for display-subsystem.\n");
- return -ENODEV;
- }
- /*
- * For each bound crtc, bind the encoders attached to its
- * remote endpoint.
- */
- for (i = 0;; i++) {
- port = of_parse_phandle(np, "ports", i);
- if (!port)
- break;
-
- if (!of_device_is_available(port->parent)) {
- of_node_put(port);
- continue;
- }
-
- rockchip_add_endpoints(dev, &match, port);
- of_node_put(port);
- }
-
- return component_master_add_with_match(dev, &rockchip_drm_ops, match);
+ return ret;
}
static int rockchip_drm_platform_remove(struct platform_device *pdev)
int rockchip_gem_mmap_buf(struct drm_gem_object *obj,
struct vm_area_struct *vma)
{
- struct drm_device *drm = obj->dev;
int ret;
- mutex_lock(&drm->struct_mutex);
ret = drm_gem_mmap_obj(obj, obj->size, vma);
- mutex_unlock(&drm->struct_mutex);
if (ret)
return ret;
return IRQ_HANDLED;
}
-static int shmob_drm_enable_vblank(struct drm_device *dev, int crtc)
+static int shmob_drm_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct shmob_drm_device *sdev = dev->dev_private;
return 0;
}
-static void shmob_drm_disable_vblank(struct drm_device *dev, int crtc)
+static void shmob_drm_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct shmob_drm_device *sdev = dev->dev_private;
.preclose = shmob_drm_preclose,
.set_busid = drm_platform_set_busid,
.irq_handler = shmob_drm_irq,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = shmob_drm_enable_vblank,
.disable_vblank = shmob_drm_disable_vblank,
.gem_free_object = drm_gem_cma_free_object,
struct idr object_idr;
} drm_sis_private_t;
+struct sis_file_private {
+ struct list_head obj_list;
+};
+
extern int sis_idle(struct drm_device *dev);
extern void sis_reclaim_buffers_locked(struct drm_device *dev,
struct drm_file *file_priv);
return 0;
}
-int sti_crtc_enable_vblank(struct drm_device *dev, int crtc)
+int sti_crtc_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
struct sti_private *dev_priv = dev->dev_private;
struct sti_compositor *compo = dev_priv->compo;
DRM_DEBUG_DRIVER("\n");
- if (sti_vtg_register_client(crtc == STI_MIXER_MAIN ?
+ if (sti_vtg_register_client(pipe == STI_MIXER_MAIN ?
compo->vtg_main : compo->vtg_aux,
- vtg_vblank_nb, crtc)) {
+ vtg_vblank_nb, pipe)) {
DRM_ERROR("Cannot register VTG notifier\n");
return -EINVAL;
}
}
EXPORT_SYMBOL(sti_crtc_enable_vblank);
-void sti_crtc_disable_vblank(struct drm_device *drm_dev, int crtc)
+void sti_crtc_disable_vblank(struct drm_device *drm_dev, unsigned int pipe)
{
struct sti_private *priv = drm_dev->dev_private;
struct sti_compositor *compo = priv->compo;
DRM_DEBUG_DRIVER("\n");
- if (sti_vtg_unregister_client(crtc == STI_MIXER_MAIN ?
+ if (sti_vtg_unregister_client(pipe == STI_MIXER_MAIN ?
compo->vtg_main : compo->vtg_aux, vtg_vblank_nb))
DRM_DEBUG_DRIVER("Warning: cannot unregister VTG notifier\n");
/* free the resources of the pending requests */
- if (compo->mixer[crtc]->pending_event) {
- drm_vblank_put(drm_dev, crtc);
- compo->mixer[crtc]->pending_event = NULL;
+ if (compo->mixer[pipe]->pending_event) {
+ drm_vblank_put(drm_dev, pipe);
+ compo->mixer[pipe]->pending_event = NULL;
}
}
EXPORT_SYMBOL(sti_crtc_disable_vblank);
int sti_crtc_init(struct drm_device *drm_dev, struct sti_mixer *mixer,
struct drm_plane *primary, struct drm_plane *cursor);
-int sti_crtc_enable_vblank(struct drm_device *dev, int crtc);
-void sti_crtc_disable_vblank(struct drm_device *dev, int crtc);
+int sti_crtc_enable_vblank(struct drm_device *dev, unsigned int pipe);
+void sti_crtc_disable_vblank(struct drm_device *dev, unsigned int pipe);
int sti_crtc_vblank_cb(struct notifier_block *nb,
unsigned long event, void *data);
bool sti_crtc_is_main(struct drm_crtc *drm_crtc);
*/
drm_atomic_helper_commit_modeset_disables(drm, state);
- drm_atomic_helper_commit_planes(drm, state);
+ drm_atomic_helper_commit_planes(drm, state, false);
drm_atomic_helper_commit_modeset_enables(drm, state);
drm_atomic_helper_wait_for_vblanks(drm, state);
.dumb_destroy = drm_gem_dumb_destroy,
.fops = &sti_driver_fops,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = sti_crtc_enable_vblank,
.disable_vblank = sti_crtc_disable_vblank,
};
static int tegra_plane_prepare_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state)
{
return 0;
}
static void tegra_plane_cleanup_fb(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *old_fb)
{
}
*/
if (msg->size < 1) {
switch (msg->request & ~DP_AUX_I2C_MOT) {
+ case DP_AUX_I2C_WRITE_STATUS_UPDATE:
case DP_AUX_I2C_WRITE:
case DP_AUX_I2C_READ:
value = DPAUX_DP_AUXCTL_CMD_ADDRESS_ONLY;
break;
- case DP_AUX_I2C_STATUS:
+ case DP_AUX_I2C_WRITE_STATUS_UPDATE:
if (msg->request & DP_AUX_I2C_MOT)
value |= DPAUX_DP_AUXCTL_CMD_MOT_RQ;
else
*/
drm_atomic_helper_commit_modeset_disables(drm, state);
- drm_atomic_helper_commit_planes(drm, state);
+ drm_atomic_helper_commit_planes(drm, state, false);
drm_atomic_helper_commit_modeset_enables(drm, state);
drm_atomic_helper_wait_for_vblanks(drm, state);
static const struct drm_ioctl_desc tegra_drm_ioctls[] = {
#ifdef CONFIG_DRM_TEGRA_STAGING
- DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_WAIT, tegra_syncpt_wait, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_OPEN_CHANNEL, tegra_open_channel, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_TILING, tegra_gem_set_tiling, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_TILING, tegra_gem_get_tiling, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_FLAGS, tegra_gem_set_flags, DRM_UNLOCKED),
- DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_FLAGS, tegra_gem_get_flags, DRM_UNLOCKED),
+ DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_WAIT, tegra_syncpt_wait, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_OPEN_CHANNEL, tegra_open_channel, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_TILING, tegra_gem_set_tiling, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_TILING, tegra_gem_get_tiling, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_FLAGS, tegra_gem_set_flags, 0),
+ DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_FLAGS, tegra_gem_get_flags, 0),
#endif
};
return NULL;
}
-static u32 tegra_drm_get_vblank_counter(struct drm_device *drm, int pipe)
+static u32 tegra_drm_get_vblank_counter(struct drm_device *drm,
+ unsigned int pipe)
{
struct drm_crtc *crtc = tegra_crtc_from_pipe(drm, pipe);
struct tegra_dc *dc = to_tegra_dc(crtc);
return tegra_dc_get_vblank_counter(dc);
}
-static int tegra_drm_enable_vblank(struct drm_device *drm, int pipe)
+static int tegra_drm_enable_vblank(struct drm_device *drm, unsigned int pipe)
{
struct drm_crtc *crtc = tegra_crtc_from_pipe(drm, pipe);
struct tegra_dc *dc = to_tegra_dc(crtc);
return 0;
}
-static void tegra_drm_disable_vblank(struct drm_device *drm, int pipe)
+static void tegra_drm_disable_vblank(struct drm_device *drm, unsigned int pipe)
{
struct drm_crtc *crtc = tegra_crtc_from_pipe(drm, pipe);
struct tegra_dc *dc = to_tegra_dc(crtc);
tilcdc_clear(dev, reg, mask);
}
-static int tilcdc_enable_vblank(struct drm_device *dev, int crtc)
+static int tilcdc_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
enable_vblank(dev, true);
return 0;
}
-static void tilcdc_disable_vblank(struct drm_device *dev, int crtc)
+static void tilcdc_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
enable_vblank(dev, false);
}
.irq_preinstall = tilcdc_irq_preinstall,
.irq_postinstall = tilcdc_irq_postinstall,
.irq_uninstall = tilcdc_irq_uninstall,
- .get_vblank_counter = drm_vblank_count,
+ .get_vblank_counter = drm_vblank_no_hw_counter,
.enable_vblank = tilcdc_enable_vblank,
.disable_vblank = tilcdc_disable_vblank,
.gem_free_object = drm_gem_cma_free_object,
if (ret)
return ret;
man = &bdev->man[mem_type];
+ if (!man->has_type || !man->use_type)
+ continue;
type_ok = ttm_bo_mt_compatible(man, mem_type, place,
&cur_flags);
if (!type_ok)
continue;
+ type_found = true;
cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
cur_flags);
/*
if (mem_type == TTM_PL_SYSTEM)
break;
- if (man->has_type && man->use_type) {
- type_found = true;
- ret = (*man->func->get_node)(man, bo, place, mem);
- if (unlikely(ret))
- return ret;
- }
+ ret = (*man->func->get_node)(man, bo, place, mem);
+ if (unlikely(ret))
+ return ret;
+
if (mem->mm_node)
break;
}
return 0;
}
- if (!type_found)
- return -EINVAL;
-
for (i = 0; i < placement->num_busy_placement; ++i) {
const struct ttm_place *place = &placement->busy_placement[i];
if (ret)
return ret;
man = &bdev->man[mem_type];
- if (!man->has_type)
+ if (!man->has_type || !man->use_type)
continue;
if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
continue;
+ type_found = true;
cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
cur_flags);
/*
if (ret == -ERESTARTSYS)
has_erestartsys = true;
}
- ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
- return ret;
+
+ if (!type_found) {
+ printk(KERN_ERR TTM_PFX "No compatible memory type found.\n");
+ return -EINVAL;
+ }
+
+ return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
}
EXPORT_SYMBOL(ttm_bo_mem_space);
--- /dev/null
+config DRM_VC4
+ tristate "Broadcom VC4 Graphics"
+ depends on ARCH_BCM2835 || COMPILE_TEST
+ depends on DRM
+ select DRM_KMS_HELPER
+ select DRM_KMS_CMA_HELPER
+ help
+ Choose this option if you have a system that has a Broadcom
+ VC4 GPU, such as the Raspberry Pi or other BCM2708/BCM2835.
+
+ This driver requires that "avoid_warnings=2" be present in
+ the config.txt for the firmware, to keep it from smashing
+ our display setup.
--- /dev/null
+ccflags-y := -Iinclude/drm
+
+# Please keep these build lists sorted!
+
+# core driver code
+vc4-y := \
+ vc4_bo.o \
+ vc4_crtc.o \
+ vc4_drv.o \
+ vc4_kms.o \
+ vc4_hdmi.o \
+ vc4_hvs.o \
+ vc4_plane.o
+
+vc4-$(CONFIG_DEBUG_FS) += vc4_debugfs.o
+
+obj-$(CONFIG_DRM_VC4) += vc4.o
--- /dev/null
+/*
+ * Copyright © 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* DOC: VC4 GEM BO management support.
+ *
+ * The VC4 GPU architecture (both scanout and rendering) has direct
+ * access to system memory with no MMU in between. To support it, we
+ * use the GEM CMA helper functions to allocate contiguous ranges of
+ * physical memory for our BOs.
+ */
+
+#include "vc4_drv.h"
+
+struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size)
+{
+ struct drm_gem_cma_object *cma_obj;
+
+ cma_obj = drm_gem_cma_create(dev, size);
+ if (IS_ERR(cma_obj))
+ return NULL;
+ else
+ return to_vc4_bo(&cma_obj->base);
+}
+
+int vc4_dumb_create(struct drm_file *file_priv,
+ struct drm_device *dev,
+ struct drm_mode_create_dumb *args)
+{
+ int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
+ struct vc4_bo *bo = NULL;
+ int ret;
+
+ if (args->pitch < min_pitch)
+ args->pitch = min_pitch;
+
+ if (args->size < args->pitch * args->height)
+ args->size = args->pitch * args->height;
+
+ bo = vc4_bo_create(dev, roundup(args->size, PAGE_SIZE));
+ if (!bo)
+ return -ENOMEM;
+
+ ret = drm_gem_handle_create(file_priv, &bo->base.base, &args->handle);
+ drm_gem_object_unreference_unlocked(&bo->base.base);
+
+ return ret;
+}
--- /dev/null
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/**
+ * DOC: VC4 CRTC module
+ *
+ * In VC4, the Pixel Valve is what most closely corresponds to the
+ * DRM's concept of a CRTC. The PV generates video timings from the
+ * output's clock plus its configuration. It pulls scaled pixels from
+ * the HVS at that timing, and feeds it to the encoder.
+ *
+ * However, the DRM CRTC also collects the configuration of all the
+ * DRM planes attached to it. As a result, this file also manages
+ * setup of the VC4 HVS's display elements on the CRTC.
+ *
+ * The 2835 has 3 different pixel valves. pv0 in the audio power
+ * domain feeds DSI0 or DPI, while pv1 feeds DS1 or SMI. pv2 in the
+ * image domain can feed either HDMI or the SDTV controller. The
+ * pixel valve chooses from the CPRMAN clocks (HSM for HDMI, VEC for
+ * SDTV, etc.) according to which output type is chosen in the mux.
+ *
+ * For power management, the pixel valve's registers are all clocked
+ * by the AXI clock, while the timings and FIFOs make use of the
+ * output-specific clock. Since the encoders also directly consume
+ * the CPRMAN clocks, and know what timings they need, they are the
+ * ones that set the clock.
+ */
+
+#include "drm_atomic.h"
+#include "drm_atomic_helper.h"
+#include "drm_crtc_helper.h"
+#include "linux/clk.h"
+#include "linux/component.h"
+#include "linux/of_device.h"
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+struct vc4_crtc {
+ struct drm_crtc base;
+ const struct vc4_crtc_data *data;
+ void __iomem *regs;
+
+ /* Which HVS channel we're using for our CRTC. */
+ int channel;
+
+ /* Pointer to the actual hardware display list memory for the
+ * crtc.
+ */
+ u32 __iomem *dlist;
+
+ u32 dlist_size; /* in dwords */
+
+ struct drm_pending_vblank_event *event;
+};
+
+static inline struct vc4_crtc *
+to_vc4_crtc(struct drm_crtc *crtc)
+{
+ return (struct vc4_crtc *)crtc;
+}
+
+struct vc4_crtc_data {
+ /* Which channel of the HVS this pixelvalve sources from. */
+ int hvs_channel;
+
+ enum vc4_encoder_type encoder0_type;
+ enum vc4_encoder_type encoder1_type;
+};
+
+#define CRTC_WRITE(offset, val) writel(val, vc4_crtc->regs + (offset))
+#define CRTC_READ(offset) readl(vc4_crtc->regs + (offset))
+
+#define CRTC_REG(reg) { reg, #reg }
+static const struct {
+ u32 reg;
+ const char *name;
+} crtc_regs[] = {
+ CRTC_REG(PV_CONTROL),
+ CRTC_REG(PV_V_CONTROL),
+ CRTC_REG(PV_VSYNCD),
+ CRTC_REG(PV_HORZA),
+ CRTC_REG(PV_HORZB),
+ CRTC_REG(PV_VERTA),
+ CRTC_REG(PV_VERTB),
+ CRTC_REG(PV_VERTA_EVEN),
+ CRTC_REG(PV_VERTB_EVEN),
+ CRTC_REG(PV_INTEN),
+ CRTC_REG(PV_INTSTAT),
+ CRTC_REG(PV_STAT),
+ CRTC_REG(PV_HACT_ACT),
+};
+
+static void vc4_crtc_dump_regs(struct vc4_crtc *vc4_crtc)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(crtc_regs); i++) {
+ DRM_INFO("0x%04x (%s): 0x%08x\n",
+ crtc_regs[i].reg, crtc_regs[i].name,
+ CRTC_READ(crtc_regs[i].reg));
+ }
+}
+
+#ifdef CONFIG_DEBUG_FS
+int vc4_crtc_debugfs_regs(struct seq_file *m, void *unused)
+{
+ struct drm_info_node *node = (struct drm_info_node *)m->private;
+ struct drm_device *dev = node->minor->dev;
+ int crtc_index = (uintptr_t)node->info_ent->data;
+ struct drm_crtc *crtc;
+ struct vc4_crtc *vc4_crtc;
+ int i;
+
+ i = 0;
+ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+ if (i == crtc_index)
+ break;
+ i++;
+ }
+ if (!crtc)
+ return 0;
+ vc4_crtc = to_vc4_crtc(crtc);
+
+ for (i = 0; i < ARRAY_SIZE(crtc_regs); i++) {
+ seq_printf(m, "%s (0x%04x): 0x%08x\n",
+ crtc_regs[i].name, crtc_regs[i].reg,
+ CRTC_READ(crtc_regs[i].reg));
+ }
+
+ return 0;
+}
+#endif
+
+static void vc4_crtc_destroy(struct drm_crtc *crtc)
+{
+ drm_crtc_cleanup(crtc);
+}
+
+static u32 vc4_get_fifo_full_level(u32 format)
+{
+ static const u32 fifo_len_bytes = 64;
+ static const u32 hvs_latency_pix = 6;
+
+ switch (format) {
+ case PV_CONTROL_FORMAT_DSIV_16:
+ case PV_CONTROL_FORMAT_DSIC_16:
+ return fifo_len_bytes - 2 * hvs_latency_pix;
+ case PV_CONTROL_FORMAT_DSIV_18:
+ return fifo_len_bytes - 14;
+ case PV_CONTROL_FORMAT_24:
+ case PV_CONTROL_FORMAT_DSIV_24:
+ default:
+ return fifo_len_bytes - 3 * hvs_latency_pix;
+ }
+}
+
+/*
+ * Returns the clock select bit for the connector attached to the
+ * CRTC.
+ */
+static int vc4_get_clock_select(struct drm_crtc *crtc)
+{
+ struct drm_connector *connector;
+
+ drm_for_each_connector(connector, crtc->dev) {
+ if (connector && connector->state->crtc == crtc) {
+ struct drm_encoder *encoder = connector->encoder;
+ struct vc4_encoder *vc4_encoder =
+ to_vc4_encoder(encoder);
+
+ return vc4_encoder->clock_select;
+ }
+ }
+
+ return -1;
+}
+
+static void vc4_crtc_mode_set_nofb(struct drm_crtc *crtc)
+{
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ struct drm_crtc_state *state = crtc->state;
+ struct drm_display_mode *mode = &state->adjusted_mode;
+ bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE;
+ u32 vactive = (mode->vdisplay >> (interlace ? 1 : 0));
+ u32 format = PV_CONTROL_FORMAT_24;
+ bool debug_dump_regs = false;
+ int clock_select = vc4_get_clock_select(crtc);
+
+ if (debug_dump_regs) {
+ DRM_INFO("CRTC %d regs before:\n", drm_crtc_index(crtc));
+ vc4_crtc_dump_regs(vc4_crtc);
+ }
+
+ /* Reset the PV fifo. */
+ CRTC_WRITE(PV_CONTROL, 0);
+ CRTC_WRITE(PV_CONTROL, PV_CONTROL_FIFO_CLR | PV_CONTROL_EN);
+ CRTC_WRITE(PV_CONTROL, 0);
+
+ CRTC_WRITE(PV_HORZA,
+ VC4_SET_FIELD(mode->htotal - mode->hsync_end,
+ PV_HORZA_HBP) |
+ VC4_SET_FIELD(mode->hsync_end - mode->hsync_start,
+ PV_HORZA_HSYNC));
+ CRTC_WRITE(PV_HORZB,
+ VC4_SET_FIELD(mode->hsync_start - mode->hdisplay,
+ PV_HORZB_HFP) |
+ VC4_SET_FIELD(mode->hdisplay, PV_HORZB_HACTIVE));
+
+ if (interlace) {
+ CRTC_WRITE(PV_VERTA_EVEN,
+ VC4_SET_FIELD(mode->vtotal - mode->vsync_end - 1,
+ PV_VERTA_VBP) |
+ VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+ PV_VERTA_VSYNC));
+ CRTC_WRITE(PV_VERTB_EVEN,
+ VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+ PV_VERTB_VFP) |
+ VC4_SET_FIELD(vactive, PV_VERTB_VACTIVE));
+ }
+
+ CRTC_WRITE(PV_HACT_ACT, mode->hdisplay);
+
+ CRTC_WRITE(PV_V_CONTROL,
+ PV_VCONTROL_CONTINUOUS |
+ (interlace ? PV_VCONTROL_INTERLACE : 0));
+
+ CRTC_WRITE(PV_CONTROL,
+ VC4_SET_FIELD(format, PV_CONTROL_FORMAT) |
+ VC4_SET_FIELD(vc4_get_fifo_full_level(format),
+ PV_CONTROL_FIFO_LEVEL) |
+ PV_CONTROL_CLR_AT_START |
+ PV_CONTROL_TRIGGER_UNDERFLOW |
+ PV_CONTROL_WAIT_HSTART |
+ VC4_SET_FIELD(clock_select, PV_CONTROL_CLK_SELECT) |
+ PV_CONTROL_FIFO_CLR |
+ PV_CONTROL_EN);
+
+ if (debug_dump_regs) {
+ DRM_INFO("CRTC %d regs after:\n", drm_crtc_index(crtc));
+ vc4_crtc_dump_regs(vc4_crtc);
+ }
+}
+
+static void require_hvs_enabled(struct drm_device *dev)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+ WARN_ON_ONCE((HVS_READ(SCALER_DISPCTRL) & SCALER_DISPCTRL_ENABLE) !=
+ SCALER_DISPCTRL_ENABLE);
+}
+
+static void vc4_crtc_disable(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ u32 chan = vc4_crtc->channel;
+ int ret;
+ require_hvs_enabled(dev);
+
+ CRTC_WRITE(PV_V_CONTROL,
+ CRTC_READ(PV_V_CONTROL) & ~PV_VCONTROL_VIDEN);
+ ret = wait_for(!(CRTC_READ(PV_V_CONTROL) & PV_VCONTROL_VIDEN), 1);
+ WARN_ONCE(ret, "Timeout waiting for !PV_VCONTROL_VIDEN\n");
+
+ if (HVS_READ(SCALER_DISPCTRLX(chan)) &
+ SCALER_DISPCTRLX_ENABLE) {
+ HVS_WRITE(SCALER_DISPCTRLX(chan),
+ SCALER_DISPCTRLX_RESET);
+
+ /* While the docs say that reset is self-clearing, it
+ * seems it doesn't actually.
+ */
+ HVS_WRITE(SCALER_DISPCTRLX(chan), 0);
+ }
+
+ /* Once we leave, the scaler should be disabled and its fifo empty. */
+
+ WARN_ON_ONCE(HVS_READ(SCALER_DISPCTRLX(chan)) & SCALER_DISPCTRLX_RESET);
+
+ WARN_ON_ONCE(VC4_GET_FIELD(HVS_READ(SCALER_DISPSTATX(chan)),
+ SCALER_DISPSTATX_MODE) !=
+ SCALER_DISPSTATX_MODE_DISABLED);
+
+ WARN_ON_ONCE((HVS_READ(SCALER_DISPSTATX(chan)) &
+ (SCALER_DISPSTATX_FULL | SCALER_DISPSTATX_EMPTY)) !=
+ SCALER_DISPSTATX_EMPTY);
+}
+
+static void vc4_crtc_enable(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ struct drm_crtc_state *state = crtc->state;
+ struct drm_display_mode *mode = &state->adjusted_mode;
+
+ require_hvs_enabled(dev);
+
+ /* Turn on the scaler, which will wait for vstart to start
+ * compositing.
+ */
+ HVS_WRITE(SCALER_DISPCTRLX(vc4_crtc->channel),
+ VC4_SET_FIELD(mode->hdisplay, SCALER_DISPCTRLX_WIDTH) |
+ VC4_SET_FIELD(mode->vdisplay, SCALER_DISPCTRLX_HEIGHT) |
+ SCALER_DISPCTRLX_ENABLE);
+
+ /* Turn on the pixel valve, which will emit the vstart signal. */
+ CRTC_WRITE(PV_V_CONTROL,
+ CRTC_READ(PV_V_CONTROL) | PV_VCONTROL_VIDEN);
+}
+
+static int vc4_crtc_atomic_check(struct drm_crtc *crtc,
+ struct drm_crtc_state *state)
+{
+ struct drm_device *dev = crtc->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct drm_plane *plane;
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ u32 dlist_count = 0;
+
+ /* The pixelvalve can only feed one encoder (and encoders are
+ * 1:1 with connectors.)
+ */
+ if (drm_atomic_connectors_for_crtc(state->state, crtc) > 1)
+ return -EINVAL;
+
+ drm_atomic_crtc_state_for_each_plane(plane, state) {
+ struct drm_plane_state *plane_state =
+ state->state->plane_states[drm_plane_index(plane)];
+
+ /* plane might not have changed, in which case take
+ * current state:
+ */
+ if (!plane_state)
+ plane_state = plane->state;
+
+ dlist_count += vc4_plane_dlist_size(plane_state);
+ }
+
+ dlist_count++; /* Account for SCALER_CTL0_END. */
+
+ if (!vc4_crtc->dlist || dlist_count > vc4_crtc->dlist_size) {
+ vc4_crtc->dlist = ((u32 __iomem *)vc4->hvs->dlist +
+ HVS_BOOTLOADER_DLIST_END);
+ vc4_crtc->dlist_size = ((SCALER_DLIST_SIZE >> 2) -
+ HVS_BOOTLOADER_DLIST_END);
+
+ if (dlist_count > vc4_crtc->dlist_size) {
+ DRM_DEBUG_KMS("dlist too large for CRTC (%d > %d).\n",
+ dlist_count, vc4_crtc->dlist_size);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static void vc4_crtc_atomic_flush(struct drm_crtc *crtc,
+ struct drm_crtc_state *old_state)
+{
+ struct drm_device *dev = crtc->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ struct drm_plane *plane;
+ bool debug_dump_regs = false;
+ u32 __iomem *dlist_next = vc4_crtc->dlist;
+
+ if (debug_dump_regs) {
+ DRM_INFO("CRTC %d HVS before:\n", drm_crtc_index(crtc));
+ vc4_hvs_dump_state(dev);
+ }
+
+ /* Copy all the active planes' dlist contents to the hardware dlist.
+ *
+ * XXX: If the new display list was large enough that it
+ * overlapped a currently-read display list, we need to do
+ * something like disable scanout before putting in the new
+ * list. For now, we're safe because we only have the two
+ * planes.
+ */
+ drm_atomic_crtc_for_each_plane(plane, crtc) {
+ dlist_next += vc4_plane_write_dlist(plane, dlist_next);
+ }
+
+ if (dlist_next == vc4_crtc->dlist) {
+ /* If no planes were enabled, use the SCALER_CTL0_END
+ * at the start of the display list memory (in the
+ * bootloader section). We'll rewrite that
+ * SCALER_CTL0_END, just in case, though.
+ */
+ writel(SCALER_CTL0_END, vc4->hvs->dlist);
+ HVS_WRITE(SCALER_DISPLISTX(vc4_crtc->channel), 0);
+ } else {
+ writel(SCALER_CTL0_END, dlist_next);
+ dlist_next++;
+
+ HVS_WRITE(SCALER_DISPLISTX(vc4_crtc->channel),
+ (u32 *)vc4_crtc->dlist - (u32 *)vc4->hvs->dlist);
+
+ /* Make the next display list start after ours. */
+ vc4_crtc->dlist_size -= (dlist_next - vc4_crtc->dlist);
+ vc4_crtc->dlist = dlist_next;
+ }
+
+ if (debug_dump_regs) {
+ DRM_INFO("CRTC %d HVS after:\n", drm_crtc_index(crtc));
+ vc4_hvs_dump_state(dev);
+ }
+
+ if (crtc->state->event) {
+ unsigned long flags;
+
+ crtc->state->event->pipe = drm_crtc_index(crtc);
+
+ WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+ spin_lock_irqsave(&dev->event_lock, flags);
+ vc4_crtc->event = crtc->state->event;
+ spin_unlock_irqrestore(&dev->event_lock, flags);
+ crtc->state->event = NULL;
+ }
+}
+
+int vc4_enable_vblank(struct drm_device *dev, unsigned int crtc_id)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct vc4_crtc *vc4_crtc = vc4->crtc[crtc_id];
+
+ CRTC_WRITE(PV_INTEN, PV_INT_VFP_START);
+
+ return 0;
+}
+
+void vc4_disable_vblank(struct drm_device *dev, unsigned int crtc_id)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct vc4_crtc *vc4_crtc = vc4->crtc[crtc_id];
+
+ CRTC_WRITE(PV_INTEN, 0);
+}
+
+static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
+{
+ struct drm_crtc *crtc = &vc4_crtc->base;
+ struct drm_device *dev = crtc->dev;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->event_lock, flags);
+ if (vc4_crtc->event) {
+ drm_crtc_send_vblank_event(crtc, vc4_crtc->event);
+ vc4_crtc->event = NULL;
+ }
+ spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+static irqreturn_t vc4_crtc_irq_handler(int irq, void *data)
+{
+ struct vc4_crtc *vc4_crtc = data;
+ u32 stat = CRTC_READ(PV_INTSTAT);
+ irqreturn_t ret = IRQ_NONE;
+
+ if (stat & PV_INT_VFP_START) {
+ CRTC_WRITE(PV_INTSTAT, PV_INT_VFP_START);
+ drm_crtc_handle_vblank(&vc4_crtc->base);
+ vc4_crtc_handle_page_flip(vc4_crtc);
+ ret = IRQ_HANDLED;
+ }
+
+ return ret;
+}
+
+static const struct drm_crtc_funcs vc4_crtc_funcs = {
+ .set_config = drm_atomic_helper_set_config,
+ .destroy = vc4_crtc_destroy,
+ .page_flip = drm_atomic_helper_page_flip,
+ .set_property = NULL,
+ .cursor_set = NULL, /* handled by drm_mode_cursor_universal */
+ .cursor_move = NULL, /* handled by drm_mode_cursor_universal */
+ .reset = drm_atomic_helper_crtc_reset,
+ .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+ .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
+ .mode_set_nofb = vc4_crtc_mode_set_nofb,
+ .disable = vc4_crtc_disable,
+ .enable = vc4_crtc_enable,
+ .atomic_check = vc4_crtc_atomic_check,
+ .atomic_flush = vc4_crtc_atomic_flush,
+};
+
+/* Frees the page flip event when the DRM device is closed with the
+ * event still outstanding.
+ */
+void vc4_cancel_page_flip(struct drm_crtc *crtc, struct drm_file *file)
+{
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ struct drm_device *dev = crtc->dev;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->event_lock, flags);
+
+ if (vc4_crtc->event && vc4_crtc->event->base.file_priv == file) {
+ vc4_crtc->event->base.destroy(&vc4_crtc->event->base);
+ drm_crtc_vblank_put(crtc);
+ vc4_crtc->event = NULL;
+ }
+
+ spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+static const struct vc4_crtc_data pv0_data = {
+ .hvs_channel = 0,
+ .encoder0_type = VC4_ENCODER_TYPE_DSI0,
+ .encoder1_type = VC4_ENCODER_TYPE_DPI,
+};
+
+static const struct vc4_crtc_data pv1_data = {
+ .hvs_channel = 2,
+ .encoder0_type = VC4_ENCODER_TYPE_DSI1,
+ .encoder1_type = VC4_ENCODER_TYPE_SMI,
+};
+
+static const struct vc4_crtc_data pv2_data = {
+ .hvs_channel = 1,
+ .encoder0_type = VC4_ENCODER_TYPE_VEC,
+ .encoder1_type = VC4_ENCODER_TYPE_HDMI,
+};
+
+static const struct of_device_id vc4_crtc_dt_match[] = {
+ { .compatible = "brcm,bcm2835-pixelvalve0", .data = &pv0_data },
+ { .compatible = "brcm,bcm2835-pixelvalve1", .data = &pv1_data },
+ { .compatible = "brcm,bcm2835-pixelvalve2", .data = &pv2_data },
+ {}
+};
+
+static void vc4_set_crtc_possible_masks(struct drm_device *drm,
+ struct drm_crtc *crtc)
+{
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ struct drm_encoder *encoder;
+
+ drm_for_each_encoder(encoder, drm) {
+ struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
+
+ if (vc4_encoder->type == vc4_crtc->data->encoder0_type) {
+ vc4_encoder->clock_select = 0;
+ encoder->possible_crtcs |= drm_crtc_mask(crtc);
+ } else if (vc4_encoder->type == vc4_crtc->data->encoder1_type) {
+ vc4_encoder->clock_select = 1;
+ encoder->possible_crtcs |= drm_crtc_mask(crtc);
+ }
+ }
+}
+
+static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct drm_device *drm = dev_get_drvdata(master);
+ struct vc4_dev *vc4 = to_vc4_dev(drm);
+ struct vc4_crtc *vc4_crtc;
+ struct drm_crtc *crtc;
+ struct drm_plane *primary_plane, *cursor_plane;
+ const struct of_device_id *match;
+ int ret;
+
+ vc4_crtc = devm_kzalloc(dev, sizeof(*vc4_crtc), GFP_KERNEL);
+ if (!vc4_crtc)
+ return -ENOMEM;
+ crtc = &vc4_crtc->base;
+
+ match = of_match_device(vc4_crtc_dt_match, dev);
+ if (!match)
+ return -ENODEV;
+ vc4_crtc->data = match->data;
+
+ vc4_crtc->regs = vc4_ioremap_regs(pdev, 0);
+ if (IS_ERR(vc4_crtc->regs))
+ return PTR_ERR(vc4_crtc->regs);
+
+ /* For now, we create just the primary and the legacy cursor
+ * planes. We should be able to stack more planes on easily,
+ * but to do that we would need to compute the bandwidth
+ * requirement of the plane configuration, and reject ones
+ * that will take too much.
+ */
+ primary_plane = vc4_plane_init(drm, DRM_PLANE_TYPE_PRIMARY);
+ if (!primary_plane) {
+ dev_err(dev, "failed to construct primary plane\n");
+ ret = PTR_ERR(primary_plane);
+ goto err;
+ }
+
+ cursor_plane = vc4_plane_init(drm, DRM_PLANE_TYPE_CURSOR);
+ if (!cursor_plane) {
+ dev_err(dev, "failed to construct cursor plane\n");
+ ret = PTR_ERR(cursor_plane);
+ goto err_primary;
+ }
+
+ drm_crtc_init_with_planes(drm, crtc, primary_plane, cursor_plane,
+ &vc4_crtc_funcs);
+ drm_crtc_helper_add(crtc, &vc4_crtc_helper_funcs);
+ primary_plane->crtc = crtc;
+ cursor_plane->crtc = crtc;
+ vc4->crtc[drm_crtc_index(crtc)] = vc4_crtc;
+ vc4_crtc->channel = vc4_crtc->data->hvs_channel;
+
+ CRTC_WRITE(PV_INTEN, 0);
+ CRTC_WRITE(PV_INTSTAT, PV_INT_VFP_START);
+ ret = devm_request_irq(dev, platform_get_irq(pdev, 0),
+ vc4_crtc_irq_handler, 0, "vc4 crtc", vc4_crtc);
+ if (ret)
+ goto err_cursor;
+
+ vc4_set_crtc_possible_masks(drm, crtc);
+
+ platform_set_drvdata(pdev, vc4_crtc);
+
+ return 0;
+
+err_cursor:
+ cursor_plane->funcs->destroy(cursor_plane);
+err_primary:
+ primary_plane->funcs->destroy(primary_plane);
+err:
+ return ret;
+}
+
+static void vc4_crtc_unbind(struct device *dev, struct device *master,
+ void *data)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct vc4_crtc *vc4_crtc = dev_get_drvdata(dev);
+
+ vc4_crtc_destroy(&vc4_crtc->base);
+
+ CRTC_WRITE(PV_INTEN, 0);
+
+ platform_set_drvdata(pdev, NULL);
+}
+
+static const struct component_ops vc4_crtc_ops = {
+ .bind = vc4_crtc_bind,
+ .unbind = vc4_crtc_unbind,
+};
+
+static int vc4_crtc_dev_probe(struct platform_device *pdev)
+{
+ return component_add(&pdev->dev, &vc4_crtc_ops);
+}
+
+static int vc4_crtc_dev_remove(struct platform_device *pdev)
+{
+ component_del(&pdev->dev, &vc4_crtc_ops);
+ return 0;
+}
+
+struct platform_driver vc4_crtc_driver = {
+ .probe = vc4_crtc_dev_probe,
+ .remove = vc4_crtc_dev_remove,
+ .driver = {
+ .name = "vc4_crtc",
+ .of_match_table = vc4_crtc_dt_match,
+ },
+};
--- /dev/null
+/*
+ * Copyright © 2014 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/seq_file.h>
+#include <linux/circ_buf.h>
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
+#include <drm/drmP.h>
+
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+static const struct drm_info_list vc4_debugfs_list[] = {
+ {"hdmi_regs", vc4_hdmi_debugfs_regs, 0},
+ {"hvs_regs", vc4_hvs_debugfs_regs, 0},
+ {"crtc0_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)0},
+ {"crtc1_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)1},
+ {"crtc2_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)2},
+};
+
+#define VC4_DEBUGFS_ENTRIES ARRAY_SIZE(vc4_debugfs_list)
+
+int
+vc4_debugfs_init(struct drm_minor *minor)
+{
+ return drm_debugfs_create_files(vc4_debugfs_list, VC4_DEBUGFS_ENTRIES,
+ minor->debugfs_root, minor);
+}
+
+void
+vc4_debugfs_cleanup(struct drm_minor *minor)
+{
+ drm_debugfs_remove_files(vc4_debugfs_list, VC4_DEBUGFS_ENTRIES, minor);
+}
--- /dev/null
+/*
+ * Copyright (C) 2014-2015 Broadcom
+ * Copyright (C) 2013 Red Hat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include "drm_fb_cma_helper.h"
+
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+#define DRIVER_NAME "vc4"
+#define DRIVER_DESC "Broadcom VC4 graphics"
+#define DRIVER_DATE "20140616"
+#define DRIVER_MAJOR 0
+#define DRIVER_MINOR 0
+#define DRIVER_PATCHLEVEL 0
+
+/* Helper function for mapping the regs on a platform device. */
+void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index)
+{
+ struct resource *res;
+ void __iomem *map;
+
+ res = platform_get_resource(dev, IORESOURCE_MEM, index);
+ map = devm_ioremap_resource(&dev->dev, res);
+ if (IS_ERR(map)) {
+ DRM_ERROR("Failed to map registers: %ld\n", PTR_ERR(map));
+ return map;
+ }
+
+ return map;
+}
+
+static void vc4_drm_preclose(struct drm_device *dev, struct drm_file *file)
+{
+ struct drm_crtc *crtc;
+
+ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+ vc4_cancel_page_flip(crtc, file);
+}
+
+static void vc4_lastclose(struct drm_device *dev)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+ if (vc4->fbdev)
+ drm_fbdev_cma_restore_mode(vc4->fbdev);
+}
+
+static const struct file_operations vc4_drm_fops = {
+ .owner = THIS_MODULE,
+ .open = drm_open,
+ .release = drm_release,
+ .unlocked_ioctl = drm_ioctl,
+ .mmap = drm_gem_cma_mmap,
+ .poll = drm_poll,
+ .read = drm_read,
+#ifdef CONFIG_COMPAT
+ .compat_ioctl = drm_compat_ioctl,
+#endif
+ .llseek = noop_llseek,
+};
+
+static const struct drm_ioctl_desc vc4_drm_ioctls[] = {
+};
+
+static struct drm_driver vc4_drm_driver = {
+ .driver_features = (DRIVER_MODESET |
+ DRIVER_ATOMIC |
+ DRIVER_GEM |
+ DRIVER_PRIME),
+ .lastclose = vc4_lastclose,
+ .preclose = vc4_drm_preclose,
+
+ .enable_vblank = vc4_enable_vblank,
+ .disable_vblank = vc4_disable_vblank,
+ .get_vblank_counter = drm_vblank_count,
+
+#if defined(CONFIG_DEBUG_FS)
+ .debugfs_init = vc4_debugfs_init,
+ .debugfs_cleanup = vc4_debugfs_cleanup,
+#endif
+
+ .gem_free_object = drm_gem_cma_free_object,
+ .gem_vm_ops = &drm_gem_cma_vm_ops,
+
+ .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+ .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+ .gem_prime_import = drm_gem_prime_import,
+ .gem_prime_export = drm_gem_prime_export,
+ .gem_prime_get_sg_table = drm_gem_cma_prime_get_sg_table,
+ .gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+ .gem_prime_vmap = drm_gem_cma_prime_vmap,
+ .gem_prime_vunmap = drm_gem_cma_prime_vunmap,
+ .gem_prime_mmap = drm_gem_cma_prime_mmap,
+
+ .dumb_create = vc4_dumb_create,
+ .dumb_map_offset = drm_gem_cma_dumb_map_offset,
+ .dumb_destroy = drm_gem_dumb_destroy,
+
+ .ioctls = vc4_drm_ioctls,
+ .num_ioctls = ARRAY_SIZE(vc4_drm_ioctls),
+ .fops = &vc4_drm_fops,
+
+ .name = DRIVER_NAME,
+ .desc = DRIVER_DESC,
+ .date = DRIVER_DATE,
+ .major = DRIVER_MAJOR,
+ .minor = DRIVER_MINOR,
+ .patchlevel = DRIVER_PATCHLEVEL,
+};
+
+static int compare_dev(struct device *dev, void *data)
+{
+ return dev == data;
+}
+
+static void vc4_match_add_drivers(struct device *dev,
+ struct component_match **match,
+ struct platform_driver *const *drivers,
+ int count)
+{
+ int i;
+
+ for (i = 0; i < count; i++) {
+ struct device_driver *drv = &drivers[i]->driver;
+ struct device *p = NULL, *d;
+
+ while ((d = bus_find_device(&platform_bus_type, p, drv,
+ (void *)platform_bus_type.match))) {
+ put_device(p);
+ component_match_add(dev, match, compare_dev, d);
+ p = d;
+ }
+ put_device(p);
+ }
+}
+
+static int vc4_drm_bind(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct drm_device *drm;
+ struct drm_connector *connector;
+ struct vc4_dev *vc4;
+ int ret = 0;
+
+ dev->coherent_dma_mask = DMA_BIT_MASK(32);
+
+ vc4 = devm_kzalloc(dev, sizeof(*vc4), GFP_KERNEL);
+ if (!vc4)
+ return -ENOMEM;
+
+ drm = drm_dev_alloc(&vc4_drm_driver, dev);
+ if (!drm)
+ return -ENOMEM;
+ platform_set_drvdata(pdev, drm);
+ vc4->dev = drm;
+ drm->dev_private = vc4;
+
+ drm_dev_set_unique(drm, dev_name(dev));
+
+ drm_mode_config_init(drm);
+ if (ret)
+ goto unref;
+
+ ret = component_bind_all(dev, drm);
+ if (ret)
+ goto unref;
+
+ ret = drm_dev_register(drm, 0);
+ if (ret < 0)
+ goto unbind_all;
+
+ /* Connector registration has to occur after DRM device
+ * registration, because it creates sysfs entries based on the
+ * DRM device.
+ */
+ list_for_each_entry(connector, &drm->mode_config.connector_list, head) {
+ ret = drm_connector_register(connector);
+ if (ret)
+ goto unregister;
+ }
+
+ vc4_kms_load(drm);
+
+ return 0;
+
+unregister:
+ drm_dev_unregister(drm);
+unbind_all:
+ component_unbind_all(dev, drm);
+unref:
+ drm_dev_unref(drm);
+ return ret;
+}
+
+static void vc4_drm_unbind(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct drm_device *drm = platform_get_drvdata(pdev);
+ struct vc4_dev *vc4 = to_vc4_dev(drm);
+
+ if (vc4->fbdev)
+ drm_fbdev_cma_fini(vc4->fbdev);
+
+ drm_mode_config_cleanup(drm);
+
+ drm_put_dev(drm);
+}
+
+static const struct component_master_ops vc4_drm_ops = {
+ .bind = vc4_drm_bind,
+ .unbind = vc4_drm_unbind,
+};
+
+static struct platform_driver *const component_drivers[] = {
+ &vc4_hdmi_driver,
+ &vc4_crtc_driver,
+ &vc4_hvs_driver,
+};
+
+static int vc4_platform_drm_probe(struct platform_device *pdev)
+{
+ struct component_match *match = NULL;
+ struct device *dev = &pdev->dev;
+
+ vc4_match_add_drivers(dev, &match,
+ component_drivers, ARRAY_SIZE(component_drivers));
+
+ return component_master_add_with_match(dev, &vc4_drm_ops, match);
+}
+
+static int vc4_platform_drm_remove(struct platform_device *pdev)
+{
+ component_master_del(&pdev->dev, &vc4_drm_ops);
+
+ return 0;
+}
+
+static const struct of_device_id vc4_of_match[] = {
+ { .compatible = "brcm,bcm2835-vc4", },
+ {},
+};
+MODULE_DEVICE_TABLE(of, vc4_of_match);
+
+static struct platform_driver vc4_platform_driver = {
+ .probe = vc4_platform_drm_probe,
+ .remove = vc4_platform_drm_remove,
+ .driver = {
+ .name = "vc4-drm",
+ .owner = THIS_MODULE,
+ .of_match_table = vc4_of_match,
+ },
+};
+
+static int __init vc4_drm_register(void)
+{
+ int i, ret;
+
+ for (i = 0; i < ARRAY_SIZE(component_drivers); i++) {
+ ret = platform_driver_register(component_drivers[i]);
+ if (ret) {
+ while (--i >= 0)
+ platform_driver_unregister(component_drivers[i]);
+ return ret;
+ }
+ }
+ return platform_driver_register(&vc4_platform_driver);
+}
+
+static void __exit vc4_drm_unregister(void)
+{
+ int i;
+
+ for (i = ARRAY_SIZE(component_drivers) - 1; i >= 0; i--)
+ platform_driver_unregister(component_drivers[i]);
+
+ platform_driver_unregister(&vc4_platform_driver);
+}
+
+module_init(vc4_drm_register);
+module_exit(vc4_drm_unregister);
+
+MODULE_ALIAS("platform:vc4-drm");
+MODULE_DESCRIPTION("Broadcom VC4 DRM Driver");
+MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
+MODULE_LICENSE("GPL v2");
--- /dev/null
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include "drmP.h"
+#include "drm_gem_cma_helper.h"
+
+struct vc4_dev {
+ struct drm_device *dev;
+
+ struct vc4_hdmi *hdmi;
+ struct vc4_hvs *hvs;
+ struct vc4_crtc *crtc[3];
+
+ struct drm_fbdev_cma *fbdev;
+};
+
+static inline struct vc4_dev *
+to_vc4_dev(struct drm_device *dev)
+{
+ return (struct vc4_dev *)dev->dev_private;
+}
+
+struct vc4_bo {
+ struct drm_gem_cma_object base;
+};
+
+static inline struct vc4_bo *
+to_vc4_bo(struct drm_gem_object *bo)
+{
+ return (struct vc4_bo *)bo;
+}
+
+struct vc4_hvs {
+ struct platform_device *pdev;
+ void __iomem *regs;
+ void __iomem *dlist;
+};
+
+struct vc4_plane {
+ struct drm_plane base;
+};
+
+static inline struct vc4_plane *
+to_vc4_plane(struct drm_plane *plane)
+{
+ return (struct vc4_plane *)plane;
+}
+
+enum vc4_encoder_type {
+ VC4_ENCODER_TYPE_HDMI,
+ VC4_ENCODER_TYPE_VEC,
+ VC4_ENCODER_TYPE_DSI0,
+ VC4_ENCODER_TYPE_DSI1,
+ VC4_ENCODER_TYPE_SMI,
+ VC4_ENCODER_TYPE_DPI,
+};
+
+struct vc4_encoder {
+ struct drm_encoder base;
+ enum vc4_encoder_type type;
+ u32 clock_select;
+};
+
+static inline struct vc4_encoder *
+to_vc4_encoder(struct drm_encoder *encoder)
+{
+ return container_of(encoder, struct vc4_encoder, base);
+}
+
+#define HVS_READ(offset) readl(vc4->hvs->regs + offset)
+#define HVS_WRITE(offset, val) writel(val, vc4->hvs->regs + offset)
+
+/**
+ * _wait_for - magic (register) wait macro
+ *
+ * Does the right thing for modeset paths when run under kdgb or similar atomic
+ * contexts. Note that it's important that we check the condition again after
+ * having timed out, since the timeout could be due to preemption or similar and
+ * we've never had a chance to check the condition before the timeout.
+ */
+#define _wait_for(COND, MS, W) ({ \
+ unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \
+ int ret__ = 0; \
+ while (!(COND)) { \
+ if (time_after(jiffies, timeout__)) { \
+ if (!(COND)) \
+ ret__ = -ETIMEDOUT; \
+ break; \
+ } \
+ if (W && drm_can_sleep()) { \
+ msleep(W); \
+ } else { \
+ cpu_relax(); \
+ } \
+ } \
+ ret__; \
+})
+
+#define wait_for(COND, MS) _wait_for(COND, MS, 1)
+
+/* vc4_bo.c */
+void vc4_free_object(struct drm_gem_object *gem_obj);
+struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size);
+int vc4_dumb_create(struct drm_file *file_priv,
+ struct drm_device *dev,
+ struct drm_mode_create_dumb *args);
+struct dma_buf *vc4_prime_export(struct drm_device *dev,
+ struct drm_gem_object *obj, int flags);
+
+/* vc4_crtc.c */
+extern struct platform_driver vc4_crtc_driver;
+int vc4_enable_vblank(struct drm_device *dev, unsigned int crtc_id);
+void vc4_disable_vblank(struct drm_device *dev, unsigned int crtc_id);
+void vc4_cancel_page_flip(struct drm_crtc *crtc, struct drm_file *file);
+int vc4_crtc_debugfs_regs(struct seq_file *m, void *arg);
+
+/* vc4_debugfs.c */
+int vc4_debugfs_init(struct drm_minor *minor);
+void vc4_debugfs_cleanup(struct drm_minor *minor);
+
+/* vc4_drv.c */
+void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index);
+
+/* vc4_hdmi.c */
+extern struct platform_driver vc4_hdmi_driver;
+int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused);
+
+/* vc4_hvs.c */
+extern struct platform_driver vc4_hvs_driver;
+void vc4_hvs_dump_state(struct drm_device *dev);
+int vc4_hvs_debugfs_regs(struct seq_file *m, void *unused);
+
+/* vc4_kms.c */
+int vc4_kms_load(struct drm_device *dev);
+
+/* vc4_plane.c */
+struct drm_plane *vc4_plane_init(struct drm_device *dev,
+ enum drm_plane_type type);
+u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist);
+u32 vc4_plane_dlist_size(struct drm_plane_state *state);
--- /dev/null
+/*
+ * Copyright (C) 2015 Broadcom
+ * Copyright (c) 2014 The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * DOC: VC4 Falcon HDMI module
+ *
+ * The HDMI core has a state machine and a PHY. Most of the unit
+ * operates off of the HSM clock from CPRMAN. It also internally uses
+ * the PLLH_PIX clock for the PHY.
+ */
+
+#include "drm_atomic_helper.h"
+#include "drm_crtc_helper.h"
+#include "drm_edid.h"
+#include "linux/clk.h"
+#include "linux/component.h"
+#include "linux/i2c.h"
+#include "linux/of_gpio.h"
+#include "linux/of_platform.h"
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+/* General HDMI hardware state. */
+struct vc4_hdmi {
+ struct platform_device *pdev;
+
+ struct drm_encoder *encoder;
+ struct drm_connector *connector;
+
+ struct i2c_adapter *ddc;
+ void __iomem *hdmicore_regs;
+ void __iomem *hd_regs;
+ int hpd_gpio;
+
+ struct clk *pixel_clock;
+ struct clk *hsm_clock;
+};
+
+#define HDMI_READ(offset) readl(vc4->hdmi->hdmicore_regs + offset)
+#define HDMI_WRITE(offset, val) writel(val, vc4->hdmi->hdmicore_regs + offset)
+#define HD_READ(offset) readl(vc4->hdmi->hd_regs + offset)
+#define HD_WRITE(offset, val) writel(val, vc4->hdmi->hd_regs + offset)
+
+/* VC4 HDMI encoder KMS struct */
+struct vc4_hdmi_encoder {
+ struct vc4_encoder base;
+ bool hdmi_monitor;
+};
+
+static inline struct vc4_hdmi_encoder *
+to_vc4_hdmi_encoder(struct drm_encoder *encoder)
+{
+ return container_of(encoder, struct vc4_hdmi_encoder, base.base);
+}
+
+/* VC4 HDMI connector KMS struct */
+struct vc4_hdmi_connector {
+ struct drm_connector base;
+
+ /* Since the connector is attached to just the one encoder,
+ * this is the reference to it so we can do the best_encoder()
+ * hook.
+ */
+ struct drm_encoder *encoder;
+};
+
+static inline struct vc4_hdmi_connector *
+to_vc4_hdmi_connector(struct drm_connector *connector)
+{
+ return container_of(connector, struct vc4_hdmi_connector, base);
+}
+
+#define HDMI_REG(reg) { reg, #reg }
+static const struct {
+ u32 reg;
+ const char *name;
+} hdmi_regs[] = {
+ HDMI_REG(VC4_HDMI_CORE_REV),
+ HDMI_REG(VC4_HDMI_SW_RESET_CONTROL),
+ HDMI_REG(VC4_HDMI_HOTPLUG_INT),
+ HDMI_REG(VC4_HDMI_HOTPLUG),
+ HDMI_REG(VC4_HDMI_HORZA),
+ HDMI_REG(VC4_HDMI_HORZB),
+ HDMI_REG(VC4_HDMI_FIFO_CTL),
+ HDMI_REG(VC4_HDMI_SCHEDULER_CONTROL),
+ HDMI_REG(VC4_HDMI_VERTA0),
+ HDMI_REG(VC4_HDMI_VERTA1),
+ HDMI_REG(VC4_HDMI_VERTB0),
+ HDMI_REG(VC4_HDMI_VERTB1),
+ HDMI_REG(VC4_HDMI_TX_PHY_RESET_CTL),
+};
+
+static const struct {
+ u32 reg;
+ const char *name;
+} hd_regs[] = {
+ HDMI_REG(VC4_HD_M_CTL),
+ HDMI_REG(VC4_HD_MAI_CTL),
+ HDMI_REG(VC4_HD_VID_CTL),
+ HDMI_REG(VC4_HD_CSC_CTL),
+ HDMI_REG(VC4_HD_FRAME_COUNT),
+};
+
+#ifdef CONFIG_DEBUG_FS
+int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused)
+{
+ struct drm_info_node *node = (struct drm_info_node *)m->private;
+ struct drm_device *dev = node->minor->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(hdmi_regs); i++) {
+ seq_printf(m, "%s (0x%04x): 0x%08x\n",
+ hdmi_regs[i].name, hdmi_regs[i].reg,
+ HDMI_READ(hdmi_regs[i].reg));
+ }
+
+ for (i = 0; i < ARRAY_SIZE(hd_regs); i++) {
+ seq_printf(m, "%s (0x%04x): 0x%08x\n",
+ hd_regs[i].name, hd_regs[i].reg,
+ HD_READ(hd_regs[i].reg));
+ }
+
+ return 0;
+}
+#endif /* CONFIG_DEBUG_FS */
+
+static void vc4_hdmi_dump_regs(struct drm_device *dev)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(hdmi_regs); i++) {
+ DRM_INFO("0x%04x (%s): 0x%08x\n",
+ hdmi_regs[i].reg, hdmi_regs[i].name,
+ HDMI_READ(hdmi_regs[i].reg));
+ }
+ for (i = 0; i < ARRAY_SIZE(hd_regs); i++) {
+ DRM_INFO("0x%04x (%s): 0x%08x\n",
+ hd_regs[i].reg, hd_regs[i].name,
+ HD_READ(hd_regs[i].reg));
+ }
+}
+
+static enum drm_connector_status
+vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
+{
+ struct drm_device *dev = connector->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+ if (vc4->hdmi->hpd_gpio) {
+ if (gpio_get_value(vc4->hdmi->hpd_gpio))
+ return connector_status_connected;
+ else
+ return connector_status_disconnected;
+ }
+
+ if (HDMI_READ(VC4_HDMI_HOTPLUG) & VC4_HDMI_HOTPLUG_CONNECTED)
+ return connector_status_connected;
+ else
+ return connector_status_disconnected;
+}
+
+static void vc4_hdmi_connector_destroy(struct drm_connector *connector)
+{
+ drm_connector_unregister(connector);
+ drm_connector_cleanup(connector);
+}
+
+static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
+{
+ struct vc4_hdmi_connector *vc4_connector =
+ to_vc4_hdmi_connector(connector);
+ struct drm_encoder *encoder = vc4_connector->encoder;
+ struct vc4_hdmi_encoder *vc4_encoder = to_vc4_hdmi_encoder(encoder);
+ struct drm_device *dev = connector->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ int ret = 0;
+ struct edid *edid;
+
+ edid = drm_get_edid(connector, vc4->hdmi->ddc);
+ if (!edid)
+ return -ENODEV;
+
+ vc4_encoder->hdmi_monitor = drm_detect_hdmi_monitor(edid);
+ drm_mode_connector_update_edid_property(connector, edid);
+ ret = drm_add_edid_modes(connector, edid);
+
+ return ret;
+}
+
+static struct drm_encoder *
+vc4_hdmi_connector_best_encoder(struct drm_connector *connector)
+{
+ struct vc4_hdmi_connector *hdmi_connector =
+ to_vc4_hdmi_connector(connector);
+ return hdmi_connector->encoder;
+}
+
+static const struct drm_connector_funcs vc4_hdmi_connector_funcs = {
+ .dpms = drm_atomic_helper_connector_dpms,
+ .detect = vc4_hdmi_connector_detect,
+ .fill_modes = drm_helper_probe_single_connector_modes,
+ .destroy = vc4_hdmi_connector_destroy,
+ .reset = drm_atomic_helper_connector_reset,
+ .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+ .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static const struct drm_connector_helper_funcs vc4_hdmi_connector_helper_funcs = {
+ .get_modes = vc4_hdmi_connector_get_modes,
+ .best_encoder = vc4_hdmi_connector_best_encoder,
+};
+
+static struct drm_connector *vc4_hdmi_connector_init(struct drm_device *dev,
+ struct drm_encoder *encoder)
+{
+ struct drm_connector *connector = NULL;
+ struct vc4_hdmi_connector *hdmi_connector;
+ int ret = 0;
+
+ hdmi_connector = devm_kzalloc(dev->dev, sizeof(*hdmi_connector),
+ GFP_KERNEL);
+ if (!hdmi_connector) {
+ ret = -ENOMEM;
+ goto fail;
+ }
+ connector = &hdmi_connector->base;
+
+ hdmi_connector->encoder = encoder;
+
+ drm_connector_init(dev, connector, &vc4_hdmi_connector_funcs,
+ DRM_MODE_CONNECTOR_HDMIA);
+ drm_connector_helper_add(connector, &vc4_hdmi_connector_helper_funcs);
+
+ connector->polled = (DRM_CONNECTOR_POLL_CONNECT |
+ DRM_CONNECTOR_POLL_DISCONNECT);
+
+ connector->interlace_allowed = 0;
+ connector->doublescan_allowed = 0;
+
+ drm_mode_connector_attach_encoder(connector, encoder);
+
+ return connector;
+
+ fail:
+ if (connector)
+ vc4_hdmi_connector_destroy(connector);
+
+ return ERR_PTR(ret);
+}
+
+static void vc4_hdmi_encoder_destroy(struct drm_encoder *encoder)
+{
+ drm_encoder_cleanup(encoder);
+}
+
+static const struct drm_encoder_funcs vc4_hdmi_encoder_funcs = {
+ .destroy = vc4_hdmi_encoder_destroy,
+};
+
+static void vc4_hdmi_encoder_mode_set(struct drm_encoder *encoder,
+ struct drm_display_mode *unadjusted_mode,
+ struct drm_display_mode *mode)
+{
+ struct drm_device *dev = encoder->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ bool debug_dump_regs = false;
+ bool hsync_pos = mode->flags & DRM_MODE_FLAG_PHSYNC;
+ bool vsync_pos = mode->flags & DRM_MODE_FLAG_PVSYNC;
+ u32 vactive = (mode->vdisplay >>
+ ((mode->flags & DRM_MODE_FLAG_INTERLACE) ? 1 : 0));
+ u32 verta = (VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+ VC4_HDMI_VERTA_VSP) |
+ VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+ VC4_HDMI_VERTA_VFP) |
+ VC4_SET_FIELD(vactive, VC4_HDMI_VERTA_VAL));
+ u32 vertb = (VC4_SET_FIELD(0, VC4_HDMI_VERTB_VSPO) |
+ VC4_SET_FIELD(mode->vtotal - mode->vsync_end,
+ VC4_HDMI_VERTB_VBP));
+
+ if (debug_dump_regs) {
+ DRM_INFO("HDMI regs before:\n");
+ vc4_hdmi_dump_regs(dev);
+ }
+
+ HD_WRITE(VC4_HD_VID_CTL, 0);
+
+ clk_set_rate(vc4->hdmi->pixel_clock, mode->clock * 1000);
+
+ HDMI_WRITE(VC4_HDMI_SCHEDULER_CONTROL,
+ HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) |
+ VC4_HDMI_SCHEDULER_CONTROL_MANUAL_FORMAT |
+ VC4_HDMI_SCHEDULER_CONTROL_IGNORE_VSYNC_PREDICTS);
+
+ HDMI_WRITE(VC4_HDMI_HORZA,
+ (vsync_pos ? VC4_HDMI_HORZA_VPOS : 0) |
+ (hsync_pos ? VC4_HDMI_HORZA_HPOS : 0) |
+ VC4_SET_FIELD(mode->hdisplay, VC4_HDMI_HORZA_HAP));
+
+ HDMI_WRITE(VC4_HDMI_HORZB,
+ VC4_SET_FIELD(mode->htotal - mode->hsync_end,
+ VC4_HDMI_HORZB_HBP) |
+ VC4_SET_FIELD(mode->hsync_end - mode->hsync_start,
+ VC4_HDMI_HORZB_HSP) |
+ VC4_SET_FIELD(mode->hsync_start - mode->hdisplay,
+ VC4_HDMI_HORZB_HFP));
+
+ HDMI_WRITE(VC4_HDMI_VERTA0, verta);
+ HDMI_WRITE(VC4_HDMI_VERTA1, verta);
+
+ HDMI_WRITE(VC4_HDMI_VERTB0, vertb);
+ HDMI_WRITE(VC4_HDMI_VERTB1, vertb);
+
+ HD_WRITE(VC4_HD_VID_CTL,
+ (vsync_pos ? 0 : VC4_HD_VID_CTL_VSYNC_LOW) |
+ (hsync_pos ? 0 : VC4_HD_VID_CTL_HSYNC_LOW));
+
+ /* The RGB order applies even when CSC is disabled. */
+ HD_WRITE(VC4_HD_CSC_CTL, VC4_SET_FIELD(VC4_HD_CSC_CTL_ORDER_BGR,
+ VC4_HD_CSC_CTL_ORDER));
+
+ HDMI_WRITE(VC4_HDMI_FIFO_CTL, VC4_HDMI_FIFO_CTL_MASTER_SLAVE_N);
+
+ if (debug_dump_regs) {
+ DRM_INFO("HDMI regs after:\n");
+ vc4_hdmi_dump_regs(dev);
+ }
+}
+
+static void vc4_hdmi_encoder_disable(struct drm_encoder *encoder)
+{
+ struct drm_device *dev = encoder->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+ HDMI_WRITE(VC4_HDMI_TX_PHY_RESET_CTL, 0xf << 16);
+ HD_WRITE(VC4_HD_VID_CTL,
+ HD_READ(VC4_HD_VID_CTL) & ~VC4_HD_VID_CTL_ENABLE);
+}
+
+static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
+{
+ struct vc4_hdmi_encoder *vc4_encoder = to_vc4_hdmi_encoder(encoder);
+ struct drm_device *dev = encoder->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ int ret;
+
+ HDMI_WRITE(VC4_HDMI_TX_PHY_RESET_CTL, 0);
+
+ HD_WRITE(VC4_HD_VID_CTL,
+ HD_READ(VC4_HD_VID_CTL) |
+ VC4_HD_VID_CTL_ENABLE |
+ VC4_HD_VID_CTL_UNDERFLOW_ENABLE |
+ VC4_HD_VID_CTL_FRAME_COUNTER_RESET);
+
+ if (vc4_encoder->hdmi_monitor) {
+ HDMI_WRITE(VC4_HDMI_SCHEDULER_CONTROL,
+ HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) |
+ VC4_HDMI_SCHEDULER_CONTROL_MODE_HDMI);
+
+ ret = wait_for(HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
+ VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE, 1);
+ WARN_ONCE(ret, "Timeout waiting for "
+ "VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE\n");
+ } else {
+ HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG,
+ HDMI_READ(VC4_HDMI_RAM_PACKET_CONFIG) &
+ ~(VC4_HDMI_RAM_PACKET_ENABLE));
+ HDMI_WRITE(VC4_HDMI_SCHEDULER_CONTROL,
+ HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
+ ~VC4_HDMI_SCHEDULER_CONTROL_MODE_HDMI);
+
+ ret = wait_for(!(HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
+ VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE), 1);
+ WARN_ONCE(ret, "Timeout waiting for "
+ "!VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE\n");
+ }
+
+ if (vc4_encoder->hdmi_monitor) {
+ u32 drift;
+
+ WARN_ON(!(HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
+ VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE));
+ HDMI_WRITE(VC4_HDMI_SCHEDULER_CONTROL,
+ HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) |
+ VC4_HDMI_SCHEDULER_CONTROL_VERT_ALWAYS_KEEPOUT);
+
+ /* XXX: Set HDMI_RAM_PACKET_CONFIG (1 << 16) and set
+ * up the infoframe.
+ */
+
+ drift = HDMI_READ(VC4_HDMI_FIFO_CTL);
+ drift &= VC4_HDMI_FIFO_VALID_WRITE_MASK;
+
+ HDMI_WRITE(VC4_HDMI_FIFO_CTL,
+ drift & ~VC4_HDMI_FIFO_CTL_RECENTER);
+ HDMI_WRITE(VC4_HDMI_FIFO_CTL,
+ drift | VC4_HDMI_FIFO_CTL_RECENTER);
+ udelay(1000);
+ HDMI_WRITE(VC4_HDMI_FIFO_CTL,
+ drift & ~VC4_HDMI_FIFO_CTL_RECENTER);
+ HDMI_WRITE(VC4_HDMI_FIFO_CTL,
+ drift | VC4_HDMI_FIFO_CTL_RECENTER);
+
+ ret = wait_for(HDMI_READ(VC4_HDMI_FIFO_CTL) &
+ VC4_HDMI_FIFO_CTL_RECENTER_DONE, 1);
+ WARN_ONCE(ret, "Timeout waiting for "
+ "VC4_HDMI_FIFO_CTL_RECENTER_DONE");
+ }
+}
+
+static const struct drm_encoder_helper_funcs vc4_hdmi_encoder_helper_funcs = {
+ .mode_set = vc4_hdmi_encoder_mode_set,
+ .disable = vc4_hdmi_encoder_disable,
+ .enable = vc4_hdmi_encoder_enable,
+};
+
+static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct drm_device *drm = dev_get_drvdata(master);
+ struct vc4_dev *vc4 = drm->dev_private;
+ struct vc4_hdmi *hdmi;
+ struct vc4_hdmi_encoder *vc4_hdmi_encoder;
+ struct device_node *ddc_node;
+ u32 value;
+ int ret;
+
+ hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+ if (!hdmi)
+ return -ENOMEM;
+
+ vc4_hdmi_encoder = devm_kzalloc(dev, sizeof(*vc4_hdmi_encoder),
+ GFP_KERNEL);
+ if (!vc4_hdmi_encoder)
+ return -ENOMEM;
+ vc4_hdmi_encoder->base.type = VC4_ENCODER_TYPE_HDMI;
+ hdmi->encoder = &vc4_hdmi_encoder->base.base;
+
+ hdmi->pdev = pdev;
+ hdmi->hdmicore_regs = vc4_ioremap_regs(pdev, 0);
+ if (IS_ERR(hdmi->hdmicore_regs))
+ return PTR_ERR(hdmi->hdmicore_regs);
+
+ hdmi->hd_regs = vc4_ioremap_regs(pdev, 1);
+ if (IS_ERR(hdmi->hd_regs))
+ return PTR_ERR(hdmi->hd_regs);
+
+ ddc_node = of_parse_phandle(dev->of_node, "ddc", 0);
+ if (!ddc_node) {
+ DRM_ERROR("Failed to find ddc node in device tree\n");
+ return -ENODEV;
+ }
+
+ hdmi->pixel_clock = devm_clk_get(dev, "pixel");
+ if (IS_ERR(hdmi->pixel_clock)) {
+ DRM_ERROR("Failed to get pixel clock\n");
+ return PTR_ERR(hdmi->pixel_clock);
+ }
+ hdmi->hsm_clock = devm_clk_get(dev, "hdmi");
+ if (IS_ERR(hdmi->hsm_clock)) {
+ DRM_ERROR("Failed to get HDMI state machine clock\n");
+ return PTR_ERR(hdmi->hsm_clock);
+ }
+
+ hdmi->ddc = of_find_i2c_adapter_by_node(ddc_node);
+ if (!hdmi->ddc) {
+ DRM_DEBUG("Failed to get ddc i2c adapter by node\n");
+ return -EPROBE_DEFER;
+ }
+
+ /* Enable the clocks at startup. We can't quite recover from
+ * turning off the pixel clock during disable/enables yet, so
+ * it's always running.
+ */
+ ret = clk_prepare_enable(hdmi->pixel_clock);
+ if (ret) {
+ DRM_ERROR("Failed to turn on pixel clock: %d\n", ret);
+ goto err_put_i2c;
+ }
+
+ ret = clk_prepare_enable(hdmi->hsm_clock);
+ if (ret) {
+ DRM_ERROR("Failed to turn on HDMI state machine clock: %d\n",
+ ret);
+ goto err_unprepare_pix;
+ }
+
+ /* Only use the GPIO HPD pin if present in the DT, otherwise
+ * we'll use the HDMI core's register.
+ */
+ if (of_find_property(dev->of_node, "hpd-gpios", &value)) {
+ hdmi->hpd_gpio = of_get_named_gpio(dev->of_node, "hpd-gpios", 0);
+ if (hdmi->hpd_gpio < 0) {
+ ret = hdmi->hpd_gpio;
+ goto err_unprepare_hsm;
+ }
+ }
+
+ vc4->hdmi = hdmi;
+
+ /* HDMI core must be enabled. */
+ WARN_ON_ONCE((HD_READ(VC4_HD_M_CTL) & VC4_HD_M_ENABLE) == 0);
+
+ drm_encoder_init(drm, hdmi->encoder, &vc4_hdmi_encoder_funcs,
+ DRM_MODE_ENCODER_TMDS);
+ drm_encoder_helper_add(hdmi->encoder, &vc4_hdmi_encoder_helper_funcs);
+
+ hdmi->connector = vc4_hdmi_connector_init(drm, hdmi->encoder);
+ if (IS_ERR(hdmi->connector)) {
+ ret = PTR_ERR(hdmi->connector);
+ goto err_destroy_encoder;
+ }
+
+ return 0;
+
+err_destroy_encoder:
+ vc4_hdmi_encoder_destroy(hdmi->encoder);
+err_unprepare_hsm:
+ clk_disable_unprepare(hdmi->hsm_clock);
+err_unprepare_pix:
+ clk_disable_unprepare(hdmi->pixel_clock);
+err_put_i2c:
+ put_device(&vc4->hdmi->ddc->dev);
+
+ return ret;
+}
+
+static void vc4_hdmi_unbind(struct device *dev, struct device *master,
+ void *data)
+{
+ struct drm_device *drm = dev_get_drvdata(master);
+ struct vc4_dev *vc4 = drm->dev_private;
+ struct vc4_hdmi *hdmi = vc4->hdmi;
+
+ vc4_hdmi_connector_destroy(hdmi->connector);
+ vc4_hdmi_encoder_destroy(hdmi->encoder);
+
+ clk_disable_unprepare(hdmi->pixel_clock);
+ clk_disable_unprepare(hdmi->hsm_clock);
+ put_device(&hdmi->ddc->dev);
+
+ vc4->hdmi = NULL;
+}
+
+static const struct component_ops vc4_hdmi_ops = {
+ .bind = vc4_hdmi_bind,
+ .unbind = vc4_hdmi_unbind,
+};
+
+static int vc4_hdmi_dev_probe(struct platform_device *pdev)
+{
+ return component_add(&pdev->dev, &vc4_hdmi_ops);
+}
+
+static int vc4_hdmi_dev_remove(struct platform_device *pdev)
+{
+ component_del(&pdev->dev, &vc4_hdmi_ops);
+ return 0;
+}
+
+static const struct of_device_id vc4_hdmi_dt_match[] = {
+ { .compatible = "brcm,bcm2835-hdmi" },
+ {}
+};
+
+struct platform_driver vc4_hdmi_driver = {
+ .probe = vc4_hdmi_dev_probe,
+ .remove = vc4_hdmi_dev_remove,
+ .driver = {
+ .name = "vc4_hdmi",
+ .of_match_table = vc4_hdmi_dt_match,
+ },
+};
--- /dev/null
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/**
+ * DOC: VC4 HVS module.
+ *
+ * The HVS is the piece of hardware that does translation, scaling,
+ * colorspace conversion, and compositing of pixels stored in
+ * framebuffers into a FIFO of pixels going out to the Pixel Valve
+ * (CRTC). It operates at the system clock rate (the system audio
+ * clock gate, specifically), which is much higher than the pixel
+ * clock rate.
+ *
+ * There is a single global HVS, with multiple output FIFOs that can
+ * be consumed by the PVs. This file just manages the resources for
+ * the HVS, while the vc4_crtc.c code actually drives HVS setup for
+ * each CRTC.
+ */
+
+#include "linux/component.h"
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+#define HVS_REG(reg) { reg, #reg }
+static const struct {
+ u32 reg;
+ const char *name;
+} hvs_regs[] = {
+ HVS_REG(SCALER_DISPCTRL),
+ HVS_REG(SCALER_DISPSTAT),
+ HVS_REG(SCALER_DISPID),
+ HVS_REG(SCALER_DISPECTRL),
+ HVS_REG(SCALER_DISPPROF),
+ HVS_REG(SCALER_DISPDITHER),
+ HVS_REG(SCALER_DISPEOLN),
+ HVS_REG(SCALER_DISPLIST0),
+ HVS_REG(SCALER_DISPLIST1),
+ HVS_REG(SCALER_DISPLIST2),
+ HVS_REG(SCALER_DISPLSTAT),
+ HVS_REG(SCALER_DISPLACT0),
+ HVS_REG(SCALER_DISPLACT1),
+ HVS_REG(SCALER_DISPLACT2),
+ HVS_REG(SCALER_DISPCTRL0),
+ HVS_REG(SCALER_DISPBKGND0),
+ HVS_REG(SCALER_DISPSTAT0),
+ HVS_REG(SCALER_DISPBASE0),
+ HVS_REG(SCALER_DISPCTRL1),
+ HVS_REG(SCALER_DISPBKGND1),
+ HVS_REG(SCALER_DISPSTAT1),
+ HVS_REG(SCALER_DISPBASE1),
+ HVS_REG(SCALER_DISPCTRL2),
+ HVS_REG(SCALER_DISPBKGND2),
+ HVS_REG(SCALER_DISPSTAT2),
+ HVS_REG(SCALER_DISPBASE2),
+ HVS_REG(SCALER_DISPALPHA2),
+};
+
+void vc4_hvs_dump_state(struct drm_device *dev)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(hvs_regs); i++) {
+ DRM_INFO("0x%04x (%s): 0x%08x\n",
+ hvs_regs[i].reg, hvs_regs[i].name,
+ HVS_READ(hvs_regs[i].reg));
+ }
+
+ DRM_INFO("HVS ctx:\n");
+ for (i = 0; i < 64; i += 4) {
+ DRM_INFO("0x%08x (%s): 0x%08x 0x%08x 0x%08x 0x%08x\n",
+ i * 4, i < HVS_BOOTLOADER_DLIST_END ? "B" : "D",
+ ((uint32_t *)vc4->hvs->dlist)[i + 0],
+ ((uint32_t *)vc4->hvs->dlist)[i + 1],
+ ((uint32_t *)vc4->hvs->dlist)[i + 2],
+ ((uint32_t *)vc4->hvs->dlist)[i + 3]);
+ }
+}
+
+#ifdef CONFIG_DEBUG_FS
+int vc4_hvs_debugfs_regs(struct seq_file *m, void *unused)
+{
+ struct drm_info_node *node = (struct drm_info_node *)m->private;
+ struct drm_device *dev = node->minor->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(hvs_regs); i++) {
+ seq_printf(m, "%s (0x%04x): 0x%08x\n",
+ hvs_regs[i].name, hvs_regs[i].reg,
+ HVS_READ(hvs_regs[i].reg));
+ }
+
+ return 0;
+}
+#endif
+
+static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct drm_device *drm = dev_get_drvdata(master);
+ struct vc4_dev *vc4 = drm->dev_private;
+ struct vc4_hvs *hvs = NULL;
+
+ hvs = devm_kzalloc(&pdev->dev, sizeof(*hvs), GFP_KERNEL);
+ if (!hvs)
+ return -ENOMEM;
+
+ hvs->pdev = pdev;
+
+ hvs->regs = vc4_ioremap_regs(pdev, 0);
+ if (IS_ERR(hvs->regs))
+ return PTR_ERR(hvs->regs);
+
+ hvs->dlist = hvs->regs + SCALER_DLIST_START;
+
+ vc4->hvs = hvs;
+ return 0;
+}
+
+static void vc4_hvs_unbind(struct device *dev, struct device *master,
+ void *data)
+{
+ struct drm_device *drm = dev_get_drvdata(master);
+ struct vc4_dev *vc4 = drm->dev_private;
+
+ vc4->hvs = NULL;
+}
+
+static const struct component_ops vc4_hvs_ops = {
+ .bind = vc4_hvs_bind,
+ .unbind = vc4_hvs_unbind,
+};
+
+static int vc4_hvs_dev_probe(struct platform_device *pdev)
+{
+ return component_add(&pdev->dev, &vc4_hvs_ops);
+}
+
+static int vc4_hvs_dev_remove(struct platform_device *pdev)
+{
+ component_del(&pdev->dev, &vc4_hvs_ops);
+ return 0;
+}
+
+static const struct of_device_id vc4_hvs_dt_match[] = {
+ { .compatible = "brcm,bcm2835-hvs" },
+ {}
+};
+
+struct platform_driver vc4_hvs_driver = {
+ .probe = vc4_hvs_dev_probe,
+ .remove = vc4_hvs_dev_remove,
+ .driver = {
+ .name = "vc4_hvs",
+ .of_match_table = vc4_hvs_dt_match,
+ },
+};
--- /dev/null
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/**
+ * DOC: VC4 KMS
+ *
+ * This is the general code for implementing KMS mode setting that
+ * doesn't clearly associate with any of the other objects (plane,
+ * crtc, HDMI encoder).
+ */
+
+#include "drm_crtc.h"
+#include "drm_atomic_helper.h"
+#include "drm_crtc_helper.h"
+#include "drm_plane_helper.h"
+#include "drm_fb_cma_helper.h"
+#include "vc4_drv.h"
+
+static void vc4_output_poll_changed(struct drm_device *dev)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+ if (vc4->fbdev)
+ drm_fbdev_cma_hotplug_event(vc4->fbdev);
+}
+
+static const struct drm_mode_config_funcs vc4_mode_funcs = {
+ .output_poll_changed = vc4_output_poll_changed,
+ .atomic_check = drm_atomic_helper_check,
+ .atomic_commit = drm_atomic_helper_commit,
+ .fb_create = drm_fb_cma_create,
+};
+
+int vc4_kms_load(struct drm_device *dev)
+{
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ int ret;
+
+ ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
+ if (ret < 0) {
+ dev_err(dev->dev, "failed to initialize vblank\n");
+ return ret;
+ }
+
+ dev->mode_config.max_width = 2048;
+ dev->mode_config.max_height = 2048;
+ dev->mode_config.funcs = &vc4_mode_funcs;
+ dev->mode_config.preferred_depth = 24;
+ dev->vblank_disable_allowed = true;
+
+ drm_mode_config_reset(dev);
+
+ vc4->fbdev = drm_fbdev_cma_init(dev, 32,
+ dev->mode_config.num_crtc,
+ dev->mode_config.num_connector);
+ if (IS_ERR(vc4->fbdev))
+ vc4->fbdev = NULL;
+
+ drm_kms_helper_poll_init(dev);
+
+ return 0;
+}
--- /dev/null
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/**
+ * DOC: VC4 plane module
+ *
+ * Each DRM plane is a layer of pixels being scanned out by the HVS.
+ *
+ * At atomic modeset check time, we compute the HVS display element
+ * state that would be necessary for displaying the plane (giving us a
+ * chance to figure out if a plane configuration is invalid), then at
+ * atomic flush time the CRTC will ask us to write our element state
+ * into the region of the HVS that it has allocated for us.
+ */
+
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+#include "drm_atomic_helper.h"
+#include "drm_fb_cma_helper.h"
+#include "drm_plane_helper.h"
+
+struct vc4_plane_state {
+ struct drm_plane_state base;
+ u32 *dlist;
+ u32 dlist_size; /* Number of dwords in allocated for the display list */
+ u32 dlist_count; /* Number of used dwords in the display list. */
+};
+
+static inline struct vc4_plane_state *
+to_vc4_plane_state(struct drm_plane_state *state)
+{
+ return (struct vc4_plane_state *)state;
+}
+
+static const struct hvs_format {
+ u32 drm; /* DRM_FORMAT_* */
+ u32 hvs; /* HVS_FORMAT_* */
+ u32 pixel_order;
+ bool has_alpha;
+} hvs_formats[] = {
+ {
+ .drm = DRM_FORMAT_XRGB8888, .hvs = HVS_PIXEL_FORMAT_RGBA8888,
+ .pixel_order = HVS_PIXEL_ORDER_ABGR, .has_alpha = false,
+ },
+ {
+ .drm = DRM_FORMAT_ARGB8888, .hvs = HVS_PIXEL_FORMAT_RGBA8888,
+ .pixel_order = HVS_PIXEL_ORDER_ABGR, .has_alpha = true,
+ },
+};
+
+static const struct hvs_format *vc4_get_hvs_format(u32 drm_format)
+{
+ unsigned i;
+
+ for (i = 0; i < ARRAY_SIZE(hvs_formats); i++) {
+ if (hvs_formats[i].drm == drm_format)
+ return &hvs_formats[i];
+ }
+
+ return NULL;
+}
+
+static bool plane_enabled(struct drm_plane_state *state)
+{
+ return state->fb && state->crtc;
+}
+
+struct drm_plane_state *vc4_plane_duplicate_state(struct drm_plane *plane)
+{
+ struct vc4_plane_state *vc4_state;
+
+ if (WARN_ON(!plane->state))
+ return NULL;
+
+ vc4_state = kmemdup(plane->state, sizeof(*vc4_state), GFP_KERNEL);
+ if (!vc4_state)
+ return NULL;
+
+ __drm_atomic_helper_plane_duplicate_state(plane, &vc4_state->base);
+
+ if (vc4_state->dlist) {
+ vc4_state->dlist = kmemdup(vc4_state->dlist,
+ vc4_state->dlist_count * 4,
+ GFP_KERNEL);
+ if (!vc4_state->dlist) {
+ kfree(vc4_state);
+ return NULL;
+ }
+ vc4_state->dlist_size = vc4_state->dlist_count;
+ }
+
+ return &vc4_state->base;
+}
+
+void vc4_plane_destroy_state(struct drm_plane *plane,
+ struct drm_plane_state *state)
+{
+ struct vc4_plane_state *vc4_state = to_vc4_plane_state(state);
+
+ kfree(vc4_state->dlist);
+ __drm_atomic_helper_plane_destroy_state(plane, &vc4_state->base);
+ kfree(state);
+}
+
+/* Called during init to allocate the plane's atomic state. */
+void vc4_plane_reset(struct drm_plane *plane)
+{
+ struct vc4_plane_state *vc4_state;
+
+ WARN_ON(plane->state);
+
+ vc4_state = kzalloc(sizeof(*vc4_state), GFP_KERNEL);
+ if (!vc4_state)
+ return;
+
+ plane->state = &vc4_state->base;
+ vc4_state->base.plane = plane;
+}
+
+static void vc4_dlist_write(struct vc4_plane_state *vc4_state, u32 val)
+{
+ if (vc4_state->dlist_count == vc4_state->dlist_size) {
+ u32 new_size = max(4u, vc4_state->dlist_count * 2);
+ u32 *new_dlist = kmalloc(new_size * 4, GFP_KERNEL);
+
+ if (!new_dlist)
+ return;
+ memcpy(new_dlist, vc4_state->dlist, vc4_state->dlist_count * 4);
+
+ kfree(vc4_state->dlist);
+ vc4_state->dlist = new_dlist;
+ vc4_state->dlist_size = new_size;
+ }
+
+ vc4_state->dlist[vc4_state->dlist_count++] = val;
+}
+
+/* Writes out a full display list for an active plane to the plane's
+ * private dlist state.
+ */
+static int vc4_plane_mode_set(struct drm_plane *plane,
+ struct drm_plane_state *state)
+{
+ struct vc4_plane_state *vc4_state = to_vc4_plane_state(state);
+ struct drm_framebuffer *fb = state->fb;
+ struct drm_gem_cma_object *bo = drm_fb_cma_get_gem_obj(fb, 0);
+ u32 ctl0_offset = vc4_state->dlist_count;
+ const struct hvs_format *format = vc4_get_hvs_format(fb->pixel_format);
+ uint32_t offset = fb->offsets[0];
+ int crtc_x = state->crtc_x;
+ int crtc_y = state->crtc_y;
+ int crtc_w = state->crtc_w;
+ int crtc_h = state->crtc_h;
+
+ if (crtc_x < 0) {
+ offset += drm_format_plane_cpp(fb->pixel_format, 0) * -crtc_x;
+ crtc_w += crtc_x;
+ crtc_x = 0;
+ }
+
+ if (crtc_y < 0) {
+ offset += fb->pitches[0] * -crtc_y;
+ crtc_h += crtc_y;
+ crtc_y = 0;
+ }
+
+ vc4_dlist_write(vc4_state,
+ SCALER_CTL0_VALID |
+ (format->pixel_order << SCALER_CTL0_ORDER_SHIFT) |
+ (format->hvs << SCALER_CTL0_PIXEL_FORMAT_SHIFT) |
+ SCALER_CTL0_UNITY);
+
+ /* Position Word 0: Image Positions and Alpha Value */
+ vc4_dlist_write(vc4_state,
+ VC4_SET_FIELD(0xff, SCALER_POS0_FIXED_ALPHA) |
+ VC4_SET_FIELD(crtc_x, SCALER_POS0_START_X) |
+ VC4_SET_FIELD(crtc_y, SCALER_POS0_START_Y));
+
+ /* Position Word 1: Scaled Image Dimensions.
+ * Skipped due to SCALER_CTL0_UNITY scaling.
+ */
+
+ /* Position Word 2: Source Image Size, Alpha Mode */
+ vc4_dlist_write(vc4_state,
+ VC4_SET_FIELD(format->has_alpha ?
+ SCALER_POS2_ALPHA_MODE_PIPELINE :
+ SCALER_POS2_ALPHA_MODE_FIXED,
+ SCALER_POS2_ALPHA_MODE) |
+ VC4_SET_FIELD(crtc_w, SCALER_POS2_WIDTH) |
+ VC4_SET_FIELD(crtc_h, SCALER_POS2_HEIGHT));
+
+ /* Position Word 3: Context. Written by the HVS. */
+ vc4_dlist_write(vc4_state, 0xc0c0c0c0);
+
+ /* Pointer Word 0: RGB / Y Pointer */
+ vc4_dlist_write(vc4_state, bo->paddr + offset);
+
+ /* Pointer Context Word 0: Written by the HVS */
+ vc4_dlist_write(vc4_state, 0xc0c0c0c0);
+
+ /* Pitch word 0: Pointer 0 Pitch */
+ vc4_dlist_write(vc4_state,
+ VC4_SET_FIELD(fb->pitches[0], SCALER_SRC_PITCH));
+
+ vc4_state->dlist[ctl0_offset] |=
+ VC4_SET_FIELD(vc4_state->dlist_count, SCALER_CTL0_SIZE);
+
+ return 0;
+}
+
+/* If a modeset involves changing the setup of a plane, the atomic
+ * infrastructure will call this to validate a proposed plane setup.
+ * However, if a plane isn't getting updated, this (and the
+ * corresponding vc4_plane_atomic_update) won't get called. Thus, we
+ * compute the dlist here and have all active plane dlists get updated
+ * in the CRTC's flush.
+ */
+static int vc4_plane_atomic_check(struct drm_plane *plane,
+ struct drm_plane_state *state)
+{
+ struct vc4_plane_state *vc4_state = to_vc4_plane_state(state);
+
+ vc4_state->dlist_count = 0;
+
+ if (plane_enabled(state))
+ return vc4_plane_mode_set(plane, state);
+ else
+ return 0;
+}
+
+static void vc4_plane_atomic_update(struct drm_plane *plane,
+ struct drm_plane_state *old_state)
+{
+ /* No contents here. Since we don't know where in the CRTC's
+ * dlist we should be stored, our dlist is uploaded to the
+ * hardware with vc4_plane_write_dlist() at CRTC atomic_flush
+ * time.
+ */
+}
+
+u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist)
+{
+ struct vc4_plane_state *vc4_state = to_vc4_plane_state(plane->state);
+ int i;
+
+ /* Can't memcpy_toio() because it needs to be 32-bit writes. */
+ for (i = 0; i < vc4_state->dlist_count; i++)
+ writel(vc4_state->dlist[i], &dlist[i]);
+
+ return vc4_state->dlist_count;
+}
+
+u32 vc4_plane_dlist_size(struct drm_plane_state *state)
+{
+ struct vc4_plane_state *vc4_state = to_vc4_plane_state(state);
+
+ return vc4_state->dlist_count;
+}
+
+static const struct drm_plane_helper_funcs vc4_plane_helper_funcs = {
+ .prepare_fb = NULL,
+ .cleanup_fb = NULL,
+ .atomic_check = vc4_plane_atomic_check,
+ .atomic_update = vc4_plane_atomic_update,
+};
+
+static void vc4_plane_destroy(struct drm_plane *plane)
+{
+ drm_plane_helper_disable(plane);
+ drm_plane_cleanup(plane);
+}
+
+static const struct drm_plane_funcs vc4_plane_funcs = {
+ .update_plane = drm_atomic_helper_update_plane,
+ .disable_plane = drm_atomic_helper_disable_plane,
+ .destroy = vc4_plane_destroy,
+ .set_property = NULL,
+ .reset = vc4_plane_reset,
+ .atomic_duplicate_state = vc4_plane_duplicate_state,
+ .atomic_destroy_state = vc4_plane_destroy_state,
+};
+
+struct drm_plane *vc4_plane_init(struct drm_device *dev,
+ enum drm_plane_type type)
+{
+ struct drm_plane *plane = NULL;
+ struct vc4_plane *vc4_plane;
+ u32 formats[ARRAY_SIZE(hvs_formats)];
+ int ret = 0;
+ unsigned i;
+
+ vc4_plane = devm_kzalloc(dev->dev, sizeof(*vc4_plane),
+ GFP_KERNEL);
+ if (!vc4_plane) {
+ ret = -ENOMEM;
+ goto fail;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(hvs_formats); i++)
+ formats[i] = hvs_formats[i].drm;
+ plane = &vc4_plane->base;
+ ret = drm_universal_plane_init(dev, plane, 0xff,
+ &vc4_plane_funcs,
+ formats, ARRAY_SIZE(formats),
+ type);
+
+ drm_plane_helper_add(plane, &vc4_plane_helper_funcs);
+
+ return plane;
+fail:
+ if (plane)
+ vc4_plane_destroy(plane);
+
+ return ERR_PTR(ret);
+}
--- /dev/null
+/*
+ * Copyright © 2014-2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef VC4_REGS_H
+#define VC4_REGS_H
+
+#include <linux/bitops.h>
+
+#define VC4_MASK(high, low) ((u32)GENMASK(high, low))
+/* Using the GNU statement expression extension */
+#define VC4_SET_FIELD(value, field) \
+ ({ \
+ uint32_t fieldval = (value) << field##_SHIFT; \
+ WARN_ON((fieldval & ~field##_MASK) != 0); \
+ fieldval & field##_MASK; \
+ })
+
+#define VC4_GET_FIELD(word, field) (((word) & field##_MASK) >> \
+ field##_SHIFT)
+
+#define V3D_IDENT0 0x00000
+# define V3D_EXPECTED_IDENT0 \
+ ((2 << 24) | \
+ ('V' << 0) | \
+ ('3' << 8) | \
+ ('D' << 16))
+
+#define V3D_IDENT1 0x00004
+/* Multiples of 1kb */
+# define V3D_IDENT1_VPM_SIZE_MASK VC4_MASK(31, 28)
+# define V3D_IDENT1_VPM_SIZE_SHIFT 28
+# define V3D_IDENT1_NSEM_MASK VC4_MASK(23, 16)
+# define V3D_IDENT1_NSEM_SHIFT 16
+# define V3D_IDENT1_TUPS_MASK VC4_MASK(15, 12)
+# define V3D_IDENT1_TUPS_SHIFT 12
+# define V3D_IDENT1_QUPS_MASK VC4_MASK(11, 8)
+# define V3D_IDENT1_QUPS_SHIFT 8
+# define V3D_IDENT1_NSLC_MASK VC4_MASK(7, 4)
+# define V3D_IDENT1_NSLC_SHIFT 4
+# define V3D_IDENT1_REV_MASK VC4_MASK(3, 0)
+# define V3D_IDENT1_REV_SHIFT 0
+
+#define V3D_IDENT2 0x00008
+#define V3D_SCRATCH 0x00010
+#define V3D_L2CACTL 0x00020
+# define V3D_L2CACTL_L2CCLR BIT(2)
+# define V3D_L2CACTL_L2CDIS BIT(1)
+# define V3D_L2CACTL_L2CENA BIT(0)
+
+#define V3D_SLCACTL 0x00024
+# define V3D_SLCACTL_T1CC_MASK VC4_MASK(27, 24)
+# define V3D_SLCACTL_T1CC_SHIFT 24
+# define V3D_SLCACTL_T0CC_MASK VC4_MASK(19, 16)
+# define V3D_SLCACTL_T0CC_SHIFT 16
+# define V3D_SLCACTL_UCC_MASK VC4_MASK(11, 8)
+# define V3D_SLCACTL_UCC_SHIFT 8
+# define V3D_SLCACTL_ICC_MASK VC4_MASK(3, 0)
+# define V3D_SLCACTL_ICC_SHIFT 0
+
+#define V3D_INTCTL 0x00030
+#define V3D_INTENA 0x00034
+#define V3D_INTDIS 0x00038
+# define V3D_INT_SPILLUSE BIT(3)
+# define V3D_INT_OUTOMEM BIT(2)
+# define V3D_INT_FLDONE BIT(1)
+# define V3D_INT_FRDONE BIT(0)
+
+#define V3D_CT0CS 0x00100
+#define V3D_CT1CS 0x00104
+#define V3D_CTNCS(n) (V3D_CT0CS + 4 * n)
+# define V3D_CTRSTA BIT(15)
+# define V3D_CTSEMA BIT(12)
+# define V3D_CTRTSD BIT(8)
+# define V3D_CTRUN BIT(5)
+# define V3D_CTSUBS BIT(4)
+# define V3D_CTERR BIT(3)
+# define V3D_CTMODE BIT(0)
+
+#define V3D_CT0EA 0x00108
+#define V3D_CT1EA 0x0010c
+#define V3D_CTNEA(n) (V3D_CT0EA + 4 * (n))
+#define V3D_CT0CA 0x00110
+#define V3D_CT1CA 0x00114
+#define V3D_CTNCA(n) (V3D_CT0CA + 4 * (n))
+#define V3D_CT00RA0 0x00118
+#define V3D_CT01RA0 0x0011c
+#define V3D_CTNRA0(n) (V3D_CT00RA0 + 4 * (n))
+#define V3D_CT0LC 0x00120
+#define V3D_CT1LC 0x00124
+#define V3D_CTNLC(n) (V3D_CT0LC + 4 * (n))
+#define V3D_CT0PC 0x00128
+#define V3D_CT1PC 0x0012c
+#define V3D_CTNPC(n) (V3D_CT0PC + 4 * (n))
+
+#define V3D_PCS 0x00130
+# define V3D_BMOOM BIT(8)
+# define V3D_RMBUSY BIT(3)
+# define V3D_RMACTIVE BIT(2)
+# define V3D_BMBUSY BIT(1)
+# define V3D_BMACTIVE BIT(0)
+
+#define V3D_BFC 0x00134
+#define V3D_RFC 0x00138
+#define V3D_BPCA 0x00300
+#define V3D_BPCS 0x00304
+#define V3D_BPOA 0x00308
+#define V3D_BPOS 0x0030c
+#define V3D_BXCF 0x00310
+#define V3D_SQRSV0 0x00410
+#define V3D_SQRSV1 0x00414
+#define V3D_SQCNTL 0x00418
+#define V3D_SRQPC 0x00430
+#define V3D_SRQUA 0x00434
+#define V3D_SRQUL 0x00438
+#define V3D_SRQCS 0x0043c
+#define V3D_VPACNTL 0x00500
+#define V3D_VPMBASE 0x00504
+#define V3D_PCTRC 0x00670
+#define V3D_PCTRE 0x00674
+#define V3D_PCTR0 0x00680
+#define V3D_PCTRS0 0x00684
+#define V3D_PCTR1 0x00688
+#define V3D_PCTRS1 0x0068c
+#define V3D_PCTR2 0x00690
+#define V3D_PCTRS2 0x00694
+#define V3D_PCTR3 0x00698
+#define V3D_PCTRS3 0x0069c
+#define V3D_PCTR4 0x006a0
+#define V3D_PCTRS4 0x006a4
+#define V3D_PCTR5 0x006a8
+#define V3D_PCTRS5 0x006ac
+#define V3D_PCTR6 0x006b0
+#define V3D_PCTRS6 0x006b4
+#define V3D_PCTR7 0x006b8
+#define V3D_PCTRS7 0x006bc
+#define V3D_PCTR8 0x006c0
+#define V3D_PCTRS8 0x006c4
+#define V3D_PCTR9 0x006c8
+#define V3D_PCTRS9 0x006cc
+#define V3D_PCTR10 0x006d0
+#define V3D_PCTRS10 0x006d4
+#define V3D_PCTR11 0x006d8
+#define V3D_PCTRS11 0x006dc
+#define V3D_PCTR12 0x006e0
+#define V3D_PCTRS12 0x006e4
+#define V3D_PCTR13 0x006e8
+#define V3D_PCTRS13 0x006ec
+#define V3D_PCTR14 0x006f0
+#define V3D_PCTRS14 0x006f4
+#define V3D_PCTR15 0x006f8
+#define V3D_PCTRS15 0x006fc
+#define V3D_BGE 0x00f00
+#define V3D_FDBGO 0x00f04
+#define V3D_FDBGB 0x00f08
+#define V3D_FDBGR 0x00f0c
+#define V3D_FDBGS 0x00f10
+#define V3D_ERRSTAT 0x00f20
+
+#define PV_CONTROL 0x00
+# define PV_CONTROL_FORMAT_MASK VC4_MASK(23, 21)
+# define PV_CONTROL_FORMAT_SHIFT 21
+# define PV_CONTROL_FORMAT_24 0
+# define PV_CONTROL_FORMAT_DSIV_16 1
+# define PV_CONTROL_FORMAT_DSIC_16 2
+# define PV_CONTROL_FORMAT_DSIV_18 3
+# define PV_CONTROL_FORMAT_DSIV_24 4
+
+# define PV_CONTROL_FIFO_LEVEL_MASK VC4_MASK(20, 15)
+# define PV_CONTROL_FIFO_LEVEL_SHIFT 15
+# define PV_CONTROL_CLR_AT_START BIT(14)
+# define PV_CONTROL_TRIGGER_UNDERFLOW BIT(13)
+# define PV_CONTROL_WAIT_HSTART BIT(12)
+# define PV_CONTROL_CLK_SELECT_DSI_VEC 0
+# define PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI 1
+# define PV_CONTROL_CLK_SELECT_MASK VC4_MASK(3, 2)
+# define PV_CONTROL_CLK_SELECT_SHIFT 2
+# define PV_CONTROL_FIFO_CLR BIT(1)
+# define PV_CONTROL_EN BIT(0)
+
+#define PV_V_CONTROL 0x04
+# define PV_VCONTROL_INTERLACE BIT(4)
+# define PV_VCONTROL_CONTINUOUS BIT(1)
+# define PV_VCONTROL_VIDEN BIT(0)
+
+#define PV_VSYNCD 0x08
+
+#define PV_HORZA 0x0c
+# define PV_HORZA_HBP_MASK VC4_MASK(31, 16)
+# define PV_HORZA_HBP_SHIFT 16
+# define PV_HORZA_HSYNC_MASK VC4_MASK(15, 0)
+# define PV_HORZA_HSYNC_SHIFT 0
+
+#define PV_HORZB 0x10
+# define PV_HORZB_HFP_MASK VC4_MASK(31, 16)
+# define PV_HORZB_HFP_SHIFT 16
+# define PV_HORZB_HACTIVE_MASK VC4_MASK(15, 0)
+# define PV_HORZB_HACTIVE_SHIFT 0
+
+#define PV_VERTA 0x14
+# define PV_VERTA_VBP_MASK VC4_MASK(31, 16)
+# define PV_VERTA_VBP_SHIFT 16
+# define PV_VERTA_VSYNC_MASK VC4_MASK(15, 0)
+# define PV_VERTA_VSYNC_SHIFT 0
+
+#define PV_VERTB 0x18
+# define PV_VERTB_VFP_MASK VC4_MASK(31, 16)
+# define PV_VERTB_VFP_SHIFT 16
+# define PV_VERTB_VACTIVE_MASK VC4_MASK(15, 0)
+# define PV_VERTB_VACTIVE_SHIFT 0
+
+#define PV_VERTA_EVEN 0x1c
+#define PV_VERTB_EVEN 0x20
+
+#define PV_INTEN 0x24
+#define PV_INTSTAT 0x28
+# define PV_INT_VID_IDLE BIT(9)
+# define PV_INT_VFP_END BIT(8)
+# define PV_INT_VFP_START BIT(7)
+# define PV_INT_VACT_START BIT(6)
+# define PV_INT_VBP_START BIT(5)
+# define PV_INT_VSYNC_START BIT(4)
+# define PV_INT_HFP_START BIT(3)
+# define PV_INT_HACT_START BIT(2)
+# define PV_INT_HBP_START BIT(1)
+# define PV_INT_HSYNC_START BIT(0)
+
+#define PV_STAT 0x2c
+
+#define PV_HACT_ACT 0x30
+
+#define SCALER_DISPCTRL 0x00000000
+/* Global register for clock gating the HVS */
+# define SCALER_DISPCTRL_ENABLE BIT(31)
+# define SCALER_DISPCTRL_DSP2EISLUR BIT(15)
+# define SCALER_DISPCTRL_DSP1EISLUR BIT(14)
+/* Enables Display 0 short line and underrun contribution to
+ * SCALER_DISPSTAT_IRQDISP0. Note that short frame contributions are
+ * always enabled.
+ */
+# define SCALER_DISPCTRL_DSP0EISLUR BIT(13)
+# define SCALER_DISPCTRL_DSP2EIEOLN BIT(12)
+# define SCALER_DISPCTRL_DSP2EIEOF BIT(11)
+# define SCALER_DISPCTRL_DSP1EIEOLN BIT(10)
+# define SCALER_DISPCTRL_DSP1EIEOF BIT(9)
+/* Enables Display 0 end-of-line-N contribution to
+ * SCALER_DISPSTAT_IRQDISP0
+ */
+# define SCALER_DISPCTRL_DSP0EIEOLN BIT(8)
+/* Enables Display 0 EOF contribution to SCALER_DISPSTAT_IRQDISP0 */
+# define SCALER_DISPCTRL_DSP0EIEOF BIT(7)
+
+# define SCALER_DISPCTRL_SLVRDEIRQ BIT(6)
+# define SCALER_DISPCTRL_SLVWREIRQ BIT(5)
+# define SCALER_DISPCTRL_DMAEIRQ BIT(4)
+# define SCALER_DISPCTRL_DISP2EIRQ BIT(3)
+# define SCALER_DISPCTRL_DISP1EIRQ BIT(2)
+/* Enables interrupt generation on the enabled EOF/EOLN/EISLUR
+ * bits and short frames..
+ */
+# define SCALER_DISPCTRL_DISP0EIRQ BIT(1)
+/* Enables interrupt generation on scaler profiler interrupt. */
+# define SCALER_DISPCTRL_SCLEIRQ BIT(0)
+
+#define SCALER_DISPSTAT 0x00000004
+# define SCALER_DISPSTAT_COBLOW2 BIT(29)
+# define SCALER_DISPSTAT_EOLN2 BIT(28)
+# define SCALER_DISPSTAT_ESFRAME2 BIT(27)
+# define SCALER_DISPSTAT_ESLINE2 BIT(26)
+# define SCALER_DISPSTAT_EUFLOW2 BIT(25)
+# define SCALER_DISPSTAT_EOF2 BIT(24)
+
+# define SCALER_DISPSTAT_COBLOW1 BIT(21)
+# define SCALER_DISPSTAT_EOLN1 BIT(20)
+# define SCALER_DISPSTAT_ESFRAME1 BIT(19)
+# define SCALER_DISPSTAT_ESLINE1 BIT(18)
+# define SCALER_DISPSTAT_EUFLOW1 BIT(17)
+# define SCALER_DISPSTAT_EOF1 BIT(16)
+
+# define SCALER_DISPSTAT_RESP_MASK VC4_MASK(15, 14)
+# define SCALER_DISPSTAT_RESP_SHIFT 14
+# define SCALER_DISPSTAT_RESP_OKAY 0
+# define SCALER_DISPSTAT_RESP_EXOKAY 1
+# define SCALER_DISPSTAT_RESP_SLVERR 2
+# define SCALER_DISPSTAT_RESP_DECERR 3
+
+# define SCALER_DISPSTAT_COBLOW0 BIT(13)
+/* Set when the DISPEOLN line is done compositing. */
+# define SCALER_DISPSTAT_EOLN0 BIT(12)
+/* Set when VSTART is seen but there are still pixels in the current
+ * output line.
+ */
+# define SCALER_DISPSTAT_ESFRAME0 BIT(11)
+/* Set when HSTART is seen but there are still pixels in the current
+ * output line.
+ */
+# define SCALER_DISPSTAT_ESLINE0 BIT(10)
+/* Set when the the downstream tries to read from the display FIFO
+ * while it's empty.
+ */
+# define SCALER_DISPSTAT_EUFLOW0 BIT(9)
+/* Set when the display mode changes from RUN to EOF */
+# define SCALER_DISPSTAT_EOF0 BIT(8)
+
+/* Set on AXI invalid DMA ID error. */
+# define SCALER_DISPSTAT_DMA_ERROR BIT(7)
+/* Set on AXI slave read decode error */
+# define SCALER_DISPSTAT_IRQSLVRD BIT(6)
+/* Set on AXI slave write decode error */
+# define SCALER_DISPSTAT_IRQSLVWR BIT(5)
+/* Set when SCALER_DISPSTAT_DMA_ERROR is set, or
+ * SCALER_DISPSTAT_RESP_ERROR is not SCALER_DISPSTAT_RESP_OKAY.
+ */
+# define SCALER_DISPSTAT_IRQDMA BIT(4)
+# define SCALER_DISPSTAT_IRQDISP2 BIT(3)
+# define SCALER_DISPSTAT_IRQDISP1 BIT(2)
+/* Set when any of the EOF/EOLN/ESFRAME/ESLINE bits are set and their
+ * corresponding interrupt bit is enabled in DISPCTRL.
+ */
+# define SCALER_DISPSTAT_IRQDISP0 BIT(1)
+/* On read, the profiler interrupt. On write, clear *all* interrupt bits. */
+# define SCALER_DISPSTAT_IRQSCL BIT(0)
+
+#define SCALER_DISPID 0x00000008
+#define SCALER_DISPECTRL 0x0000000c
+#define SCALER_DISPPROF 0x00000010
+#define SCALER_DISPDITHER 0x00000014
+#define SCALER_DISPEOLN 0x00000018
+#define SCALER_DISPLIST0 0x00000020
+#define SCALER_DISPLIST1 0x00000024
+#define SCALER_DISPLIST2 0x00000028
+#define SCALER_DISPLSTAT 0x0000002c
+#define SCALER_DISPLISTX(x) (SCALER_DISPLIST0 + \
+ (x) * (SCALER_DISPLIST1 - \
+ SCALER_DISPLIST0))
+
+#define SCALER_DISPLACT0 0x00000030
+#define SCALER_DISPLACT1 0x00000034
+#define SCALER_DISPLACT2 0x00000038
+#define SCALER_DISPCTRL0 0x00000040
+# define SCALER_DISPCTRLX_ENABLE BIT(31)
+# define SCALER_DISPCTRLX_RESET BIT(30)
+# define SCALER_DISPCTRLX_WIDTH_MASK VC4_MASK(23, 12)
+# define SCALER_DISPCTRLX_WIDTH_SHIFT 12
+# define SCALER_DISPCTRLX_HEIGHT_MASK VC4_MASK(11, 0)
+# define SCALER_DISPCTRLX_HEIGHT_SHIFT 0
+
+#define SCALER_DISPBKGND0 0x00000044
+#define SCALER_DISPSTAT0 0x00000048
+#define SCALER_DISPBASE0 0x0000004c
+# define SCALER_DISPSTATX_MODE_MASK VC4_MASK(31, 30)
+# define SCALER_DISPSTATX_MODE_SHIFT 30
+# define SCALER_DISPSTATX_MODE_DISABLED 0
+# define SCALER_DISPSTATX_MODE_INIT 1
+# define SCALER_DISPSTATX_MODE_RUN 2
+# define SCALER_DISPSTATX_MODE_EOF 3
+# define SCALER_DISPSTATX_FULL BIT(29)
+# define SCALER_DISPSTATX_EMPTY BIT(28)
+#define SCALER_DISPCTRL1 0x00000050
+#define SCALER_DISPBKGND1 0x00000054
+#define SCALER_DISPSTAT1 0x00000058
+#define SCALER_DISPSTATX(x) (SCALER_DISPSTAT0 + \
+ (x) * (SCALER_DISPSTAT1 - \
+ SCALER_DISPSTAT0))
+#define SCALER_DISPBASE1 0x0000005c
+#define SCALER_DISPCTRL2 0x00000060
+#define SCALER_DISPCTRLX(x) (SCALER_DISPCTRL0 + \
+ (x) * (SCALER_DISPCTRL1 - \
+ SCALER_DISPCTRL0))
+#define SCALER_DISPBKGND2 0x00000064
+#define SCALER_DISPSTAT2 0x00000068
+#define SCALER_DISPBASE2 0x0000006c
+#define SCALER_DISPALPHA2 0x00000070
+#define SCALER_GAMADDR 0x00000078
+#define SCALER_GAMDATA 0x000000e0
+#define SCALER_DLIST_START 0x00002000
+#define SCALER_DLIST_SIZE 0x00004000
+
+#define VC4_HDMI_CORE_REV 0x000
+
+#define VC4_HDMI_SW_RESET_CONTROL 0x004
+# define VC4_HDMI_SW_RESET_FORMAT_DETECT BIT(1)
+# define VC4_HDMI_SW_RESET_HDMI BIT(0)
+
+#define VC4_HDMI_HOTPLUG_INT 0x008
+
+#define VC4_HDMI_HOTPLUG 0x00c
+# define VC4_HDMI_HOTPLUG_CONNECTED BIT(0)
+
+#define VC4_HDMI_RAM_PACKET_CONFIG 0x0a0
+# define VC4_HDMI_RAM_PACKET_ENABLE BIT(16)
+
+#define VC4_HDMI_HORZA 0x0c4
+# define VC4_HDMI_HORZA_VPOS BIT(14)
+# define VC4_HDMI_HORZA_HPOS BIT(13)
+/* Horizontal active pixels (hdisplay). */
+# define VC4_HDMI_HORZA_HAP_MASK VC4_MASK(12, 0)
+# define VC4_HDMI_HORZA_HAP_SHIFT 0
+
+#define VC4_HDMI_HORZB 0x0c8
+/* Horizontal pack porch (htotal - hsync_end). */
+# define VC4_HDMI_HORZB_HBP_MASK VC4_MASK(29, 20)
+# define VC4_HDMI_HORZB_HBP_SHIFT 20
+/* Horizontal sync pulse (hsync_end - hsync_start). */
+# define VC4_HDMI_HORZB_HSP_MASK VC4_MASK(19, 10)
+# define VC4_HDMI_HORZB_HSP_SHIFT 10
+/* Horizontal front porch (hsync_start - hdisplay). */
+# define VC4_HDMI_HORZB_HFP_MASK VC4_MASK(9, 0)
+# define VC4_HDMI_HORZB_HFP_SHIFT 0
+
+#define VC4_HDMI_FIFO_CTL 0x05c
+# define VC4_HDMI_FIFO_CTL_RECENTER_DONE BIT(14)
+# define VC4_HDMI_FIFO_CTL_USE_EMPTY BIT(13)
+# define VC4_HDMI_FIFO_CTL_ON_VB BIT(7)
+# define VC4_HDMI_FIFO_CTL_RECENTER BIT(6)
+# define VC4_HDMI_FIFO_CTL_FIFO_RESET BIT(5)
+# define VC4_HDMI_FIFO_CTL_USE_PLL_LOCK BIT(4)
+# define VC4_HDMI_FIFO_CTL_INV_CLK_XFR BIT(3)
+# define VC4_HDMI_FIFO_CTL_CAPTURE_PTR BIT(2)
+# define VC4_HDMI_FIFO_CTL_USE_FULL BIT(1)
+# define VC4_HDMI_FIFO_CTL_MASTER_SLAVE_N BIT(0)
+# define VC4_HDMI_FIFO_VALID_WRITE_MASK 0xefff
+
+#define VC4_HDMI_SCHEDULER_CONTROL 0x0c0
+# define VC4_HDMI_SCHEDULER_CONTROL_MANUAL_FORMAT BIT(15)
+# define VC4_HDMI_SCHEDULER_CONTROL_IGNORE_VSYNC_PREDICTS BIT(5)
+# define VC4_HDMI_SCHEDULER_CONTROL_VERT_ALWAYS_KEEPOUT BIT(3)
+# define VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE BIT(1)
+# define VC4_HDMI_SCHEDULER_CONTROL_MODE_HDMI BIT(0)
+
+#define VC4_HDMI_VERTA0 0x0cc
+#define VC4_HDMI_VERTA1 0x0d4
+/* Vertical sync pulse (vsync_end - vsync_start). */
+# define VC4_HDMI_VERTA_VSP_MASK VC4_MASK(24, 20)
+# define VC4_HDMI_VERTA_VSP_SHIFT 20
+/* Vertical front porch (vsync_start - vdisplay). */
+# define VC4_HDMI_VERTA_VFP_MASK VC4_MASK(19, 13)
+# define VC4_HDMI_VERTA_VFP_SHIFT 13
+/* Vertical active lines (vdisplay). */
+# define VC4_HDMI_VERTA_VAL_MASK VC4_MASK(12, 0)
+# define VC4_HDMI_VERTA_VAL_SHIFT 0
+
+#define VC4_HDMI_VERTB0 0x0d0
+#define VC4_HDMI_VERTB1 0x0d8
+/* Vertical sync pulse offset (for interlaced) */
+# define VC4_HDMI_VERTB_VSPO_MASK VC4_MASK(21, 9)
+# define VC4_HDMI_VERTB_VSPO_SHIFT 9
+/* Vertical pack porch (vtotal - vsync_end). */
+# define VC4_HDMI_VERTB_VBP_MASK VC4_MASK(8, 0)
+# define VC4_HDMI_VERTB_VBP_SHIFT 0
+
+#define VC4_HDMI_TX_PHY_RESET_CTL 0x2c0
+
+#define VC4_HD_M_CTL 0x00c
+# define VC4_HD_M_SW_RST BIT(2)
+# define VC4_HD_M_ENABLE BIT(0)
+
+#define VC4_HD_MAI_CTL 0x014
+
+#define VC4_HD_VID_CTL 0x038
+# define VC4_HD_VID_CTL_ENABLE BIT(31)
+# define VC4_HD_VID_CTL_UNDERFLOW_ENABLE BIT(30)
+# define VC4_HD_VID_CTL_FRAME_COUNTER_RESET BIT(29)
+# define VC4_HD_VID_CTL_VSYNC_LOW BIT(28)
+# define VC4_HD_VID_CTL_HSYNC_LOW BIT(27)
+
+#define VC4_HD_CSC_CTL 0x040
+# define VC4_HD_CSC_CTL_ORDER_MASK VC4_MASK(7, 5)
+# define VC4_HD_CSC_CTL_ORDER_SHIFT 5
+# define VC4_HD_CSC_CTL_ORDER_RGB 0
+# define VC4_HD_CSC_CTL_ORDER_BGR 1
+# define VC4_HD_CSC_CTL_ORDER_BRG 2
+# define VC4_HD_CSC_CTL_ORDER_GRB 3
+# define VC4_HD_CSC_CTL_ORDER_GBR 4
+# define VC4_HD_CSC_CTL_ORDER_RBG 5
+# define VC4_HD_CSC_CTL_PADMSB BIT(4)
+# define VC4_HD_CSC_CTL_MODE_MASK VC4_MASK(3, 2)
+# define VC4_HD_CSC_CTL_MODE_SHIFT 2
+# define VC4_HD_CSC_CTL_MODE_RGB_TO_SD_YPRPB 0
+# define VC4_HD_CSC_CTL_MODE_RGB_TO_HD_YPRPB 1
+# define VC4_HD_CSC_CTL_MODE_CUSTOM 2
+# define VC4_HD_CSC_CTL_RGB2YCC BIT(1)
+# define VC4_HD_CSC_CTL_ENABLE BIT(0)
+
+#define VC4_HD_FRAME_COUNT 0x068
+
+/* HVS display list information. */
+#define HVS_BOOTLOADER_DLIST_END 32
+
+enum hvs_pixel_format {
+ /* 8bpp */
+ HVS_PIXEL_FORMAT_RGB332 = 0,
+ /* 16bpp */
+ HVS_PIXEL_FORMAT_RGBA4444 = 1,
+ HVS_PIXEL_FORMAT_RGB555 = 2,
+ HVS_PIXEL_FORMAT_RGBA5551 = 3,
+ HVS_PIXEL_FORMAT_RGB565 = 4,
+ /* 24bpp */
+ HVS_PIXEL_FORMAT_RGB888 = 5,
+ HVS_PIXEL_FORMAT_RGBA6666 = 6,
+ /* 32bpp */
+ HVS_PIXEL_FORMAT_RGBA8888 = 7
+};
+
+/* Note: the LSB is the rightmost character shown. Only valid for
+ * HVS_PIXEL_FORMAT_RGB8888, not RGB888.
+ */
+#define HVS_PIXEL_ORDER_RGBA 0
+#define HVS_PIXEL_ORDER_BGRA 1
+#define HVS_PIXEL_ORDER_ARGB 2
+#define HVS_PIXEL_ORDER_ABGR 3
+
+#define HVS_PIXEL_ORDER_XBRG 0
+#define HVS_PIXEL_ORDER_XRBG 1
+#define HVS_PIXEL_ORDER_XRGB 2
+#define HVS_PIXEL_ORDER_XBGR 3
+
+#define HVS_PIXEL_ORDER_XYCBCR 0
+#define HVS_PIXEL_ORDER_XYCRCB 1
+#define HVS_PIXEL_ORDER_YXCBCR 2
+#define HVS_PIXEL_ORDER_YXCRCB 3
+
+#define SCALER_CTL0_END BIT(31)
+#define SCALER_CTL0_VALID BIT(30)
+
+#define SCALER_CTL0_SIZE_MASK VC4_MASK(29, 24)
+#define SCALER_CTL0_SIZE_SHIFT 24
+
+#define SCALER_CTL0_HFLIP BIT(16)
+#define SCALER_CTL0_VFLIP BIT(15)
+
+#define SCALER_CTL0_ORDER_MASK VC4_MASK(14, 13)
+#define SCALER_CTL0_ORDER_SHIFT 13
+
+/* Set to indicate no scaling. */
+#define SCALER_CTL0_UNITY BIT(4)
+
+#define SCALER_CTL0_PIXEL_FORMAT_MASK VC4_MASK(3, 0)
+#define SCALER_CTL0_PIXEL_FORMAT_SHIFT 0
+
+#define SCALER_POS0_FIXED_ALPHA_MASK VC4_MASK(31, 24)
+#define SCALER_POS0_FIXED_ALPHA_SHIFT 24
+
+#define SCALER_POS0_START_Y_MASK VC4_MASK(23, 12)
+#define SCALER_POS0_START_Y_SHIFT 12
+
+#define SCALER_POS0_START_X_MASK VC4_MASK(11, 0)
+#define SCALER_POS0_START_X_SHIFT 0
+
+#define SCALER_POS2_ALPHA_MODE_MASK VC4_MASK(31, 30)
+#define SCALER_POS2_ALPHA_MODE_SHIFT 30
+#define SCALER_POS2_ALPHA_MODE_PIPELINE 0
+#define SCALER_POS2_ALPHA_MODE_FIXED 1
+#define SCALER_POS2_ALPHA_MODE_FIXED_NONZERO 2
+#define SCALER_POS2_ALPHA_MODE_FIXED_OVER_0x07 3
+
+#define SCALER_POS2_HEIGHT_MASK VC4_MASK(27, 16)
+#define SCALER_POS2_HEIGHT_SHIFT 16
+
+#define SCALER_POS2_WIDTH_MASK VC4_MASK(11, 0)
+#define SCALER_POS2_WIDTH_SHIFT 0
+
+#define SCALER_SRC_PITCH_MASK VC4_MASK(15, 0)
+#define SCALER_SRC_PITCH_SHIFT 0
+
+#endif /* VC4_REGS_H */
return ret;
}
-int vgem_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
-{
- struct drm_file *priv = filp->private_data;
- struct drm_device *dev = priv->minor->dev;
- struct drm_vma_offset_node *node;
- struct drm_gem_object *obj;
- struct drm_vgem_gem_object *vgem_obj;
- int ret = 0;
-
- mutex_lock(&dev->struct_mutex);
-
- node = drm_vma_offset_exact_lookup(dev->vma_offset_manager,
- vma->vm_pgoff,
- vma_pages(vma));
- if (!node) {
- ret = -EINVAL;
- goto out_unlock;
- } else if (!drm_vma_node_is_allowed(node, filp)) {
- ret = -EACCES;
- goto out_unlock;
- }
-
- obj = container_of(node, struct drm_gem_object, vma_node);
-
- vgem_obj = to_vgem_bo(obj);
-
- if (obj->dma_buf && vgem_obj->use_dma_buf) {
- ret = dma_buf_mmap(obj->dma_buf, vma, 0);
- goto out_unlock;
- }
-
- if (!obj->dev->driver->gem_vm_ops) {
- ret = -EINVAL;
- goto out_unlock;
- }
-
- vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP;
- vma->vm_ops = obj->dev->driver->gem_vm_ops;
- vma->vm_private_data = vgem_obj;
- vma->vm_page_prot =
- pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
-
- mutex_unlock(&dev->struct_mutex);
- drm_gem_vm_open(vma);
- return ret;
-
-out_unlock:
- mutex_unlock(&dev->struct_mutex);
-
- return ret;
-}
-
-
static struct drm_ioctl_desc vgem_ioctls[] = {
};
static const struct file_operations vgem_driver_fops = {
.owner = THIS_MODULE,
.open = drm_open,
- .mmap = vgem_drm_gem_mmap,
+ .mmap = drm_gem_mmap,
.poll = drm_poll,
.read = drm_read,
.unlocked_ioctl = drm_ioctl,
uint32_t dma_diff;
} drm_via_private_t;
+struct via_file_private {
+ struct list_head obj_list;
+};
+
enum via_family {
VIA_OTHER = 0, /* Baseline */
VIA_PRO_GROUP_A, /* Another video engine and DMA commands */
extern int via_final_context(struct drm_device *dev, int context);
extern int via_do_cleanup_map(struct drm_device *dev);
-extern u32 via_get_vblank_counter(struct drm_device *dev, int crtc);
-extern int via_enable_vblank(struct drm_device *dev, int crtc);
-extern void via_disable_vblank(struct drm_device *dev, int crtc);
+extern u32 via_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
+extern int via_enable_vblank(struct drm_device *dev, unsigned int pipe);
+extern void via_disable_vblank(struct drm_device *dev, unsigned int pipe);
extern irqreturn_t via_driver_irq_handler(int irq, void *arg);
extern void via_driver_irq_preinstall(struct drm_device *dev);
1000000 - (then->tv_usec - now->tv_usec);
}
-u32 via_get_vblank_counter(struct drm_device *dev, int crtc)
+u32 via_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
drm_via_private_t *dev_priv = dev->dev_private;
- if (crtc != 0)
+
+ if (pipe != 0)
return 0;
return atomic_read(&dev_priv->vbl_received);
}
}
-int via_enable_vblank(struct drm_device *dev, int crtc)
+int via_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
drm_via_private_t *dev_priv = dev->dev_private;
u32 status;
- if (crtc != 0) {
- DRM_ERROR("%s: bad crtc %d\n", __func__, crtc);
+ if (pipe != 0) {
+ DRM_ERROR("%s: bad crtc %u\n", __func__, pipe);
return -EINVAL;
}
return 0;
}
-void via_disable_vblank(struct drm_device *dev, int crtc)
+void via_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
drm_via_private_t *dev_priv = dev->dev_private;
u32 status;
VIA_WRITE8(0x83d4, 0x11);
VIA_WRITE8(0x83d5, VIA_READ8(0x83d5) & ~0x30);
- if (crtc != 0)
- DRM_ERROR("%s: bad crtc %d\n", __func__, crtc);
+ if (pipe != 0)
+ DRM_ERROR("%s: bad crtc %u\n", __func__, pipe);
}
static int
virtio-gpu-y := virtgpu_drv.o virtgpu_kms.o virtgpu_drm_bus.o virtgpu_gem.o \
virtgpu_fb.o virtgpu_display.o virtgpu_vq.o virtgpu_ttm.o \
- virtgpu_fence.o virtgpu_object.o virtgpu_debugfs.o virtgpu_plane.o
+ virtgpu_fence.o virtgpu_object.o virtgpu_debugfs.o virtgpu_plane.o \
+ virtgpu_ioctl.o virtgpu_prime.o
obj-$(CONFIG_DRM_VIRTIO_GPU) += virtio-gpu.o
cpu_to_le32(64),
cpu_to_le32(64),
0, 0, &fence);
+ ret = virtio_gpu_object_reserve(qobj, false);
+ if (!ret) {
+ reservation_object_add_excl_fence(qobj->tbo.resv,
+ &fence->f);
+ fence_put(&fence->f);
+ virtio_gpu_object_unreserve(qobj);
+ virtio_gpu_object_wait(qobj, false);
+ }
output->cursor.hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_UPDATE_CURSOR);
output->cursor.resource_id = cpu_to_le32(qobj->hw_res_handle);
return 0;
}
+static int virtio_gpu_page_flip(struct drm_crtc *crtc,
+ struct drm_framebuffer *fb,
+ struct drm_pending_vblank_event *event,
+ uint32_t flags)
+{
+ struct virtio_gpu_device *vgdev = crtc->dev->dev_private;
+ struct virtio_gpu_output *output =
+ container_of(crtc, struct virtio_gpu_output, crtc);
+ struct drm_plane *plane = crtc->primary;
+ struct virtio_gpu_framebuffer *vgfb;
+ struct virtio_gpu_object *bo;
+ unsigned long irqflags;
+ uint32_t handle;
+
+ plane->fb = fb;
+ vgfb = to_virtio_gpu_framebuffer(plane->fb);
+ bo = gem_to_virtio_gpu_obj(vgfb->obj);
+ handle = bo->hw_res_handle;
+
+ DRM_DEBUG("handle 0x%x%s, crtc %dx%d\n", handle,
+ bo->dumb ? ", dumb" : "",
+ crtc->mode.hdisplay, crtc->mode.vdisplay);
+ if (bo->dumb) {
+ virtio_gpu_cmd_transfer_to_host_2d
+ (vgdev, handle, 0,
+ cpu_to_le32(crtc->mode.hdisplay),
+ cpu_to_le32(crtc->mode.vdisplay),
+ 0, 0, NULL);
+ }
+ virtio_gpu_cmd_set_scanout(vgdev, output->index, handle,
+ crtc->mode.hdisplay,
+ crtc->mode.vdisplay, 0, 0);
+ virtio_gpu_cmd_resource_flush(vgdev, handle, 0, 0,
+ crtc->mode.hdisplay,
+ crtc->mode.vdisplay);
+
+ if (event) {
+ spin_lock_irqsave(&crtc->dev->event_lock, irqflags);
+ drm_send_vblank_event(crtc->dev, -1, event);
+ spin_unlock_irqrestore(&crtc->dev->event_lock, irqflags);
+ }
+
+ return 0;
+}
+
static const struct drm_crtc_funcs virtio_gpu_crtc_funcs = {
.cursor_set2 = virtio_gpu_crtc_cursor_set,
.cursor_move = virtio_gpu_crtc_cursor_move,
.set_config = drm_atomic_helper_set_config,
.destroy = drm_crtc_cleanup,
-#if 0 /* not (yet) working without vblank support according to docs */
- .page_flip = drm_atomic_helper_page_flip,
-#endif
+ .page_flip = virtio_gpu_page_flip,
.reset = drm_atomic_helper_crtc_reset,
.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
};
static unsigned int features[] = {
+#ifdef __LITTLE_ENDIAN
+ /*
+ * Gallium command stream send by virgl is native endian.
+ * Because of that we only support little endian guests on
+ * little endian hosts.
+ */
+ VIRTIO_GPU_F_VIRGL,
+#endif
};
static struct virtio_driver virtio_gpu_driver = {
.feature_table = features,
static struct drm_driver driver = {
- .driver_features = DRIVER_MODESET | DRIVER_GEM,
+ .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
.set_busid = drm_virtio_set_busid,
.load = virtio_gpu_driver_load,
.unload = virtio_gpu_driver_unload,
+ .open = virtio_gpu_driver_open,
+ .postclose = virtio_gpu_driver_postclose,
.dumb_create = virtio_gpu_mode_dumb_create,
.dumb_map_offset = virtio_gpu_mode_dumb_mmap,
.debugfs_init = virtio_gpu_debugfs_init,
.debugfs_cleanup = virtio_gpu_debugfs_takedown,
#endif
+ .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+ .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+ .gem_prime_export = drm_gem_prime_export,
+ .gem_prime_import = drm_gem_prime_import,
+ .gem_prime_pin = virtgpu_gem_prime_pin,
+ .gem_prime_unpin = virtgpu_gem_prime_unpin,
+ .gem_prime_get_sg_table = virtgpu_gem_prime_get_sg_table,
+ .gem_prime_import_sg_table = virtgpu_gem_prime_import_sg_table,
+ .gem_prime_vmap = virtgpu_gem_prime_vmap,
+ .gem_prime_vunmap = virtgpu_gem_prime_vunmap,
+ .gem_prime_mmap = virtgpu_gem_prime_mmap,
.gem_free_object = virtio_gpu_gem_free_object,
+ .gem_open_object = virtio_gpu_gem_object_open,
+ .gem_close_object = virtio_gpu_gem_object_close,
.fops = &virtio_gpu_driver_fops,
+ .ioctls = virtio_gpu_ioctls,
+ .num_ioctls = DRM_VIRTIO_NUM_IOCTLS,
+
.name = DRIVER_NAME,
.desc = DRIVER_DESC,
.date = DRIVER_DATE,
struct work_struct dequeue_work;
};
+struct virtio_gpu_drv_capset {
+ uint32_t id;
+ uint32_t max_version;
+ uint32_t max_size;
+};
+
+struct virtio_gpu_drv_cap_cache {
+ struct list_head head;
+ void *caps_cache;
+ uint32_t id;
+ uint32_t version;
+ uint32_t size;
+ atomic_t is_valid;
+};
+
struct virtio_gpu_device {
struct device *dev;
struct drm_device *ddev;
struct idr ctx_id_idr;
spinlock_t ctx_id_idr_lock;
+ bool has_virgl_3d;
+
struct work_struct config_changed_work;
+
+ struct virtio_gpu_drv_capset *capsets;
+ uint32_t num_capsets;
+ struct list_head cap_cache;
};
struct virtio_gpu_fpriv {
/* virtio_kms.c */
int virtio_gpu_driver_load(struct drm_device *dev, unsigned long flags);
int virtio_gpu_driver_unload(struct drm_device *dev);
+int virtio_gpu_driver_open(struct drm_device *dev, struct drm_file *file);
+void virtio_gpu_driver_postclose(struct drm_device *dev, struct drm_file *file);
/* virtio_gem.c */
void virtio_gpu_gem_free_object(struct drm_gem_object *gem_obj);
uint64_t size,
struct drm_gem_object **obj_p,
uint32_t *handle_p);
+int virtio_gpu_gem_object_open(struct drm_gem_object *obj,
+ struct drm_file *file);
+void virtio_gpu_gem_object_close(struct drm_gem_object *obj,
+ struct drm_file *file);
struct virtio_gpu_object *virtio_gpu_alloc_object(struct drm_device *dev,
size_t size, bool kernel,
bool pinned);
int virtio_gpu_cmd_get_display_info(struct virtio_gpu_device *vgdev);
void virtio_gpu_cmd_resource_inval_backing(struct virtio_gpu_device *vgdev,
uint32_t resource_id);
+int virtio_gpu_cmd_get_capset_info(struct virtio_gpu_device *vgdev, int idx);
+int virtio_gpu_cmd_get_capset(struct virtio_gpu_device *vgdev,
+ int idx, int version,
+ struct virtio_gpu_drv_cap_cache **cache_p);
+void virtio_gpu_cmd_context_create(struct virtio_gpu_device *vgdev, uint32_t id,
+ uint32_t nlen, const char *name);
+void virtio_gpu_cmd_context_destroy(struct virtio_gpu_device *vgdev,
+ uint32_t id);
+void virtio_gpu_cmd_context_attach_resource(struct virtio_gpu_device *vgdev,
+ uint32_t ctx_id,
+ uint32_t resource_id);
+void virtio_gpu_cmd_context_detach_resource(struct virtio_gpu_device *vgdev,
+ uint32_t ctx_id,
+ uint32_t resource_id);
+void virtio_gpu_cmd_submit(struct virtio_gpu_device *vgdev,
+ void *data, uint32_t data_size,
+ uint32_t ctx_id, struct virtio_gpu_fence **fence);
+void virtio_gpu_cmd_transfer_from_host_3d(struct virtio_gpu_device *vgdev,
+ uint32_t resource_id, uint32_t ctx_id,
+ uint64_t offset, uint32_t level,
+ struct virtio_gpu_box *box,
+ struct virtio_gpu_fence **fence);
+void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev,
+ uint32_t resource_id, uint32_t ctx_id,
+ uint64_t offset, uint32_t level,
+ struct virtio_gpu_box *box,
+ struct virtio_gpu_fence **fence);
+void
+virtio_gpu_cmd_resource_create_3d(struct virtio_gpu_device *vgdev,
+ struct virtio_gpu_resource_create_3d *rc_3d,
+ struct virtio_gpu_fence **fence);
void virtio_gpu_ctrl_ack(struct virtqueue *vq);
void virtio_gpu_cursor_ack(struct virtqueue *vq);
+void virtio_gpu_fence_ack(struct virtqueue *vq);
void virtio_gpu_dequeue_ctrl_func(struct work_struct *work);
void virtio_gpu_dequeue_cursor_func(struct work_struct *work);
+void virtio_gpu_dequeue_fence_func(struct work_struct *work);
/* virtio_gpu_display.c */
int virtio_gpu_framebuffer_init(struct drm_device *dev,
void virtio_gpu_object_free_sg_table(struct virtio_gpu_object *bo);
int virtio_gpu_object_wait(struct virtio_gpu_object *bo, bool no_wait);
+/* virtgpu_prime.c */
+int virtgpu_gem_prime_pin(struct drm_gem_object *obj);
+void virtgpu_gem_prime_unpin(struct drm_gem_object *obj);
+struct sg_table *virtgpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
+struct drm_gem_object *virtgpu_gem_prime_import_sg_table(
+ struct drm_device *dev, struct dma_buf_attachment *attach,
+ struct sg_table *sgt);
+void *virtgpu_gem_prime_vmap(struct drm_gem_object *obj);
+void virtgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+int virtgpu_gem_prime_mmap(struct drm_gem_object *obj,
+ struct vm_area_struct *vma);
+
static inline struct virtio_gpu_object*
virtio_gpu_object_ref(struct virtio_gpu_object *bo)
{
struct virtio_gpu_fence_driver *drv = &vgdev->fence_drv;
unsigned long irq_flags;
- *fence = kmalloc(sizeof(struct virtio_gpu_fence), GFP_KERNEL);
+ *fence = kmalloc(sizeof(struct virtio_gpu_fence), GFP_ATOMIC);
if ((*fence) == NULL)
return -ENOMEM;
drm_gem_object_unreference_unlocked(gobj);
return 0;
}
+
+int virtio_gpu_gem_object_open(struct drm_gem_object *obj,
+ struct drm_file *file)
+{
+ struct virtio_gpu_device *vgdev = obj->dev->dev_private;
+ struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
+ struct virtio_gpu_object *qobj = gem_to_virtio_gpu_obj(obj);
+ int r;
+
+ if (!vgdev->has_virgl_3d)
+ return 0;
+
+ r = virtio_gpu_object_reserve(qobj, false);
+ if (r)
+ return r;
+
+ virtio_gpu_cmd_context_attach_resource(vgdev, vfpriv->ctx_id,
+ qobj->hw_res_handle);
+ virtio_gpu_object_unreserve(qobj);
+ return 0;
+}
+
+void virtio_gpu_gem_object_close(struct drm_gem_object *obj,
+ struct drm_file *file)
+{
+ struct virtio_gpu_device *vgdev = obj->dev->dev_private;
+ struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
+ struct virtio_gpu_object *qobj = gem_to_virtio_gpu_obj(obj);
+ int r;
+
+ if (!vgdev->has_virgl_3d)
+ return;
+
+ r = virtio_gpu_object_reserve(qobj, false);
+ if (r)
+ return;
+
+ virtio_gpu_cmd_context_detach_resource(vgdev, vfpriv->ctx_id,
+ qobj->hw_res_handle);
+ virtio_gpu_object_unreserve(qobj);
+}
--- /dev/null
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ * All Rights Reserved.
+ *
+ * Authors:
+ * Dave Airlie
+ * Alon Levy
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <drm/drmP.h>
+#include "virtgpu_drv.h"
+#include <drm/virtgpu_drm.h>
+#include "ttm/ttm_execbuf_util.h"
+
+static void convert_to_hw_box(struct virtio_gpu_box *dst,
+ const struct drm_virtgpu_3d_box *src)
+{
+ dst->x = cpu_to_le32(src->x);
+ dst->y = cpu_to_le32(src->y);
+ dst->z = cpu_to_le32(src->z);
+ dst->w = cpu_to_le32(src->w);
+ dst->h = cpu_to_le32(src->h);
+ dst->d = cpu_to_le32(src->d);
+}
+
+static int virtio_gpu_map_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file_priv)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct drm_virtgpu_map *virtio_gpu_map = data;
+
+ return virtio_gpu_mode_dumb_mmap(file_priv, vgdev->ddev,
+ virtio_gpu_map->handle,
+ &virtio_gpu_map->offset);
+}
+
+static int virtio_gpu_object_list_validate(struct ww_acquire_ctx *ticket,
+ struct list_head *head)
+{
+ struct ttm_validate_buffer *buf;
+ struct ttm_buffer_object *bo;
+ struct virtio_gpu_object *qobj;
+ int ret;
+
+ ret = ttm_eu_reserve_buffers(ticket, head, true, NULL);
+ if (ret != 0)
+ return ret;
+
+ list_for_each_entry(buf, head, head) {
+ bo = buf->bo;
+ qobj = container_of(bo, struct virtio_gpu_object, tbo);
+ ret = ttm_bo_validate(bo, &qobj->placement, false, false);
+ if (ret) {
+ ttm_eu_backoff_reservation(ticket, head);
+ return ret;
+ }
+ }
+ return 0;
+}
+
+static void virtio_gpu_unref_list(struct list_head *head)
+{
+ struct ttm_validate_buffer *buf;
+ struct ttm_buffer_object *bo;
+ struct virtio_gpu_object *qobj;
+ list_for_each_entry(buf, head, head) {
+ bo = buf->bo;
+ qobj = container_of(bo, struct virtio_gpu_object, tbo);
+
+ drm_gem_object_unreference_unlocked(&qobj->gem_base);
+ }
+}
+
+static int virtio_gpu_execbuffer(struct drm_device *dev,
+ struct drm_virtgpu_execbuffer *exbuf,
+ struct drm_file *drm_file)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct virtio_gpu_fpriv *vfpriv = drm_file->driver_priv;
+ struct drm_gem_object *gobj;
+ struct virtio_gpu_fence *fence;
+ struct virtio_gpu_object *qobj;
+ int ret;
+ uint32_t *bo_handles = NULL;
+ void __user *user_bo_handles = NULL;
+ struct list_head validate_list;
+ struct ttm_validate_buffer *buflist = NULL;
+ int i;
+ struct ww_acquire_ctx ticket;
+ void *buf;
+
+ if (vgdev->has_virgl_3d == false)
+ return -ENOSYS;
+
+ INIT_LIST_HEAD(&validate_list);
+ if (exbuf->num_bo_handles) {
+
+ bo_handles = drm_malloc_ab(exbuf->num_bo_handles,
+ sizeof(uint32_t));
+ buflist = drm_calloc_large(exbuf->num_bo_handles,
+ sizeof(struct ttm_validate_buffer));
+ if (!bo_handles || !buflist) {
+ drm_free_large(bo_handles);
+ drm_free_large(buflist);
+ return -ENOMEM;
+ }
+
+ user_bo_handles = (void __user *)(uintptr_t)exbuf->bo_handles;
+ if (copy_from_user(bo_handles, user_bo_handles,
+ exbuf->num_bo_handles * sizeof(uint32_t))) {
+ ret = -EFAULT;
+ drm_free_large(bo_handles);
+ drm_free_large(buflist);
+ return ret;
+ }
+
+ for (i = 0; i < exbuf->num_bo_handles; i++) {
+ gobj = drm_gem_object_lookup(dev,
+ drm_file, bo_handles[i]);
+ if (!gobj) {
+ drm_free_large(bo_handles);
+ drm_free_large(buflist);
+ return -ENOENT;
+ }
+
+ qobj = gem_to_virtio_gpu_obj(gobj);
+ buflist[i].bo = &qobj->tbo;
+
+ list_add(&buflist[i].head, &validate_list);
+ }
+ drm_free_large(bo_handles);
+ }
+
+ ret = virtio_gpu_object_list_validate(&ticket, &validate_list);
+ if (ret)
+ goto out_free;
+
+ buf = kmalloc(exbuf->size, GFP_KERNEL);
+ if (!buf) {
+ ret = -ENOMEM;
+ goto out_unresv;
+ }
+ if (copy_from_user(buf, (void __user *)(uintptr_t)exbuf->command,
+ exbuf->size)) {
+ kfree(buf);
+ ret = -EFAULT;
+ goto out_unresv;
+ }
+ virtio_gpu_cmd_submit(vgdev, buf, exbuf->size,
+ vfpriv->ctx_id, &fence);
+
+ ttm_eu_fence_buffer_objects(&ticket, &validate_list, &fence->f);
+
+ /* fence the command bo */
+ virtio_gpu_unref_list(&validate_list);
+ drm_free_large(buflist);
+ fence_put(&fence->f);
+ return 0;
+
+out_unresv:
+ ttm_eu_backoff_reservation(&ticket, &validate_list);
+out_free:
+ virtio_gpu_unref_list(&validate_list);
+ drm_free_large(buflist);
+ return ret;
+}
+
+/*
+ * Usage of execbuffer:
+ * Relocations need to take into account the full VIRTIO_GPUDrawable size.
+ * However, the command as passed from user space must *not* contain the initial
+ * VIRTIO_GPUReleaseInfo struct (first XXX bytes)
+ */
+static int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file_priv)
+{
+ struct drm_virtgpu_execbuffer *execbuffer = data;
+ return virtio_gpu_execbuffer(dev, execbuffer, file_priv);
+}
+
+
+static int virtio_gpu_getparam_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file_priv)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct drm_virtgpu_getparam *param = data;
+ int value;
+
+ switch (param->param) {
+ case VIRTGPU_PARAM_3D_FEATURES:
+ value = vgdev->has_virgl_3d == true ? 1 : 0;
+ break;
+ default:
+ return -EINVAL;
+ }
+ if (copy_to_user((void __user *)(unsigned long)param->value,
+ &value, sizeof(int))) {
+ return -EFAULT;
+ }
+ return 0;
+}
+
+static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file_priv)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct drm_virtgpu_resource_create *rc = data;
+ int ret;
+ uint32_t res_id;
+ struct virtio_gpu_object *qobj;
+ struct drm_gem_object *obj;
+ uint32_t handle = 0;
+ uint32_t size;
+ struct list_head validate_list;
+ struct ttm_validate_buffer mainbuf;
+ struct virtio_gpu_fence *fence = NULL;
+ struct ww_acquire_ctx ticket;
+ struct virtio_gpu_resource_create_3d rc_3d;
+
+ if (vgdev->has_virgl_3d == false) {
+ if (rc->depth > 1)
+ return -EINVAL;
+ if (rc->nr_samples > 1)
+ return -EINVAL;
+ if (rc->last_level > 1)
+ return -EINVAL;
+ if (rc->target != 2)
+ return -EINVAL;
+ if (rc->array_size > 1)
+ return -EINVAL;
+ }
+
+ INIT_LIST_HEAD(&validate_list);
+ memset(&mainbuf, 0, sizeof(struct ttm_validate_buffer));
+
+ virtio_gpu_resource_id_get(vgdev, &res_id);
+
+ size = rc->size;
+
+ /* allocate a single page size object */
+ if (size == 0)
+ size = PAGE_SIZE;
+
+ qobj = virtio_gpu_alloc_object(dev, size, false, false);
+ if (IS_ERR(qobj)) {
+ ret = PTR_ERR(qobj);
+ goto fail_id;
+ }
+ obj = &qobj->gem_base;
+
+ if (!vgdev->has_virgl_3d) {
+ virtio_gpu_cmd_create_resource(vgdev, res_id, rc->format,
+ rc->width, rc->height);
+
+ ret = virtio_gpu_object_attach(vgdev, qobj, res_id, NULL);
+ } else {
+ /* use a gem reference since unref list undoes them */
+ drm_gem_object_reference(&qobj->gem_base);
+ mainbuf.bo = &qobj->tbo;
+ list_add(&mainbuf.head, &validate_list);
+
+ ret = virtio_gpu_object_list_validate(&ticket, &validate_list);
+ if (ret) {
+ DRM_DEBUG("failed to validate\n");
+ goto fail_unref;
+ }
+
+ rc_3d.resource_id = cpu_to_le32(res_id);
+ rc_3d.target = cpu_to_le32(rc->target);
+ rc_3d.format = cpu_to_le32(rc->format);
+ rc_3d.bind = cpu_to_le32(rc->bind);
+ rc_3d.width = cpu_to_le32(rc->width);
+ rc_3d.height = cpu_to_le32(rc->height);
+ rc_3d.depth = cpu_to_le32(rc->depth);
+ rc_3d.array_size = cpu_to_le32(rc->array_size);
+ rc_3d.last_level = cpu_to_le32(rc->last_level);
+ rc_3d.nr_samples = cpu_to_le32(rc->nr_samples);
+ rc_3d.flags = cpu_to_le32(rc->flags);
+
+ virtio_gpu_cmd_resource_create_3d(vgdev, &rc_3d, NULL);
+ ret = virtio_gpu_object_attach(vgdev, qobj, res_id, &fence);
+ if (ret) {
+ ttm_eu_backoff_reservation(&ticket, &validate_list);
+ goto fail_unref;
+ }
+ ttm_eu_fence_buffer_objects(&ticket, &validate_list, &fence->f);
+ }
+
+ qobj->hw_res_handle = res_id;
+
+ ret = drm_gem_handle_create(file_priv, obj, &handle);
+ if (ret) {
+
+ drm_gem_object_release(obj);
+ if (vgdev->has_virgl_3d) {
+ virtio_gpu_unref_list(&validate_list);
+ fence_put(&fence->f);
+ }
+ return ret;
+ }
+ drm_gem_object_unreference_unlocked(obj);
+
+ rc->res_handle = res_id; /* similiar to a VM address */
+ rc->bo_handle = handle;
+
+ if (vgdev->has_virgl_3d) {
+ virtio_gpu_unref_list(&validate_list);
+ fence_put(&fence->f);
+ }
+ return 0;
+fail_unref:
+ if (vgdev->has_virgl_3d) {
+ virtio_gpu_unref_list(&validate_list);
+ fence_put(&fence->f);
+ }
+//fail_obj:
+// drm_gem_object_handle_unreference_unlocked(obj);
+fail_id:
+ virtio_gpu_resource_id_put(vgdev, res_id);
+ return ret;
+}
+
+static int virtio_gpu_resource_info_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file_priv)
+{
+ struct drm_virtgpu_resource_info *ri = data;
+ struct drm_gem_object *gobj = NULL;
+ struct virtio_gpu_object *qobj = NULL;
+
+ gobj = drm_gem_object_lookup(dev, file_priv, ri->bo_handle);
+ if (gobj == NULL)
+ return -ENOENT;
+
+ qobj = gem_to_virtio_gpu_obj(gobj);
+
+ ri->size = qobj->gem_base.size;
+ ri->res_handle = qobj->hw_res_handle;
+ drm_gem_object_unreference_unlocked(gobj);
+ return 0;
+}
+
+static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *file)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
+ struct drm_virtgpu_3d_transfer_from_host *args = data;
+ struct drm_gem_object *gobj = NULL;
+ struct virtio_gpu_object *qobj = NULL;
+ struct virtio_gpu_fence *fence;
+ int ret;
+ u32 offset = args->offset;
+ struct virtio_gpu_box box;
+
+ if (vgdev->has_virgl_3d == false)
+ return -ENOSYS;
+
+ gobj = drm_gem_object_lookup(dev, file, args->bo_handle);
+ if (gobj == NULL)
+ return -ENOENT;
+
+ qobj = gem_to_virtio_gpu_obj(gobj);
+
+ ret = virtio_gpu_object_reserve(qobj, false);
+ if (ret)
+ goto out;
+
+ ret = ttm_bo_validate(&qobj->tbo, &qobj->placement,
+ true, false);
+ if (unlikely(ret))
+ goto out_unres;
+
+ convert_to_hw_box(&box, &args->box);
+ virtio_gpu_cmd_transfer_from_host_3d
+ (vgdev, qobj->hw_res_handle,
+ vfpriv->ctx_id, offset, args->level,
+ &box, &fence);
+ reservation_object_add_excl_fence(qobj->tbo.resv,
+ &fence->f);
+
+ fence_put(&fence->f);
+out_unres:
+ virtio_gpu_object_unreserve(qobj);
+out:
+ drm_gem_object_unreference_unlocked(gobj);
+ return ret;
+}
+
+static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
+ struct drm_virtgpu_3d_transfer_to_host *args = data;
+ struct drm_gem_object *gobj = NULL;
+ struct virtio_gpu_object *qobj = NULL;
+ struct virtio_gpu_fence *fence;
+ struct virtio_gpu_box box;
+ int ret;
+ u32 offset = args->offset;
+
+ gobj = drm_gem_object_lookup(dev, file, args->bo_handle);
+ if (gobj == NULL)
+ return -ENOENT;
+
+ qobj = gem_to_virtio_gpu_obj(gobj);
+
+ ret = virtio_gpu_object_reserve(qobj, false);
+ if (ret)
+ goto out;
+
+ ret = ttm_bo_validate(&qobj->tbo, &qobj->placement,
+ true, false);
+ if (unlikely(ret))
+ goto out_unres;
+
+ convert_to_hw_box(&box, &args->box);
+ if (!vgdev->has_virgl_3d) {
+ virtio_gpu_cmd_transfer_to_host_2d
+ (vgdev, qobj->hw_res_handle, offset,
+ box.w, box.h, box.x, box.y, NULL);
+ } else {
+ virtio_gpu_cmd_transfer_to_host_3d
+ (vgdev, qobj->hw_res_handle,
+ vfpriv ? vfpriv->ctx_id : 0, offset,
+ args->level, &box, &fence);
+ reservation_object_add_excl_fence(qobj->tbo.resv,
+ &fence->f);
+ fence_put(&fence->f);
+ }
+
+out_unres:
+ virtio_gpu_object_unreserve(qobj);
+out:
+ drm_gem_object_unreference_unlocked(gobj);
+ return ret;
+}
+
+static int virtio_gpu_wait_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file)
+{
+ struct drm_virtgpu_3d_wait *args = data;
+ struct drm_gem_object *gobj = NULL;
+ struct virtio_gpu_object *qobj = NULL;
+ int ret;
+ bool nowait = false;
+
+ gobj = drm_gem_object_lookup(dev, file, args->handle);
+ if (gobj == NULL)
+ return -ENOENT;
+
+ qobj = gem_to_virtio_gpu_obj(gobj);
+
+ if (args->flags & VIRTGPU_WAIT_NOWAIT)
+ nowait = true;
+ ret = virtio_gpu_object_wait(qobj, nowait);
+
+ drm_gem_object_unreference_unlocked(gobj);
+ return ret;
+}
+
+static int virtio_gpu_get_caps_ioctl(struct drm_device *dev,
+ void *data, struct drm_file *file)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct drm_virtgpu_get_caps *args = data;
+ int size;
+ int i;
+ int found_valid = -1;
+ int ret;
+ struct virtio_gpu_drv_cap_cache *cache_ent;
+ void *ptr;
+ if (vgdev->num_capsets == 0)
+ return -ENOSYS;
+
+ spin_lock(&vgdev->display_info_lock);
+ for (i = 0; i < vgdev->num_capsets; i++) {
+ if (vgdev->capsets[i].id == args->cap_set_id) {
+ if (vgdev->capsets[i].max_version >= args->cap_set_ver) {
+ found_valid = i;
+ break;
+ }
+ }
+ }
+
+ if (found_valid == -1) {
+ spin_unlock(&vgdev->display_info_lock);
+ return -EINVAL;
+ }
+
+ size = vgdev->capsets[found_valid].max_size;
+ if (args->size > size) {
+ spin_unlock(&vgdev->display_info_lock);
+ return -EINVAL;
+ }
+
+ list_for_each_entry(cache_ent, &vgdev->cap_cache, head) {
+ if (cache_ent->id == args->cap_set_id &&
+ cache_ent->version == args->cap_set_ver) {
+ ptr = cache_ent->caps_cache;
+ spin_unlock(&vgdev->display_info_lock);
+ goto copy_exit;
+ }
+ }
+ spin_unlock(&vgdev->display_info_lock);
+
+ /* not in cache - need to talk to hw */
+ virtio_gpu_cmd_get_capset(vgdev, found_valid, args->cap_set_ver,
+ &cache_ent);
+
+ ret = wait_event_timeout(vgdev->resp_wq,
+ atomic_read(&cache_ent->is_valid), 5 * HZ);
+
+ ptr = cache_ent->caps_cache;
+
+copy_exit:
+ if (copy_to_user((void __user *)(unsigned long)args->addr, ptr, size))
+ return -EFAULT;
+
+ return 0;
+}
+
+struct drm_ioctl_desc virtio_gpu_ioctls[DRM_VIRTIO_NUM_IOCTLS] = {
+ DRM_IOCTL_DEF_DRV(VIRTGPU_MAP, virtio_gpu_map_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+
+ DRM_IOCTL_DEF_DRV(VIRTGPU_EXECBUFFER, virtio_gpu_execbuffer_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+
+ DRM_IOCTL_DEF_DRV(VIRTGPU_GETPARAM, virtio_gpu_getparam_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+
+ DRM_IOCTL_DEF_DRV(VIRTGPU_RESOURCE_CREATE,
+ virtio_gpu_resource_create_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+
+ DRM_IOCTL_DEF_DRV(VIRTGPU_RESOURCE_INFO, virtio_gpu_resource_info_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+
+ /* make transfer async to the main ring? - no sure, can we
+ thread these in the underlying GL */
+ DRM_IOCTL_DEF_DRV(VIRTGPU_TRANSFER_FROM_HOST,
+ virtio_gpu_transfer_from_host_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(VIRTGPU_TRANSFER_TO_HOST,
+ virtio_gpu_transfer_to_host_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+
+ DRM_IOCTL_DEF_DRV(VIRTGPU_WAIT, virtio_gpu_wait_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+
+ DRM_IOCTL_DEF_DRV(VIRTGPU_GET_CAPS, virtio_gpu_get_caps_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+};
events_clear, &events_clear);
}
+static void virtio_gpu_ctx_id_get(struct virtio_gpu_device *vgdev,
+ uint32_t *resid)
+{
+ int handle;
+
+ idr_preload(GFP_KERNEL);
+ spin_lock(&vgdev->ctx_id_idr_lock);
+ handle = idr_alloc(&vgdev->ctx_id_idr, NULL, 1, 0, 0);
+ spin_unlock(&vgdev->ctx_id_idr_lock);
+ idr_preload_end();
+ *resid = handle;
+}
+
+static void virtio_gpu_ctx_id_put(struct virtio_gpu_device *vgdev, uint32_t id)
+{
+ spin_lock(&vgdev->ctx_id_idr_lock);
+ idr_remove(&vgdev->ctx_id_idr, id);
+ spin_unlock(&vgdev->ctx_id_idr_lock);
+}
+
+static void virtio_gpu_context_create(struct virtio_gpu_device *vgdev,
+ uint32_t nlen, const char *name,
+ uint32_t *ctx_id)
+{
+ virtio_gpu_ctx_id_get(vgdev, ctx_id);
+ virtio_gpu_cmd_context_create(vgdev, *ctx_id, nlen, name);
+}
+
+static void virtio_gpu_context_destroy(struct virtio_gpu_device *vgdev,
+ uint32_t ctx_id)
+{
+ virtio_gpu_cmd_context_destroy(vgdev, ctx_id);
+ virtio_gpu_ctx_id_put(vgdev, ctx_id);
+}
+
static void virtio_gpu_init_vq(struct virtio_gpu_queue *vgvq,
void (*work_func)(struct work_struct *work))
{
INIT_WORK(&vgvq->dequeue_work, work_func);
}
+static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev,
+ int num_capsets)
+{
+ int i, ret;
+
+ vgdev->capsets = kcalloc(num_capsets,
+ sizeof(struct virtio_gpu_drv_capset),
+ GFP_KERNEL);
+ if (!vgdev->capsets) {
+ DRM_ERROR("failed to allocate cap sets\n");
+ return;
+ }
+ for (i = 0; i < num_capsets; i++) {
+ virtio_gpu_cmd_get_capset_info(vgdev, i);
+ ret = wait_event_timeout(vgdev->resp_wq,
+ vgdev->capsets[i].id > 0, 5 * HZ);
+ if (ret == 0) {
+ DRM_ERROR("timed out waiting for cap set %d\n", i);
+ kfree(vgdev->capsets);
+ vgdev->capsets = NULL;
+ return;
+ }
+ DRM_INFO("cap set %d: id %d, max-version %d, max-size %d\n",
+ i, vgdev->capsets[i].id,
+ vgdev->capsets[i].max_version,
+ vgdev->capsets[i].max_size);
+ }
+ vgdev->num_capsets = num_capsets;
+}
+
int virtio_gpu_driver_load(struct drm_device *dev, unsigned long flags)
{
static vq_callback_t *callbacks[] = {
struct virtio_gpu_device *vgdev;
/* this will expand later */
struct virtqueue *vqs[2];
- u32 num_scanouts;
+ u32 num_scanouts, num_capsets;
int ret;
if (!virtio_has_feature(dev->virtdev, VIRTIO_F_VERSION_1))
spin_lock_init(&vgdev->fence_drv.lock);
INIT_LIST_HEAD(&vgdev->fence_drv.fences);
+ INIT_LIST_HEAD(&vgdev->cap_cache);
INIT_WORK(&vgdev->config_changed_work,
virtio_gpu_config_changed_work_func);
+ if (virtio_has_feature(vgdev->vdev, VIRTIO_GPU_F_VIRGL))
+ vgdev->has_virgl_3d = true;
+ DRM_INFO("virgl 3d acceleration %s\n",
+ vgdev->has_virgl_3d ? "enabled" : "not available");
+
ret = vgdev->vdev->config->find_vqs(vgdev->vdev, 2, vqs,
callbacks, names);
if (ret) {
ret = -EINVAL;
goto err_scanouts;
}
+ DRM_INFO("number of scanouts: %d\n", num_scanouts);
+
+ virtio_cread(vgdev->vdev, struct virtio_gpu_config,
+ num_capsets, &num_capsets);
+ DRM_INFO("number of cap sets: %d\n", num_capsets);
ret = virtio_gpu_modeset_init(vgdev);
if (ret)
virtio_device_ready(vgdev->vdev);
vgdev->vqs_ready = true;
+ if (num_capsets)
+ virtio_gpu_get_capsets(vgdev, num_capsets);
virtio_gpu_cmd_get_display_info(vgdev);
wait_event_timeout(vgdev->resp_wq, !vgdev->display_info_pending,
5 * HZ);
return ret;
}
+static void virtio_gpu_cleanup_cap_cache(struct virtio_gpu_device *vgdev)
+{
+ struct virtio_gpu_drv_cap_cache *cache_ent, *tmp;
+
+ list_for_each_entry_safe(cache_ent, tmp, &vgdev->cap_cache, head) {
+ kfree(cache_ent->caps_cache);
+ kfree(cache_ent);
+ }
+}
+
int virtio_gpu_driver_unload(struct drm_device *dev)
{
struct virtio_gpu_device *vgdev = dev->dev_private;
virtio_gpu_modeset_fini(vgdev);
virtio_gpu_ttm_fini(vgdev);
virtio_gpu_free_vbufs(vgdev);
+ virtio_gpu_cleanup_cap_cache(vgdev);
+ kfree(vgdev->capsets);
kfree(vgdev);
return 0;
}
+
+int virtio_gpu_driver_open(struct drm_device *dev, struct drm_file *file)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct virtio_gpu_fpriv *vfpriv;
+ uint32_t id;
+ char dbgname[64], tmpname[TASK_COMM_LEN];
+
+ /* can't create contexts without 3d renderer */
+ if (!vgdev->has_virgl_3d)
+ return 0;
+
+ get_task_comm(tmpname, current);
+ snprintf(dbgname, sizeof(dbgname), "%s", tmpname);
+ dbgname[63] = 0;
+ /* allocate a virt GPU context for this opener */
+ vfpriv = kzalloc(sizeof(*vfpriv), GFP_KERNEL);
+ if (!vfpriv)
+ return -ENOMEM;
+
+ virtio_gpu_context_create(vgdev, strlen(dbgname), dbgname, &id);
+
+ vfpriv->ctx_id = id;
+ file->driver_priv = vfpriv;
+ return 0;
+}
+
+void virtio_gpu_driver_postclose(struct drm_device *dev, struct drm_file *file)
+{
+ struct virtio_gpu_device *vgdev = dev->dev_private;
+ struct virtio_gpu_fpriv *vfpriv;
+
+ if (!vgdev->has_virgl_3d)
+ return;
+
+ vfpriv = file->driver_priv;
+
+ virtio_gpu_context_destroy(vgdev, vfpriv->ctx_id);
+ kfree(vfpriv);
+ file->driver_priv = NULL;
+}
size = roundup(size, PAGE_SIZE);
ret = drm_gem_object_init(vgdev->ddev, &bo->gem_base, size);
if (ret != 0)
- goto err_gem_init;
+ return ret;
bo->dumb = false;
virtio_gpu_init_ttm_placement(bo, pinned);
ret = ttm_bo_init(&vgdev->mman.bdev, &bo->tbo, size, type,
&bo->placement, 0, !kernel, NULL, acc_size,
NULL, NULL, &virtio_gpu_ttm_bo_destroy);
+ /* ttm_bo_init failure will call the destroy */
if (ret != 0)
- goto err_ttm_init;
+ return ret;
*bo_ptr = bo;
return 0;
-
-err_ttm_init:
- drm_gem_object_release(&bo->gem_base);
-err_gem_init:
- kfree(bo);
- return ret;
}
int virtio_gpu_object_kmap(struct virtio_gpu_object *bo, void **ptr)
--- /dev/null
+/*
+ * Copyright 2014 Canonical
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Andreas Pokorny
+ */
+
+#include "virtgpu_drv.h"
+
+/* Empty Implementations as there should not be any other driver for a virtual
+ * device that might share buffers with virtgpu */
+
+int virtgpu_gem_prime_pin(struct drm_gem_object *obj)
+{
+ WARN_ONCE(1, "not implemented");
+ return -ENODEV;
+}
+
+void virtgpu_gem_prime_unpin(struct drm_gem_object *obj)
+{
+ WARN_ONCE(1, "not implemented");
+}
+
+
+struct sg_table *virtgpu_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+ WARN_ONCE(1, "not implemented");
+ return ERR_PTR(-ENODEV);
+}
+
+struct drm_gem_object *virtgpu_gem_prime_import_sg_table(
+ struct drm_device *dev, struct dma_buf_attachment *attach,
+ struct sg_table *table)
+{
+ WARN_ONCE(1, "not implemented");
+ return ERR_PTR(-ENODEV);
+}
+
+void *virtgpu_gem_prime_vmap(struct drm_gem_object *obj)
+{
+ WARN_ONCE(1, "not implemented");
+ return ERR_PTR(-ENODEV);
+}
+
+void virtgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+ WARN_ONCE(1, "not implemented");
+}
+
+int virtgpu_gem_prime_mmap(struct drm_gem_object *obj,
+ struct vm_area_struct *area)
+{
+ return -ENODEV;
+}
#include <ttm/ttm_module.h>
#include <drm/drmP.h>
#include <drm/drm.h>
+#include <drm/virtgpu_drm.h>
#include "virtgpu_drv.h"
#include <linux/delay.h>
wake_up(&vgdev->cursorq.ack_queue);
}
-static int virtio_gpu_queue_ctrl_buffer(struct virtio_gpu_device *vgdev,
- struct virtio_gpu_vbuffer *vbuf)
+static int virtio_gpu_queue_ctrl_buffer_locked(struct virtio_gpu_device *vgdev,
+ struct virtio_gpu_vbuffer *vbuf)
{
struct virtqueue *vq = vgdev->ctrlq.vq;
struct scatterlist *sgs[3], vcmd, vout, vresp;
incnt++;
}
- spin_lock(&vgdev->ctrlq.qlock);
retry:
ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC);
if (ret == -ENOSPC) {
} else {
virtqueue_kick(vq);
}
- spin_unlock(&vgdev->ctrlq.qlock);
if (!ret)
ret = vq->num_free;
return ret;
}
+static int virtio_gpu_queue_ctrl_buffer(struct virtio_gpu_device *vgdev,
+ struct virtio_gpu_vbuffer *vbuf)
+{
+ int rc;
+
+ spin_lock(&vgdev->ctrlq.qlock);
+ rc = virtio_gpu_queue_ctrl_buffer_locked(vgdev, vbuf);
+ spin_unlock(&vgdev->ctrlq.qlock);
+ return rc;
+}
+
+static int virtio_gpu_queue_fenced_ctrl_buffer(struct virtio_gpu_device *vgdev,
+ struct virtio_gpu_vbuffer *vbuf,
+ struct virtio_gpu_ctrl_hdr *hdr,
+ struct virtio_gpu_fence **fence)
+{
+ struct virtqueue *vq = vgdev->ctrlq.vq;
+ int rc;
+
+again:
+ spin_lock(&vgdev->ctrlq.qlock);
+
+ /*
+ * Make sure we have enouth space in the virtqueue. If not
+ * wait here until we have.
+ *
+ * Without that virtio_gpu_queue_ctrl_buffer_nolock might have
+ * to wait for free space, which can result in fence ids being
+ * submitted out-of-order.
+ */
+ if (vq->num_free < 3) {
+ spin_unlock(&vgdev->ctrlq.qlock);
+ wait_event(vgdev->ctrlq.ack_queue, vq->num_free >= 3);
+ goto again;
+ }
+
+ if (fence)
+ virtio_gpu_fence_emit(vgdev, hdr, fence);
+ rc = virtio_gpu_queue_ctrl_buffer_locked(vgdev, vbuf);
+ spin_unlock(&vgdev->ctrlq.qlock);
+ return rc;
+}
+
static int virtio_gpu_queue_cursor(struct virtio_gpu_device *vgdev,
struct virtio_gpu_vbuffer *vbuf)
{
cmd_p->r.x = x;
cmd_p->r.y = y;
- if (fence)
- virtio_gpu_fence_emit(vgdev, &cmd_p->hdr, fence);
- virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+ virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
}
static void
vbuf->data_buf = ents;
vbuf->data_size = sizeof(*ents) * nents;
- if (fence)
- virtio_gpu_fence_emit(vgdev, &cmd_p->hdr, fence);
- virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+ virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
}
static void virtio_gpu_cmd_get_display_info_cb(struct virtio_gpu_device *vgdev,
drm_kms_helper_hotplug_event(vgdev->ddev);
}
+static void virtio_gpu_cmd_get_capset_info_cb(struct virtio_gpu_device *vgdev,
+ struct virtio_gpu_vbuffer *vbuf)
+{
+ struct virtio_gpu_get_capset_info *cmd =
+ (struct virtio_gpu_get_capset_info *)vbuf->buf;
+ struct virtio_gpu_resp_capset_info *resp =
+ (struct virtio_gpu_resp_capset_info *)vbuf->resp_buf;
+ int i = le32_to_cpu(cmd->capset_index);
+
+ spin_lock(&vgdev->display_info_lock);
+ vgdev->capsets[i].id = le32_to_cpu(resp->capset_id);
+ vgdev->capsets[i].max_version = le32_to_cpu(resp->capset_max_version);
+ vgdev->capsets[i].max_size = le32_to_cpu(resp->capset_max_size);
+ spin_unlock(&vgdev->display_info_lock);
+ wake_up(&vgdev->resp_wq);
+}
+
+static void virtio_gpu_cmd_capset_cb(struct virtio_gpu_device *vgdev,
+ struct virtio_gpu_vbuffer *vbuf)
+{
+ struct virtio_gpu_get_capset *cmd =
+ (struct virtio_gpu_get_capset *)vbuf->buf;
+ struct virtio_gpu_resp_capset *resp =
+ (struct virtio_gpu_resp_capset *)vbuf->resp_buf;
+ struct virtio_gpu_drv_cap_cache *cache_ent;
+
+ spin_lock(&vgdev->display_info_lock);
+ list_for_each_entry(cache_ent, &vgdev->cap_cache, head) {
+ if (cache_ent->version == le32_to_cpu(cmd->capset_version) &&
+ cache_ent->id == le32_to_cpu(cmd->capset_id)) {
+ memcpy(cache_ent->caps_cache, resp->capset_data,
+ cache_ent->size);
+ atomic_set(&cache_ent->is_valid, 1);
+ break;
+ }
+ }
+ spin_unlock(&vgdev->display_info_lock);
+ wake_up(&vgdev->resp_wq);
+}
+
+
int virtio_gpu_cmd_get_display_info(struct virtio_gpu_device *vgdev)
{
struct virtio_gpu_ctrl_hdr *cmd_p;
return 0;
}
+int virtio_gpu_cmd_get_capset_info(struct virtio_gpu_device *vgdev, int idx)
+{
+ struct virtio_gpu_get_capset_info *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+ void *resp_buf;
+
+ resp_buf = kzalloc(sizeof(struct virtio_gpu_resp_capset_info),
+ GFP_KERNEL);
+ if (!resp_buf)
+ return -ENOMEM;
+
+ cmd_p = virtio_gpu_alloc_cmd_resp
+ (vgdev, &virtio_gpu_cmd_get_capset_info_cb, &vbuf,
+ sizeof(*cmd_p), sizeof(struct virtio_gpu_resp_capset_info),
+ resp_buf);
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_GET_CAPSET_INFO);
+ cmd_p->capset_index = cpu_to_le32(idx);
+ virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+ return 0;
+}
+
+int virtio_gpu_cmd_get_capset(struct virtio_gpu_device *vgdev,
+ int idx, int version,
+ struct virtio_gpu_drv_cap_cache **cache_p)
+{
+ struct virtio_gpu_get_capset *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+ int max_size = vgdev->capsets[idx].max_size;
+ struct virtio_gpu_drv_cap_cache *cache_ent;
+ void *resp_buf;
+
+ if (idx > vgdev->num_capsets)
+ return -EINVAL;
+
+ if (version > vgdev->capsets[idx].max_version)
+ return -EINVAL;
+
+ cache_ent = kzalloc(sizeof(*cache_ent), GFP_KERNEL);
+ if (!cache_ent)
+ return -ENOMEM;
+
+ cache_ent->caps_cache = kmalloc(max_size, GFP_KERNEL);
+ if (!cache_ent->caps_cache) {
+ kfree(cache_ent);
+ return -ENOMEM;
+ }
+
+ resp_buf = kzalloc(sizeof(struct virtio_gpu_resp_capset) + max_size,
+ GFP_KERNEL);
+ if (!resp_buf) {
+ kfree(cache_ent->caps_cache);
+ kfree(cache_ent);
+ return -ENOMEM;
+ }
+
+ cache_ent->version = version;
+ cache_ent->id = vgdev->capsets[idx].id;
+ atomic_set(&cache_ent->is_valid, 0);
+ cache_ent->size = max_size;
+ spin_lock(&vgdev->display_info_lock);
+ list_add_tail(&cache_ent->head, &vgdev->cap_cache);
+ spin_unlock(&vgdev->display_info_lock);
+
+ cmd_p = virtio_gpu_alloc_cmd_resp
+ (vgdev, &virtio_gpu_cmd_capset_cb, &vbuf, sizeof(*cmd_p),
+ sizeof(struct virtio_gpu_resp_capset) + max_size,
+ resp_buf);
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_GET_CAPSET);
+ cmd_p->capset_id = cpu_to_le32(vgdev->capsets[idx].id);
+ cmd_p->capset_version = cpu_to_le32(version);
+ *cache_p = cache_ent;
+ virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+
+ return 0;
+}
+
+void virtio_gpu_cmd_context_create(struct virtio_gpu_device *vgdev, uint32_t id,
+ uint32_t nlen, const char *name)
+{
+ struct virtio_gpu_ctx_create *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_CTX_CREATE);
+ cmd_p->hdr.ctx_id = cpu_to_le32(id);
+ cmd_p->nlen = cpu_to_le32(nlen);
+ strncpy(cmd_p->debug_name, name, sizeof(cmd_p->debug_name)-1);
+ cmd_p->debug_name[sizeof(cmd_p->debug_name)-1] = 0;
+ virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+}
+
+void virtio_gpu_cmd_context_destroy(struct virtio_gpu_device *vgdev,
+ uint32_t id)
+{
+ struct virtio_gpu_ctx_destroy *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_CTX_DESTROY);
+ cmd_p->hdr.ctx_id = cpu_to_le32(id);
+ virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+}
+
+void virtio_gpu_cmd_context_attach_resource(struct virtio_gpu_device *vgdev,
+ uint32_t ctx_id,
+ uint32_t resource_id)
+{
+ struct virtio_gpu_ctx_resource *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE);
+ cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id);
+ cmd_p->resource_id = cpu_to_le32(resource_id);
+ virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+
+}
+
+void virtio_gpu_cmd_context_detach_resource(struct virtio_gpu_device *vgdev,
+ uint32_t ctx_id,
+ uint32_t resource_id)
+{
+ struct virtio_gpu_ctx_resource *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE);
+ cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id);
+ cmd_p->resource_id = cpu_to_le32(resource_id);
+ virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+}
+
+void
+virtio_gpu_cmd_resource_create_3d(struct virtio_gpu_device *vgdev,
+ struct virtio_gpu_resource_create_3d *rc_3d,
+ struct virtio_gpu_fence **fence)
+{
+ struct virtio_gpu_resource_create_3d *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ *cmd_p = *rc_3d;
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_CREATE_3D);
+ cmd_p->hdr.flags = 0;
+
+ virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
+}
+
+void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev,
+ uint32_t resource_id, uint32_t ctx_id,
+ uint64_t offset, uint32_t level,
+ struct virtio_gpu_box *box,
+ struct virtio_gpu_fence **fence)
+{
+ struct virtio_gpu_transfer_host_3d *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D);
+ cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id);
+ cmd_p->resource_id = cpu_to_le32(resource_id);
+ cmd_p->box = *box;
+ cmd_p->offset = cpu_to_le64(offset);
+ cmd_p->level = cpu_to_le32(level);
+
+ virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
+}
+
+void virtio_gpu_cmd_transfer_from_host_3d(struct virtio_gpu_device *vgdev,
+ uint32_t resource_id, uint32_t ctx_id,
+ uint64_t offset, uint32_t level,
+ struct virtio_gpu_box *box,
+ struct virtio_gpu_fence **fence)
+{
+ struct virtio_gpu_transfer_host_3d *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D);
+ cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id);
+ cmd_p->resource_id = cpu_to_le32(resource_id);
+ cmd_p->box = *box;
+ cmd_p->offset = cpu_to_le64(offset);
+ cmd_p->level = cpu_to_le32(level);
+
+ virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
+}
+
+void virtio_gpu_cmd_submit(struct virtio_gpu_device *vgdev,
+ void *data, uint32_t data_size,
+ uint32_t ctx_id, struct virtio_gpu_fence **fence)
+{
+ struct virtio_gpu_cmd_submit *cmd_p;
+ struct virtio_gpu_vbuffer *vbuf;
+
+ cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
+ memset(cmd_p, 0, sizeof(*cmd_p));
+
+ vbuf->data_buf = data;
+ vbuf->data_size = data_size;
+
+ cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_SUBMIT_3D);
+ cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id);
+ cmd_p->size = cpu_to_le32(data_size);
+
+ virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
+}
+
int virtio_gpu_object_attach(struct virtio_gpu_device *vgdev,
struct virtio_gpu_object *obj,
uint32_t resource_id,
config DRM_VMWGFX
tristate "DRM driver for VMware Virtual GPU"
- depends on DRM && PCI
+ depends on DRM && PCI && X86
select FB_DEFERRED_IO
select FB_CFB_FILLRECT
select FB_CFB_COPYAREA
0, 0,
DRM_MM_SEARCH_DEFAULT,
DRM_MM_CREATE_DEFAULT);
+ if (ret) {
+ (void) vmw_cmdbuf_man_process(man);
+ ret = drm_mm_insert_node_generic(&man->mm, info->node,
+ info->page_size, 0, 0,
+ DRM_MM_SEARCH_DEFAULT,
+ DRM_MM_CREATE_DEFAULT);
+ }
+
spin_unlock_bh(&man->lock);
info->done = !ret;
struct vmw_private *dev_priv = res->dev_priv;
struct ttm_buffer_object *bo = val_buf->bo;
struct vmw_fence_obj *fence;
- int ret;
if (list_empty(&res->mob_head))
return 0;
if (likely(fence != NULL))
vmw_fence_obj_unreference(&fence);
- return ret;
+ return 0;
}
/**
static const struct drm_ioctl_desc vmw_ioctls[] = {
VMW_IOCTL_DEF(VMW_GET_PARAM, vmw_getparam_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_ALLOC_DMABUF, vmw_dmabuf_alloc_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_UNREF_DMABUF, vmw_dmabuf_unref_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_CURSOR_BYPASS,
vmw_kms_cursor_bypass_ioctl,
- DRM_MASTER | DRM_CONTROL_ALLOW | DRM_UNLOCKED),
+ DRM_MASTER | DRM_CONTROL_ALLOW),
VMW_IOCTL_DEF(VMW_CONTROL_STREAM, vmw_overlay_ioctl,
- DRM_MASTER | DRM_CONTROL_ALLOW | DRM_UNLOCKED),
+ DRM_MASTER | DRM_CONTROL_ALLOW),
VMW_IOCTL_DEF(VMW_CLAIM_STREAM, vmw_stream_claim_ioctl,
- DRM_MASTER | DRM_CONTROL_ALLOW | DRM_UNLOCKED),
+ DRM_MASTER | DRM_CONTROL_ALLOW),
VMW_IOCTL_DEF(VMW_UNREF_STREAM, vmw_stream_unref_ioctl,
- DRM_MASTER | DRM_CONTROL_ALLOW | DRM_UNLOCKED),
+ DRM_MASTER | DRM_CONTROL_ALLOW),
VMW_IOCTL_DEF(VMW_CREATE_CONTEXT, vmw_context_define_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_UNREF_CONTEXT, vmw_context_destroy_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_CREATE_SURFACE, vmw_surface_define_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_UNREF_SURFACE, vmw_surface_destroy_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_REF_SURFACE, vmw_surface_reference_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
- VMW_IOCTL_DEF(VMW_EXECBUF, NULL, DRM_AUTH | DRM_UNLOCKED |
+ DRM_AUTH | DRM_RENDER_ALLOW),
+ VMW_IOCTL_DEF(VMW_EXECBUF, NULL, DRM_AUTH |
DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_FENCE_WAIT, vmw_fence_obj_wait_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_FENCE_SIGNALED,
vmw_fence_obj_signaled_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_FENCE_UNREF, vmw_fence_obj_unref_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_FENCE_EVENT, vmw_fence_event_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_GET_3D_CAP, vmw_get_cap_3d_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
/* these allow direct access to the framebuffers mark as master only */
VMW_IOCTL_DEF(VMW_PRESENT, vmw_present_ioctl,
- DRM_MASTER | DRM_AUTH | DRM_UNLOCKED),
+ DRM_MASTER | DRM_AUTH),
VMW_IOCTL_DEF(VMW_PRESENT_READBACK,
vmw_present_readback_ioctl,
- DRM_MASTER | DRM_AUTH | DRM_UNLOCKED),
+ DRM_MASTER | DRM_AUTH),
VMW_IOCTL_DEF(VMW_UPDATE_LAYOUT,
vmw_kms_update_layout_ioctl,
- DRM_MASTER | DRM_UNLOCKED),
+ DRM_MASTER),
VMW_IOCTL_DEF(VMW_CREATE_SHADER,
vmw_shader_define_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_UNREF_SHADER,
vmw_shader_destroy_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_GB_SURFACE_CREATE,
vmw_gb_surface_define_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_GB_SURFACE_REF,
vmw_gb_surface_reference_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_SYNCCPU,
vmw_user_dmabuf_synccpu_ioctl,
- DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_RENDER_ALLOW),
VMW_IOCTL_DEF(VMW_CREATE_EXTENDED_CONTEXT,
vmw_extended_context_define_ioctl,
- DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
+ DRM_AUTH | DRM_RENDER_ALLOW),
};
static struct pci_device_id vmw_pci_id_list[] = {
ttm_lock_set_kill(&dev_priv->fbdev_master.lock, false, SIGTERM);
dev_priv->active_master = &dev_priv->fbdev_master;
-
- dev_priv->mmio_mtrr = arch_phys_wc_add(dev_priv->mmio_start,
- dev_priv->mmio_size);
-
- dev_priv->mmio_virt = ioremap_wc(dev_priv->mmio_start,
- dev_priv->mmio_size);
+ dev_priv->mmio_virt = memremap(dev_priv->mmio_start,
+ dev_priv->mmio_size, MEMREMAP_WB);
if (unlikely(dev_priv->mmio_virt == NULL)) {
ret = -ENOMEM;
out_no_device:
ttm_object_device_release(&dev_priv->tdev);
out_err4:
- iounmap(dev_priv->mmio_virt);
+ memunmap(dev_priv->mmio_virt);
out_err3:
- arch_phys_wc_del(dev_priv->mmio_mtrr);
vmw_ttm_global_release(dev_priv);
out_err0:
for (i = vmw_res_context; i < vmw_res_max; ++i)
pci_release_regions(dev->pdev);
ttm_object_device_release(&dev_priv->tdev);
- iounmap(dev_priv->mmio_virt);
- arch_phys_wc_del(dev_priv->mmio_mtrr);
+ memunmap(dev_priv->mmio_virt);
if (dev_priv->ctx.staged_bindings)
vmw_binding_state_free(dev_priv->ctx.staged_bindings);
vmw_ttm_global_release(dev_priv);
}
mutex_unlock(&dev->master_mutex);
- /*
- * Taking the drm_global_mutex after the TTM lock might deadlock
- */
- if (!(flags & DRM_UNLOCKED)) {
- DRM_ERROR("Refusing locked ioctl access.\n");
- return ERR_PTR(-EDEADLK);
- }
-
/*
* Take the TTM lock. Possibly sleep waiting for the authenticating
* master to become master again, or for a SIGTERM if the
uint32_t stdu_max_height;
uint32_t initial_width;
uint32_t initial_height;
- u32 __iomem *mmio_virt;
- int mmio_mtrr;
+ u32 *mmio_virt;
uint32_t capabilities;
uint32_t max_gmr_ids;
uint32_t max_gmr_pages;
uint32_t size,
bool shareable,
uint32_t *handle,
- struct vmw_dma_buffer **p_dma_buf);
+ struct vmw_dma_buffer **p_dma_buf,
+ struct ttm_base_object **p_base);
extern int vmw_user_dmabuf_reference(struct ttm_object_file *tfile,
struct vmw_dma_buffer *dma_buf,
uint32_t *handle);
uint32_t cur_validate_node);
extern void vmw_dmabuf_validate_clear(struct ttm_buffer_object *bo);
extern int vmw_user_dmabuf_lookup(struct ttm_object_file *tfile,
- uint32_t id, struct vmw_dma_buffer **out);
+ uint32_t id, struct vmw_dma_buffer **out,
+ struct ttm_base_object **base);
extern int vmw_stream_claim_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
extern int vmw_stream_unref_ioctl(struct drm_device *dev, void *data,
bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv,
uint32_t pitch,
uint32_t height);
-u32 vmw_get_vblank_counter(struct drm_device *dev, int crtc);
-int vmw_enable_vblank(struct drm_device *dev, int crtc);
-void vmw_disable_vblank(struct drm_device *dev, int crtc);
+u32 vmw_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
+int vmw_enable_vblank(struct drm_device *dev, unsigned int pipe);
+void vmw_disable_vblank(struct drm_device *dev, unsigned int pipe);
int vmw_kms_present(struct vmw_private *dev_priv,
struct drm_file *file_priv,
struct vmw_framebuffer *vfb,
{
atomic_dec(&dev_priv->num_fifo_resources);
}
+
+/**
+ * vmw_mmio_read - Perform a MMIO read from volatile memory
+ *
+ * @addr: The address to read from
+ *
+ * This function is intended to be equivalent to ioread32() on
+ * memremap'd memory, but without byteswapping.
+ */
+static inline u32 vmw_mmio_read(u32 *addr)
+{
+ return READ_ONCE(*addr);
+}
+
+/**
+ * vmw_mmio_write - Perform a MMIO write to volatile memory
+ *
+ * @addr: The address to write to
+ *
+ * This function is intended to be equivalent to iowrite32 on
+ * memremap'd memory, but without byteswapping.
+ */
+static inline void vmw_mmio_write(u32 value, u32 *addr)
+{
+ WRITE_ONCE(*addr, value);
+}
#endif
struct vmw_relocation *reloc;
int ret;
- ret = vmw_user_dmabuf_lookup(sw_context->fp->tfile, handle, &vmw_bo);
+ ret = vmw_user_dmabuf_lookup(sw_context->fp->tfile, handle, &vmw_bo,
+ NULL);
if (unlikely(ret != 0)) {
DRM_ERROR("Could not find or use MOB buffer.\n");
ret = -EINVAL;
struct vmw_relocation *reloc;
int ret;
- ret = vmw_user_dmabuf_lookup(sw_context->fp->tfile, handle, &vmw_bo);
+ ret = vmw_user_dmabuf_lookup(sw_context->fp->tfile, handle, &vmw_bo,
+ NULL);
if (unlikely(ret != 0)) {
DRM_ERROR("Could not find or use GMR region.\n");
ret = -EINVAL;
struct vmw_fence_manager *fman = fman_from_fence(fence);
struct vmw_private *dev_priv = fman->dev_priv;
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
- u32 seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
+ u32 *fifo_mem = dev_priv->mmio_virt;
+ u32 seqno = vmw_mmio_read(fifo_mem + SVGA_FIFO_FENCE);
if (seqno - fence->base.seqno < VMW_FENCE_WRAP)
return false;
u32 passed_seqno)
{
u32 goal_seqno;
- u32 __iomem *fifo_mem;
+ u32 *fifo_mem;
struct vmw_fence_obj *fence;
if (likely(!fman->seqno_valid))
return false;
fifo_mem = fman->dev_priv->mmio_virt;
- goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL);
+ goal_seqno = vmw_mmio_read(fifo_mem + SVGA_FIFO_FENCE_GOAL);
if (likely(passed_seqno - goal_seqno >= VMW_FENCE_WRAP))
return false;
list_for_each_entry(fence, &fman->fence_list, head) {
if (!list_empty(&fence->seq_passed_actions)) {
fman->seqno_valid = true;
- iowrite32(fence->base.seqno,
- fifo_mem + SVGA_FIFO_FENCE_GOAL);
+ vmw_mmio_write(fence->base.seqno,
+ fifo_mem + SVGA_FIFO_FENCE_GOAL);
break;
}
}
{
struct vmw_fence_manager *fman = fman_from_fence(fence);
u32 goal_seqno;
- u32 __iomem *fifo_mem;
+ u32 *fifo_mem;
if (fence_is_signaled_locked(&fence->base))
return false;
fifo_mem = fman->dev_priv->mmio_virt;
- goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL);
+ goal_seqno = vmw_mmio_read(fifo_mem + SVGA_FIFO_FENCE_GOAL);
if (likely(fman->seqno_valid &&
goal_seqno - fence->base.seqno < VMW_FENCE_WRAP))
return false;
- iowrite32(fence->base.seqno, fifo_mem + SVGA_FIFO_FENCE_GOAL);
+ vmw_mmio_write(fence->base.seqno, fifo_mem + SVGA_FIFO_FENCE_GOAL);
fman->seqno_valid = true;
return true;
struct list_head action_list;
bool needs_rerun;
uint32_t seqno, new_seqno;
- u32 __iomem *fifo_mem = fman->dev_priv->mmio_virt;
+ u32 *fifo_mem = fman->dev_priv->mmio_virt;
- seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
+ seqno = vmw_mmio_read(fifo_mem + SVGA_FIFO_FENCE);
rerun:
list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {
if (seqno - fence->base.seqno < VMW_FENCE_WRAP) {
needs_rerun = vmw_fence_goal_new_locked(fman, seqno);
if (unlikely(needs_rerun)) {
- new_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
+ new_seqno = vmw_mmio_read(fifo_mem + SVGA_FIFO_FENCE);
if (new_seqno != seqno) {
seqno = new_seqno;
goto rerun;
bool vmw_fifo_have_3d(struct vmw_private *dev_priv)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
+ u32 *fifo_mem = dev_priv->mmio_virt;
uint32_t fifo_min, hwversion;
const struct vmw_fifo_state *fifo = &dev_priv->fifo;
if (!(dev_priv->capabilities & SVGA_CAP_EXTENDED_FIFO))
return false;
- fifo_min = ioread32(fifo_mem + SVGA_FIFO_MIN);
+ fifo_min = vmw_mmio_read(fifo_mem + SVGA_FIFO_MIN);
if (fifo_min <= SVGA_FIFO_3D_HWVERSION * sizeof(unsigned int))
return false;
- hwversion = ioread32(fifo_mem +
- ((fifo->capabilities &
- SVGA_FIFO_CAP_3D_HWVERSION_REVISED) ?
- SVGA_FIFO_3D_HWVERSION_REVISED :
- SVGA_FIFO_3D_HWVERSION));
+ hwversion = vmw_mmio_read(fifo_mem +
+ ((fifo->capabilities &
+ SVGA_FIFO_CAP_3D_HWVERSION_REVISED) ?
+ SVGA_FIFO_3D_HWVERSION_REVISED :
+ SVGA_FIFO_3D_HWVERSION));
if (hwversion == 0)
return false;
bool vmw_fifo_have_pitchlock(struct vmw_private *dev_priv)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
+ u32 *fifo_mem = dev_priv->mmio_virt;
uint32_t caps;
if (!(dev_priv->capabilities & SVGA_CAP_EXTENDED_FIFO))
return false;
- caps = ioread32(fifo_mem + SVGA_FIFO_CAPABILITIES);
+ caps = vmw_mmio_read(fifo_mem + SVGA_FIFO_CAPABILITIES);
if (caps & SVGA_FIFO_CAP_PITCHLOCK)
return true;
int vmw_fifo_init(struct vmw_private *dev_priv, struct vmw_fifo_state *fifo)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
+ u32 *fifo_mem = dev_priv->mmio_virt;
uint32_t max;
uint32_t min;
if (min < PAGE_SIZE)
min = PAGE_SIZE;
- iowrite32(min, fifo_mem + SVGA_FIFO_MIN);
- iowrite32(dev_priv->mmio_size, fifo_mem + SVGA_FIFO_MAX);
+ vmw_mmio_write(min, fifo_mem + SVGA_FIFO_MIN);
+ vmw_mmio_write(dev_priv->mmio_size, fifo_mem + SVGA_FIFO_MAX);
wmb();
- iowrite32(min, fifo_mem + SVGA_FIFO_NEXT_CMD);
- iowrite32(min, fifo_mem + SVGA_FIFO_STOP);
- iowrite32(0, fifo_mem + SVGA_FIFO_BUSY);
+ vmw_mmio_write(min, fifo_mem + SVGA_FIFO_NEXT_CMD);
+ vmw_mmio_write(min, fifo_mem + SVGA_FIFO_STOP);
+ vmw_mmio_write(0, fifo_mem + SVGA_FIFO_BUSY);
mb();
vmw_write(dev_priv, SVGA_REG_CONFIG_DONE, 1);
- max = ioread32(fifo_mem + SVGA_FIFO_MAX);
- min = ioread32(fifo_mem + SVGA_FIFO_MIN);
- fifo->capabilities = ioread32(fifo_mem + SVGA_FIFO_CAPABILITIES);
+ max = vmw_mmio_read(fifo_mem + SVGA_FIFO_MAX);
+ min = vmw_mmio_read(fifo_mem + SVGA_FIFO_MIN);
+ fifo->capabilities = vmw_mmio_read(fifo_mem + SVGA_FIFO_CAPABILITIES);
DRM_INFO("Fifo max 0x%08x min 0x%08x cap 0x%08x\n",
(unsigned int) max,
(unsigned int) fifo->capabilities);
atomic_set(&dev_priv->marker_seq, dev_priv->last_read_seqno);
- iowrite32(dev_priv->last_read_seqno, fifo_mem + SVGA_FIFO_FENCE);
+ vmw_mmio_write(dev_priv->last_read_seqno, fifo_mem + SVGA_FIFO_FENCE);
vmw_marker_queue_init(&fifo->marker_queue);
return 0;
void vmw_fifo_ping_host(struct vmw_private *dev_priv, uint32_t reason)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
- static DEFINE_SPINLOCK(ping_lock);
- unsigned long irq_flags;
+ u32 *fifo_mem = dev_priv->mmio_virt;
- /*
- * The ping_lock is needed because we don't have an atomic
- * test-and-set of the SVGA_FIFO_BUSY register.
- */
- spin_lock_irqsave(&ping_lock, irq_flags);
- if (unlikely(ioread32(fifo_mem + SVGA_FIFO_BUSY) == 0)) {
- iowrite32(1, fifo_mem + SVGA_FIFO_BUSY);
+ preempt_disable();
+ if (cmpxchg(fifo_mem + SVGA_FIFO_BUSY, 0, 1) == 0)
vmw_write(dev_priv, SVGA_REG_SYNC, reason);
- }
- spin_unlock_irqrestore(&ping_lock, irq_flags);
+ preempt_enable();
}
void vmw_fifo_release(struct vmw_private *dev_priv, struct vmw_fifo_state *fifo)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
+ u32 *fifo_mem = dev_priv->mmio_virt;
vmw_write(dev_priv, SVGA_REG_SYNC, SVGA_SYNC_GENERIC);
while (vmw_read(dev_priv, SVGA_REG_BUSY) != 0)
;
- dev_priv->last_read_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
+ dev_priv->last_read_seqno = vmw_mmio_read(fifo_mem + SVGA_FIFO_FENCE);
vmw_write(dev_priv, SVGA_REG_CONFIG_DONE,
dev_priv->config_done_state);
static bool vmw_fifo_is_full(struct vmw_private *dev_priv, uint32_t bytes)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
- uint32_t max = ioread32(fifo_mem + SVGA_FIFO_MAX);
- uint32_t next_cmd = ioread32(fifo_mem + SVGA_FIFO_NEXT_CMD);
- uint32_t min = ioread32(fifo_mem + SVGA_FIFO_MIN);
- uint32_t stop = ioread32(fifo_mem + SVGA_FIFO_STOP);
+ u32 *fifo_mem = dev_priv->mmio_virt;
+ uint32_t max = vmw_mmio_read(fifo_mem + SVGA_FIFO_MAX);
+ uint32_t next_cmd = vmw_mmio_read(fifo_mem + SVGA_FIFO_NEXT_CMD);
+ uint32_t min = vmw_mmio_read(fifo_mem + SVGA_FIFO_MIN);
+ uint32_t stop = vmw_mmio_read(fifo_mem + SVGA_FIFO_STOP);
return ((max - next_cmd) + (stop - min) <= bytes);
}
uint32_t bytes)
{
struct vmw_fifo_state *fifo_state = &dev_priv->fifo;
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
+ u32 *fifo_mem = dev_priv->mmio_virt;
uint32_t max;
uint32_t min;
uint32_t next_cmd;
int ret;
mutex_lock(&fifo_state->fifo_mutex);
- max = ioread32(fifo_mem + SVGA_FIFO_MAX);
- min = ioread32(fifo_mem + SVGA_FIFO_MIN);
- next_cmd = ioread32(fifo_mem + SVGA_FIFO_NEXT_CMD);
+ max = vmw_mmio_read(fifo_mem + SVGA_FIFO_MAX);
+ min = vmw_mmio_read(fifo_mem + SVGA_FIFO_MIN);
+ next_cmd = vmw_mmio_read(fifo_mem + SVGA_FIFO_NEXT_CMD);
if (unlikely(bytes >= (max - min)))
goto out_err;
fifo_state->reserved_size = bytes;
while (1) {
- uint32_t stop = ioread32(fifo_mem + SVGA_FIFO_STOP);
+ uint32_t stop = vmw_mmio_read(fifo_mem + SVGA_FIFO_STOP);
bool need_bounce = false;
bool reserve_in_place = false;
fifo_state->using_bounce_buffer = false;
if (reserveable)
- iowrite32(bytes, fifo_mem +
- SVGA_FIFO_RESERVED);
+ vmw_mmio_write(bytes, fifo_mem +
+ SVGA_FIFO_RESERVED);
return (void __force *) (fifo_mem +
(next_cmd >> 2));
} else {
}
static void vmw_fifo_res_copy(struct vmw_fifo_state *fifo_state,
- u32 __iomem *fifo_mem,
+ u32 *fifo_mem,
uint32_t next_cmd,
uint32_t max, uint32_t min, uint32_t bytes)
{
if (bytes < chunk_size)
chunk_size = bytes;
- iowrite32(bytes, fifo_mem + SVGA_FIFO_RESERVED);
+ vmw_mmio_write(bytes, fifo_mem + SVGA_FIFO_RESERVED);
mb();
- memcpy_toio(fifo_mem + (next_cmd >> 2), buffer, chunk_size);
+ memcpy(fifo_mem + (next_cmd >> 2), buffer, chunk_size);
rest = bytes - chunk_size;
if (rest)
- memcpy_toio(fifo_mem + (min >> 2), buffer + (chunk_size >> 2),
- rest);
+ memcpy(fifo_mem + (min >> 2), buffer + (chunk_size >> 2), rest);
}
static void vmw_fifo_slow_copy(struct vmw_fifo_state *fifo_state,
- u32 __iomem *fifo_mem,
+ u32 *fifo_mem,
uint32_t next_cmd,
uint32_t max, uint32_t min, uint32_t bytes)
{
fifo_state->dynamic_buffer : fifo_state->static_buffer;
while (bytes > 0) {
- iowrite32(*buffer++, fifo_mem + (next_cmd >> 2));
+ vmw_mmio_write(*buffer++, fifo_mem + (next_cmd >> 2));
next_cmd += sizeof(uint32_t);
if (unlikely(next_cmd == max))
next_cmd = min;
mb();
- iowrite32(next_cmd, fifo_mem + SVGA_FIFO_NEXT_CMD);
+ vmw_mmio_write(next_cmd, fifo_mem + SVGA_FIFO_NEXT_CMD);
mb();
bytes -= sizeof(uint32_t);
}
static void vmw_local_fifo_commit(struct vmw_private *dev_priv, uint32_t bytes)
{
struct vmw_fifo_state *fifo_state = &dev_priv->fifo;
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
- uint32_t next_cmd = ioread32(fifo_mem + SVGA_FIFO_NEXT_CMD);
- uint32_t max = ioread32(fifo_mem + SVGA_FIFO_MAX);
- uint32_t min = ioread32(fifo_mem + SVGA_FIFO_MIN);
+ u32 *fifo_mem = dev_priv->mmio_virt;
+ uint32_t next_cmd = vmw_mmio_read(fifo_mem + SVGA_FIFO_NEXT_CMD);
+ uint32_t max = vmw_mmio_read(fifo_mem + SVGA_FIFO_MAX);
+ uint32_t min = vmw_mmio_read(fifo_mem + SVGA_FIFO_MIN);
bool reserveable = fifo_state->capabilities & SVGA_FIFO_CAP_RESERVE;
if (fifo_state->dx)
if (next_cmd >= max)
next_cmd -= max - min;
mb();
- iowrite32(next_cmd, fifo_mem + SVGA_FIFO_NEXT_CMD);
+ vmw_mmio_write(next_cmd, fifo_mem + SVGA_FIFO_NEXT_CMD);
}
if (reserveable)
- iowrite32(0, fifo_mem + SVGA_FIFO_RESERVED);
+ vmw_mmio_write(0, fifo_mem + SVGA_FIFO_RESERVED);
mb();
up_write(&fifo_state->rwsem);
vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC);
break;
case DRM_VMW_PARAM_FIFO_HW_VERSION:
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
+ u32 *fifo_mem = dev_priv->mmio_virt;
const struct vmw_fifo_state *fifo = &dev_priv->fifo;
if ((dev_priv->capabilities & SVGA_CAP_GBOBJECTS)) {
}
param->value =
- ioread32(fifo_mem +
- ((fifo->capabilities &
- SVGA_FIFO_CAP_3D_HWVERSION_REVISED) ?
- SVGA_FIFO_3D_HWVERSION_REVISED :
- SVGA_FIFO_3D_HWVERSION));
+ vmw_mmio_read(fifo_mem +
+ ((fifo->capabilities &
+ SVGA_FIFO_CAP_3D_HWVERSION_REVISED) ?
+ SVGA_FIFO_3D_HWVERSION_REVISED :
+ SVGA_FIFO_3D_HWVERSION));
break;
}
case DRM_VMW_PARAM_MAX_SURF_MEMORY:
return 0;
}
+static u32 vmw_mask_multisample(unsigned int cap, u32 fmt_value)
+{
+ /* If the header is updated, update the format test as well! */
+ BUILD_BUG_ON(SVGA3D_DEVCAP_DXFMT_BC5_UNORM + 1 != SVGA3D_DEVCAP_MAX);
+
+ if (cap >= SVGA3D_DEVCAP_DXFMT_X8R8G8B8 &&
+ cap <= SVGA3D_DEVCAP_DXFMT_BC5_UNORM)
+ fmt_value &= ~(SVGADX_DXFMT_MULTISAMPLE_2 |
+ SVGADX_DXFMT_MULTISAMPLE_4 |
+ SVGADX_DXFMT_MULTISAMPLE_8);
+ else if (cap == SVGA3D_DEVCAP_MULTISAMPLE_MASKABLESAMPLES)
+ return 0;
+
+ return fmt_value;
+}
+
static int vmw_fill_compat_cap(struct vmw_private *dev_priv, void *bounce,
size_t size)
{
for (i = 0; i < max_size; ++i) {
vmw_write(dev_priv, SVGA_REG_DEV_CAP, i);
compat_cap->pairs[i][0] = i;
- compat_cap->pairs[i][1] = vmw_read(dev_priv, SVGA_REG_DEV_CAP);
+ compat_cap->pairs[i][1] = vmw_mask_multisample
+ (i, vmw_read(dev_priv, SVGA_REG_DEV_CAP));
}
spin_unlock(&dev_priv->cap_lock);
(struct drm_vmw_get_3d_cap_arg *) data;
struct vmw_private *dev_priv = vmw_priv(dev);
uint32_t size;
- u32 __iomem *fifo_mem;
+ u32 *fifo_mem;
void __user *buffer = (void __user *)((unsigned long)(arg->buffer));
void *bounce;
int ret;
spin_lock(&dev_priv->cap_lock);
for (i = 0; i < num; ++i) {
vmw_write(dev_priv, SVGA_REG_DEV_CAP, i);
- *bounce32++ = vmw_read(dev_priv, SVGA_REG_DEV_CAP);
+ *bounce32++ = vmw_mask_multisample
+ (i, vmw_read(dev_priv, SVGA_REG_DEV_CAP));
}
spin_unlock(&dev_priv->cap_lock);
} else if (gb_objects) {
goto out_err;
} else {
fifo_mem = dev_priv->mmio_virt;
- memcpy_fromio(bounce, &fifo_mem[SVGA_FIFO_3D_CAPS], size);
+ memcpy(bounce, &fifo_mem[SVGA_FIFO_3D_CAPS], size);
}
ret = copy_to_user(buffer, bounce, size);
void vmw_update_seqno(struct vmw_private *dev_priv,
struct vmw_fifo_state *fifo_state)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
- uint32_t seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
+ u32 *fifo_mem = dev_priv->mmio_virt;
+ uint32_t seqno = vmw_mmio_read(fifo_mem + SVGA_FIFO_FENCE);
if (dev_priv->last_read_seqno != seqno) {
dev_priv->last_read_seqno = seqno;
}
finish_wait(&dev_priv->fence_queue, &__wait);
if (ret == 0 && fifo_idle) {
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
- iowrite32(signal_seq, fifo_mem + SVGA_FIFO_FENCE);
+ u32 *fifo_mem = dev_priv->mmio_virt;
+
+ vmw_mmio_write(signal_seq, fifo_mem + SVGA_FIFO_FENCE);
}
wake_up_all(&dev_priv->fence_queue);
out_err:
void vmw_cursor_update_position(struct vmw_private *dev_priv,
bool show, int x, int y)
{
- u32 __iomem *fifo_mem = dev_priv->mmio_virt;
+ u32 *fifo_mem = dev_priv->mmio_virt;
uint32_t count;
- iowrite32(show ? 1 : 0, fifo_mem + SVGA_FIFO_CURSOR_ON);
- iowrite32(x, fifo_mem + SVGA_FIFO_CURSOR_X);
- iowrite32(y, fifo_mem + SVGA_FIFO_CURSOR_Y);
- count = ioread32(fifo_mem + SVGA_FIFO_CURSOR_COUNT);
- iowrite32(++count, fifo_mem + SVGA_FIFO_CURSOR_COUNT);
+ vmw_mmio_write(show ? 1 : 0, fifo_mem + SVGA_FIFO_CURSOR_ON);
+ vmw_mmio_write(x, fifo_mem + SVGA_FIFO_CURSOR_X);
+ vmw_mmio_write(y, fifo_mem + SVGA_FIFO_CURSOR_Y);
+ count = vmw_mmio_read(fifo_mem + SVGA_FIFO_CURSOR_COUNT);
+ vmw_mmio_write(++count, fifo_mem + SVGA_FIFO_CURSOR_COUNT);
}
int vmw_du_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
if (vmw_priv->capabilities & SVGA_CAP_PITCHLOCK)
vmw_write(vmw_priv, SVGA_REG_PITCHLOCK, pitch);
else if (vmw_fifo_have_pitchlock(vmw_priv))
- iowrite32(pitch, vmw_priv->mmio_virt + SVGA_FIFO_PITCHLOCK);
+ vmw_mmio_write(pitch, vmw_priv->mmio_virt +
+ SVGA_FIFO_PITCHLOCK);
vmw_write(vmw_priv, SVGA_REG_WIDTH, width);
vmw_write(vmw_priv, SVGA_REG_HEIGHT, height);
vmw_write(vmw_priv, SVGA_REG_BITS_PER_PIXEL, bpp);
vmw_priv->vga_pitchlock =
vmw_read(vmw_priv, SVGA_REG_PITCHLOCK);
else if (vmw_fifo_have_pitchlock(vmw_priv))
- vmw_priv->vga_pitchlock = ioread32(vmw_priv->mmio_virt +
- SVGA_FIFO_PITCHLOCK);
+ vmw_priv->vga_pitchlock = vmw_mmio_read(vmw_priv->mmio_virt +
+ SVGA_FIFO_PITCHLOCK);
if (!(vmw_priv->capabilities & SVGA_CAP_DISPLAY_TOPOLOGY))
return 0;
vmw_write(vmw_priv, SVGA_REG_PITCHLOCK,
vmw_priv->vga_pitchlock);
else if (vmw_fifo_have_pitchlock(vmw_priv))
- iowrite32(vmw_priv->vga_pitchlock,
- vmw_priv->mmio_virt + SVGA_FIFO_PITCHLOCK);
+ vmw_mmio_write(vmw_priv->vga_pitchlock,
+ vmw_priv->mmio_virt + SVGA_FIFO_PITCHLOCK);
if (!(vmw_priv->capabilities & SVGA_CAP_DISPLAY_TOPOLOGY))
return 0;
/**
* Function called by DRM code called with vbl_lock held.
*/
-u32 vmw_get_vblank_counter(struct drm_device *dev, int crtc)
+u32 vmw_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
{
return 0;
}
/**
* Function called by DRM code called with vbl_lock held.
*/
-int vmw_enable_vblank(struct drm_device *dev, int crtc)
+int vmw_enable_vblank(struct drm_device *dev, unsigned int pipe)
{
return -ENOSYS;
}
/**
* Function called by DRM code called with vbl_lock held.
*/
-void vmw_disable_vblank(struct drm_device *dev, int crtc)
+void vmw_disable_vblank(struct drm_device *dev, unsigned int pipe)
{
}
struct drm_crtc *crtc;
u32 num_units = 0;
u32 i, k;
- int ret;
dirty->dev_priv = dev_priv;
if (!dirty->cmd) {
DRM_ERROR("Couldn't reserve fifo space "
"for dirty blits.\n");
- return ret;
+ return -ENOMEM;
}
memset(dirty->cmd, 0, dirty->fifo_reserve_size);
}
goto out_unlock;
}
- ret = vmw_user_dmabuf_lookup(tfile, arg->handle, &buf);
+ ret = vmw_user_dmabuf_lookup(tfile, arg->handle, &buf, NULL);
if (ret)
goto out_unlock;
}
*out_surf = NULL;
- ret = vmw_user_dmabuf_lookup(tfile, handle, out_buf);
+ ret = vmw_user_dmabuf_lookup(tfile, handle, out_buf, NULL);
return ret;
}
uint32_t size,
bool shareable,
uint32_t *handle,
- struct vmw_dma_buffer **p_dma_buf)
+ struct vmw_dma_buffer **p_dma_buf,
+ struct ttm_base_object **p_base)
{
struct vmw_user_dma_buffer *user_bo;
struct ttm_buffer_object *tmp;
}
*p_dma_buf = &user_bo->dma;
+ if (p_base) {
+ *p_base = &user_bo->prime.base;
+ kref_get(&(*p_base)->refcount);
+ }
*handle = user_bo->prime.base.hash.key;
out_no_base_object:
struct vmw_dma_buffer *dma_buf;
struct vmw_user_dma_buffer *user_bo;
struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+ struct ttm_base_object *buffer_base;
int ret;
if ((arg->flags & (drm_vmw_synccpu_read | drm_vmw_synccpu_write)) == 0
switch (arg->op) {
case drm_vmw_synccpu_grab:
- ret = vmw_user_dmabuf_lookup(tfile, arg->handle, &dma_buf);
+ ret = vmw_user_dmabuf_lookup(tfile, arg->handle, &dma_buf,
+ &buffer_base);
if (unlikely(ret != 0))
return ret;
dma);
ret = vmw_user_dmabuf_synccpu_grab(user_bo, tfile, arg->flags);
vmw_dmabuf_unreference(&dma_buf);
+ ttm_base_object_unref(&buffer_base);
if (unlikely(ret != 0 && ret != -ERESTARTSYS &&
ret != -EBUSY)) {
DRM_ERROR("Failed synccpu grab on handle 0x%08x.\n",
return ret;
ret = vmw_user_dmabuf_alloc(dev_priv, vmw_fpriv(file_priv)->tfile,
- req->size, false, &handle, &dma_buf);
+ req->size, false, &handle, &dma_buf,
+ NULL);
if (unlikely(ret != 0))
goto out_no_dmabuf;
}
int vmw_user_dmabuf_lookup(struct ttm_object_file *tfile,
- uint32_t handle, struct vmw_dma_buffer **out)
+ uint32_t handle, struct vmw_dma_buffer **out,
+ struct ttm_base_object **p_base)
{
struct vmw_user_dma_buffer *vmw_user_bo;
struct ttm_base_object *base;
vmw_user_bo = container_of(base, struct vmw_user_dma_buffer,
prime.base);
(void)ttm_bo_reference(&vmw_user_bo->dma.base);
- ttm_base_object_unref(&base);
+ if (p_base)
+ *p_base = base;
+ else
+ ttm_base_object_unref(&base);
*out = &vmw_user_bo->dma;
return 0;
ret = vmw_user_dmabuf_alloc(dev_priv, vmw_fpriv(file_priv)->tfile,
args->size, false, &args->handle,
- &dma_buf);
+ &dma_buf, NULL);
if (unlikely(ret != 0))
goto out_no_dmabuf;
struct vmw_dma_buffer *out_buf;
int ret;
- ret = vmw_user_dmabuf_lookup(tfile, handle, &out_buf);
+ ret = vmw_user_dmabuf_lookup(tfile, handle, &out_buf, NULL);
if (ret != 0)
return -EINVAL;
if (buffer_handle != SVGA3D_INVALID_ID) {
ret = vmw_user_dmabuf_lookup(tfile, buffer_handle,
- &buffer);
+ &buffer, NULL);
if (unlikely(ret != 0)) {
DRM_ERROR("Could not find buffer for shader "
"creation.\n");
struct vmw_surface srf;
uint32_t size;
struct drm_master *master;
+ struct ttm_base_object *backup_base;
};
/**
struct vmw_resource *res = &user_srf->srf.res;
*p_base = NULL;
+ ttm_base_object_unref(&user_srf->backup_base);
vmw_resource_unreference(&res);
}
res->backup_size,
true,
&backup_handle,
- &res->backup);
+ &res->backup,
+ &user_srf->backup_base);
if (unlikely(ret != 0)) {
vmw_resource_unreference(&res);
goto out_unlock;
uint32_t size;
uint32_t backup_handle;
+ if (req->multisample_count != 0)
+ return -EINVAL;
if (unlikely(vmw_user_surface_size == 0))
vmw_user_surface_size = ttm_round_pot(sizeof(*user_srf)) +
if (req->buffer_handle != SVGA3D_INVALID_ID) {
ret = vmw_user_dmabuf_lookup(tfile, req->buffer_handle,
- &res->backup);
+ &res->backup,
+ &user_srf->backup_base);
if (ret == 0 && res->backup->base.num_pages * PAGE_SIZE <
res->backup_size) {
DRM_ERROR("Surface backup buffer is too small.\n");
req->drm_surface_flags &
drm_vmw_surface_flag_shareable,
&backup_handle,
- &res->backup);
+ &res->backup,
+ &user_srf->backup_base);
if (unlikely(ret != 0)) {
vmw_resource_unreference(&res);
enum ipu_color_space ipu_drm_fourcc_to_colorspace(u32 drm_fourcc)
{
switch (drm_fourcc) {
+ case DRM_FORMAT_ARGB1555:
+ case DRM_FORMAT_ABGR1555:
+ case DRM_FORMAT_RGBA5551:
+ case DRM_FORMAT_BGRA5551:
case DRM_FORMAT_RGB565:
case DRM_FORMAT_BGR565:
case DRM_FORMAT_RGB888:
case DRM_FORMAT_BGR888:
+ case DRM_FORMAT_ARGB4444:
case DRM_FORMAT_XRGB8888:
case DRM_FORMAT_XBGR8888:
case DRM_FORMAT_RGBX8888:
}
}
-static void ipu_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ipu_irq_handler(struct irq_desc *desc)
{
struct ipu_soc *ipu = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
chained_irq_exit(chip, desc);
}
-static void ipu_err_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void ipu_err_irq_handler(struct irq_desc *desc)
{
struct ipu_soc *ipu = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
}
ret = irq_alloc_domain_generic_chips(ipu->domain, 32, 1, "IPU",
- handle_level_irq, 0,
- IRQF_VALID, 0);
+ handle_level_irq, 0, 0, 0);
if (ret < 0) {
dev_err(ipu->dev, "failed to alloc generic irq chips\n");
irq_domain_remove(ipu->domain);
}
EXPORT_SYMBOL_GPL(ipu_cpmem_set_yuv_planar);
-static const struct ipu_rgb def_rgb_32 = {
+static const struct ipu_rgb def_xrgb_32 = {
.red = { .offset = 16, .length = 8, },
.green = { .offset = 8, .length = 8, },
.blue = { .offset = 0, .length = 8, },
.bits_per_pixel = 32,
};
-static const struct ipu_rgb def_bgr_32 = {
+static const struct ipu_rgb def_xbgr_32 = {
.red = { .offset = 0, .length = 8, },
.green = { .offset = 8, .length = 8, },
.blue = { .offset = 16, .length = 8, },
.bits_per_pixel = 32,
};
+static const struct ipu_rgb def_rgbx_32 = {
+ .red = { .offset = 24, .length = 8, },
+ .green = { .offset = 16, .length = 8, },
+ .blue = { .offset = 8, .length = 8, },
+ .transp = { .offset = 0, .length = 8, },
+ .bits_per_pixel = 32,
+};
+
+static const struct ipu_rgb def_bgrx_32 = {
+ .red = { .offset = 8, .length = 8, },
+ .green = { .offset = 16, .length = 8, },
+ .blue = { .offset = 24, .length = 8, },
+ .transp = { .offset = 0, .length = 8, },
+ .bits_per_pixel = 32,
+};
+
static const struct ipu_rgb def_rgb_24 = {
.red = { .offset = 16, .length = 8, },
.green = { .offset = 8, .length = 8, },
.bits_per_pixel = 16,
};
+static const struct ipu_rgb def_argb_16 = {
+ .red = { .offset = 10, .length = 5, },
+ .green = { .offset = 5, .length = 5, },
+ .blue = { .offset = 0, .length = 5, },
+ .transp = { .offset = 15, .length = 1, },
+ .bits_per_pixel = 16,
+};
+
+static const struct ipu_rgb def_argb_16_4444 = {
+ .red = { .offset = 8, .length = 4, },
+ .green = { .offset = 4, .length = 4, },
+ .blue = { .offset = 0, .length = 4, },
+ .transp = { .offset = 12, .length = 4, },
+ .bits_per_pixel = 16,
+};
+
+static const struct ipu_rgb def_abgr_16 = {
+ .red = { .offset = 0, .length = 5, },
+ .green = { .offset = 5, .length = 5, },
+ .blue = { .offset = 10, .length = 5, },
+ .transp = { .offset = 15, .length = 1, },
+ .bits_per_pixel = 16,
+};
+
+static const struct ipu_rgb def_rgba_16 = {
+ .red = { .offset = 11, .length = 5, },
+ .green = { .offset = 6, .length = 5, },
+ .blue = { .offset = 1, .length = 5, },
+ .transp = { .offset = 0, .length = 1, },
+ .bits_per_pixel = 16,
+};
+
+static const struct ipu_rgb def_bgra_16 = {
+ .red = { .offset = 1, .length = 5, },
+ .green = { .offset = 6, .length = 5, },
+ .blue = { .offset = 11, .length = 5, },
+ .transp = { .offset = 0, .length = 1, },
+ .bits_per_pixel = 16,
+};
+
#define Y_OFFSET(pix, x, y) ((x) + pix->width * (y))
#define U_OFFSET(pix, x, y) ((pix->width * pix->height) + \
(pix->width * (y) / 4) + (x) / 2)
break;
case DRM_FORMAT_ABGR8888:
case DRM_FORMAT_XBGR8888:
- ipu_cpmem_set_format_rgb(ch, &def_bgr_32);
+ ipu_cpmem_set_format_rgb(ch, &def_xbgr_32);
break;
case DRM_FORMAT_ARGB8888:
case DRM_FORMAT_XRGB8888:
- ipu_cpmem_set_format_rgb(ch, &def_rgb_32);
+ ipu_cpmem_set_format_rgb(ch, &def_xrgb_32);
+ break;
+ case DRM_FORMAT_RGBA8888:
+ case DRM_FORMAT_RGBX8888:
+ ipu_cpmem_set_format_rgb(ch, &def_rgbx_32);
+ break;
+ case DRM_FORMAT_BGRA8888:
+ case DRM_FORMAT_BGRX8888:
+ ipu_cpmem_set_format_rgb(ch, &def_bgrx_32);
break;
case DRM_FORMAT_BGR888:
ipu_cpmem_set_format_rgb(ch, &def_bgr_24);
case DRM_FORMAT_BGR565:
ipu_cpmem_set_format_rgb(ch, &def_bgr_16);
break;
+ case DRM_FORMAT_ARGB1555:
+ ipu_cpmem_set_format_rgb(ch, &def_argb_16);
+ break;
+ case DRM_FORMAT_ABGR1555:
+ ipu_cpmem_set_format_rgb(ch, &def_abgr_16);
+ break;
+ case DRM_FORMAT_RGBA5551:
+ ipu_cpmem_set_format_rgb(ch, &def_rgba_16);
+ break;
+ case DRM_FORMAT_BGRA5551:
+ ipu_cpmem_set_format_rgb(ch, &def_bgra_16);
+ break;
+ case DRM_FORMAT_ARGB4444:
+ ipu_cpmem_set_format_rgb(ch, &def_argb_16_4444);
+ break;
default:
return -EINVAL;
}
u32 ipu_clk)
{
u32 temp;
- u32 div_ratio;
+ int div_ratio;
div_ratio = (ipu_clk / pixel_clk) - 1;
case MEDIA_BUS_FMT_SGBRG8_1X8:
case MEDIA_BUS_FMT_SGRBG8_1X8:
case MEDIA_BUS_FMT_SRGGB8_1X8:
+ case MEDIA_BUS_FMT_Y8_1X8:
cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_BAYER;
cfg->mipi_dt = MIPI_DT_RAW8;
cfg->data_width = IPU_CSI_DATA_WIDTH_8;
temp = ipu_csi_read(csi, CSI_TST_CTRL);
- if (active == false) {
+ if (!active) {
temp &= ~CSI_TEST_GEN_MODE_EN;
ipu_csi_write(csi, temp, CSI_TST_CTRL);
} else {
}
if (interlaced) {
- dc_link_event(dc, DC_EVT_NL, 0, 3);
- dc_link_event(dc, DC_EVT_EOL, 0, 2);
- dc_link_event(dc, DC_EVT_NEW_DATA, 0, 1);
+ int addr;
+
+ if (dc->di)
+ addr = 1;
+ else
+ addr = 0;
+
+ dc_link_event(dc, DC_EVT_NL, addr, 3);
+ dc_link_event(dc, DC_EVT_EOL, addr, 2);
+ dc_link_event(dc, DC_EVT_NEW_DATA, addr, 1);
/* Init template microcode */
- dc_write_tmpl(dc, 0, WROD(0), 0, map, SYNC_WAVE, 0, 8, 1);
+ dc_write_tmpl(dc, addr, WROD(0), 0, map, SYNC_WAVE, 0, 6, 1);
} else {
if (dc->di) {
dc_link_event(dc, DC_EVT_NL, 2, 3);
DI_SYNC_HSYNC = 3,
DI_SYNC_VSYNC = 4,
DI_SYNC_DE = 6,
+
+ DI_SYNC_CNT1 = 2, /* counter >= 2 only */
+ DI_SYNC_CNT4 = 5, /* counter >= 5 only */
+ DI_SYNC_CNT5 = 6, /* counter >= 6 only */
};
#define SYNC_WAVE 0
sig->mode.hback_porch + sig->mode.hfront_porch;
u32 v_total = sig->mode.vactive + sig->mode.vsync_len +
sig->mode.vback_porch + sig->mode.vfront_porch;
- u32 reg;
struct di_sync_config cfg[] = {
{
- .run_count = h_total / 2 - 1,
- .run_src = DI_SYNC_CLK,
+ /* 1: internal VSYNC for each frame */
+ .run_count = v_total * 2 - 1,
+ .run_src = 3, /* == counter 7 */
}, {
- .run_count = h_total - 11,
+ /* PIN2: HSYNC waveform */
+ .run_count = h_total - 1,
.run_src = DI_SYNC_CLK,
- .cnt_down = 4,
+ .cnt_polarity_gen_en = 1,
+ .cnt_polarity_trigger_src = DI_SYNC_CLK,
+ .cnt_down = sig->mode.hsync_len * 2,
}, {
- .run_count = v_total * 2 - 1,
- .run_src = DI_SYNC_INT_HSYNC,
- .offset_count = 1,
- .offset_src = DI_SYNC_INT_HSYNC,
- .cnt_down = 4,
+ /* PIN3: VSYNC waveform */
+ .run_count = v_total - 1,
+ .run_src = 4, /* == counter 7 */
+ .cnt_polarity_gen_en = 1,
+ .cnt_polarity_trigger_src = 4, /* == counter 7 */
+ .cnt_down = sig->mode.vsync_len * 2,
+ .cnt_clr_src = DI_SYNC_CNT1,
}, {
- .run_count = v_total / 2 - 1,
+ /* 4: Field */
+ .run_count = v_total / 2,
.run_src = DI_SYNC_HSYNC,
- .offset_count = sig->mode.vback_porch,
- .offset_src = DI_SYNC_HSYNC,
+ .offset_count = h_total / 2,
+ .offset_src = DI_SYNC_CLK,
.repeat_count = 2,
- .cnt_clr_src = DI_SYNC_VSYNC,
- }, {
- .run_src = DI_SYNC_HSYNC,
- .repeat_count = sig->mode.vactive / 2,
- .cnt_clr_src = 4,
- }, {
- .run_count = v_total - 1,
- .run_src = DI_SYNC_HSYNC,
+ .cnt_clr_src = DI_SYNC_CNT1,
}, {
- .run_count = v_total / 2 - 1,
+ /* 5: Active lines */
.run_src = DI_SYNC_HSYNC,
- .offset_count = 9,
+ .offset_count = (sig->mode.vsync_len +
+ sig->mode.vback_porch) / 2,
.offset_src = DI_SYNC_HSYNC,
- .repeat_count = 2,
- .cnt_clr_src = DI_SYNC_VSYNC,
+ .repeat_count = sig->mode.vactive / 2,
+ .cnt_clr_src = DI_SYNC_CNT4,
}, {
+ /* 6: Active pixel, referenced by DC */
.run_src = DI_SYNC_CLK,
- .offset_count = sig->mode.hback_porch,
+ .offset_count = sig->mode.hsync_len +
+ sig->mode.hback_porch,
.offset_src = DI_SYNC_CLK,
.repeat_count = sig->mode.hactive,
- .cnt_clr_src = 5,
+ .cnt_clr_src = DI_SYNC_CNT5,
}, {
- .run_count = v_total - 1,
- .run_src = DI_SYNC_INT_HSYNC,
- .offset_count = v_total / 2,
- .offset_src = DI_SYNC_INT_HSYNC,
- .cnt_clr_src = DI_SYNC_HSYNC,
- .cnt_down = 4,
+ /* 7: Half line HSYNC */
+ .run_count = h_total / 2 - 1,
+ .run_src = DI_SYNC_CLK,
}
};
ipu_di_sync_config(di, cfg, 0, ARRAY_SIZE(cfg));
- /* set gentime select and tag sel */
- reg = ipu_di_read(di, DI_SW_GEN1(9));
- reg &= 0x1FFFFFFF;
- reg |= (3 - 1) << 29 | 0x00008000;
- ipu_di_write(di, reg, DI_SW_GEN1(9));
-
ipu_di_write(di, v_total / 2 - 1, DI_SCR_CONF);
}
}
EXPORT_SYMBOL_GPL(ipu_di_adjust_videomode);
+static u32 ipu_di_gen_polarity(int pin)
+{
+ switch (pin) {
+ case 1:
+ return DI_GEN_POLARITY_1;
+ case 2:
+ return DI_GEN_POLARITY_2;
+ case 3:
+ return DI_GEN_POLARITY_3;
+ case 4:
+ return DI_GEN_POLARITY_4;
+ case 5:
+ return DI_GEN_POLARITY_5;
+ case 6:
+ return DI_GEN_POLARITY_6;
+ case 7:
+ return DI_GEN_POLARITY_7;
+ case 8:
+ return DI_GEN_POLARITY_8;
+ }
+ return 0;
+}
+
int ipu_di_init_sync_panel(struct ipu_di *di, struct ipu_di_signal_cfg *sig)
{
u32 reg;
/* set y_sel = 1 */
di_gen |= 0x10000000;
- di_gen |= DI_GEN_POLARITY_5;
- di_gen |= DI_GEN_POLARITY_8;
-
- vsync_cnt = 7;
- if (sig->mode.flags & DISPLAY_FLAGS_HSYNC_HIGH)
- di_gen |= DI_GEN_POLARITY_3;
- if (sig->mode.flags & DISPLAY_FLAGS_VSYNC_HIGH)
- di_gen |= DI_GEN_POLARITY_2;
+ vsync_cnt = 3;
} else {
ipu_di_sync_config_noninterlaced(di, sig, div);
*/
if (!(sig->hsync_pin == 2 && sig->vsync_pin == 3))
vsync_cnt = 6;
-
- if (sig->mode.flags & DISPLAY_FLAGS_HSYNC_HIGH) {
- if (sig->hsync_pin == 2)
- di_gen |= DI_GEN_POLARITY_2;
- else if (sig->hsync_pin == 4)
- di_gen |= DI_GEN_POLARITY_4;
- else if (sig->hsync_pin == 7)
- di_gen |= DI_GEN_POLARITY_7;
- }
- if (sig->mode.flags & DISPLAY_FLAGS_VSYNC_HIGH) {
- if (sig->vsync_pin == 3)
- di_gen |= DI_GEN_POLARITY_3;
- else if (sig->vsync_pin == 6)
- di_gen |= DI_GEN_POLARITY_6;
- else if (sig->vsync_pin == 8)
- di_gen |= DI_GEN_POLARITY_8;
- }
}
+ if (sig->mode.flags & DISPLAY_FLAGS_HSYNC_HIGH)
+ di_gen |= ipu_di_gen_polarity(sig->hsync_pin);
+ if (sig->mode.flags & DISPLAY_FLAGS_VSYNC_HIGH)
+ di_gen |= ipu_di_gen_polarity(sig->vsync_pin);
+
if (sig->clk_pol)
di_gen |= DI_GEN_POLARITY_DISP_CLK;
/*
+ * vga_switcheroo.c - Support for laptop with dual GPU using one set of outputs
+ *
* Copyright (c) 2010 Red Hat Inc.
* Author : Dave Airlie <airlied@redhat.com>
*
+ * Copyright (c) 2015 Lukas Wunner <lukas@wunner.de>
*
- * Licensed under GPLv2
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
*
- * vga_switcheroo.c - Support for laptop with dual GPU using one set of outputs
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
*
- * Switcher interface - methods require for ATPX and DCM
- * - switchto - this throws the output MUX switch
- * - discrete_set_power - sets the power state for the discrete card
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS
+ * IN THE SOFTWARE.
*
- * GPU driver interface
- * - set_gpu_state - this should do the equiv of s/r for the card
- * - this should *not* set the discrete power state
- * - switch_check - check if the device is in a position to switch now
*/
#define pr_fmt(fmt) "vga_switcheroo: " fmt
-#include <linux/module.h>
-#include <linux/seq_file.h>
-#include <linux/uaccess.h>
-#include <linux/fs.h>
+#include <linux/console.h>
#include <linux/debugfs.h>
#include <linux/fb.h>
-
+#include <linux/fs.h>
+#include <linux/module.h>
#include <linux/pci.h>
-#include <linux/console.h>
-#include <linux/vga_switcheroo.h>
#include <linux/pm_runtime.h>
-
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
#include <linux/vgaarb.h>
+#include <linux/vga_switcheroo.h>
+
+/**
+ * DOC: Overview
+ *
+ * vga_switcheroo is the Linux subsystem for laptop hybrid graphics.
+ * These come in two flavors:
+ *
+ * * muxed: Dual GPUs with a multiplexer chip to switch outputs between GPUs.
+ * * muxless: Dual GPUs but only one of them is connected to outputs.
+ * The other one is merely used to offload rendering, its results
+ * are copied over PCIe into the framebuffer. On Linux this is
+ * supported with DRI PRIME.
+ *
+ * Hybrid graphics started to appear in the late Naughties and were initially
+ * all muxed. Newer laptops moved to a muxless architecture for cost reasons.
+ * A notable exception is the MacBook Pro which continues to use a mux.
+ * Muxes come with varying capabilities: Some switch only the panel, others
+ * can also switch external displays. Some switch all display pins at once
+ * while others can switch just the DDC lines. (To allow EDID probing
+ * for the inactive GPU.) Also, muxes are often used to cut power to the
+ * discrete GPU while it is not used.
+ *
+ * DRM drivers register GPUs with vga_switcheroo, these are heretoforth called
+ * clients. The mux is called the handler. Muxless machines also register a
+ * handler to control the power state of the discrete GPU, its ->switchto
+ * callback is a no-op for obvious reasons. The discrete GPU is often equipped
+ * with an HDA controller for the HDMI/DP audio signal, this will also
+ * register as a client so that vga_switcheroo can take care of the correct
+ * suspend/resume order when changing the discrete GPU's power state. In total
+ * there can thus be up to three clients: Two vga clients (GPUs) and one audio
+ * client (on the discrete GPU). The code is mostly prepared to support
+ * machines with more than two GPUs should they become available.
+ * The GPU to which the outputs are currently switched is called the
+ * active client in vga_switcheroo parlance. The GPU not in use is the
+ * inactive client.
+ */
+/**
+ * struct vga_switcheroo_client - registered client
+ * @pdev: client pci device
+ * @fb_info: framebuffer to which console is remapped on switching
+ * @pwr_state: current power state
+ * @ops: client callbacks
+ * @id: client identifier. Determining the id requires the handler,
+ * so gpus are initially assigned VGA_SWITCHEROO_UNKNOWN_ID
+ * and later given their true id in vga_switcheroo_enable()
+ * @active: whether the outputs are currently switched to this client
+ * @driver_power_control: whether power state is controlled by the driver's
+ * runtime pm. If true, writing ON and OFF to the vga_switcheroo debugfs
+ * interface is a no-op so as not to interfere with runtime pm
+ * @list: client list
+ *
+ * Registered client. A client can be either a GPU or an audio device on a GPU.
+ * For audio clients, the @fb_info, @active and @driver_power_control members
+ * are bogus.
+ */
struct vga_switcheroo_client {
struct pci_dev *pdev;
struct fb_info *fb_info;
- int pwr_state;
+ enum vga_switcheroo_state pwr_state;
const struct vga_switcheroo_client_ops *ops;
- int id;
+ enum vga_switcheroo_client_id id;
bool active;
bool driver_power_control;
struct list_head list;
};
+/*
+ * protects access to struct vgasr_priv
+ */
static DEFINE_MUTEX(vgasr_mutex);
+/**
+ * struct vgasr_priv - vga_switcheroo private data
+ * @active: whether vga_switcheroo is enabled.
+ * Prerequisite is the registration of two GPUs and a handler
+ * @delayed_switch_active: whether a delayed switch is pending
+ * @delayed_client_id: client to which a delayed switch is pending
+ * @debugfs_root: directory for vga_switcheroo debugfs interface
+ * @switch_file: file for vga_switcheroo debugfs interface
+ * @registered_clients: number of registered GPUs
+ * (counting only vga clients, not audio clients)
+ * @clients: list of registered clients
+ * @handler: registered handler
+ *
+ * vga_switcheroo private data. Currently only one vga_switcheroo instance
+ * per system is supported.
+ */
struct vgasr_priv {
-
bool active;
bool delayed_switch_active;
enum vga_switcheroo_client_id delayed_client_id;
int registered_clients;
struct list_head clients;
- struct vga_switcheroo_handler *handler;
+ const struct vga_switcheroo_handler *handler;
};
#define ID_BIT_AUDIO 0x100
#define client_is_audio(c) ((c)->id & ID_BIT_AUDIO)
-#define client_is_vga(c) ((c)->id == -1 || !client_is_audio(c))
+#define client_is_vga(c) ((c)->id == VGA_SWITCHEROO_UNKNOWN_ID || \
+ !client_is_audio(c))
#define client_id(c) ((c)->id & ~ID_BIT_AUDIO)
static int vga_switcheroo_debugfs_init(struct vgasr_priv *priv);
vgasr_priv.handler->init();
list_for_each_entry(client, &vgasr_priv.clients, list) {
- if (client->id != -1)
+ if (client->id != VGA_SWITCHEROO_UNKNOWN_ID)
continue;
ret = vgasr_priv.handler->get_client_id(client->pdev);
if (ret < 0)
vgasr_priv.active = true;
}
-int vga_switcheroo_register_handler(struct vga_switcheroo_handler *handler)
+/**
+ * vga_switcheroo_register_handler() - register handler
+ * @handler: handler callbacks
+ *
+ * Register handler. Enable vga_switcheroo if two vga clients have already
+ * registered.
+ *
+ * Return: 0 on success, -EINVAL if a handler was already registered.
+ */
+int vga_switcheroo_register_handler(const struct vga_switcheroo_handler *handler)
{
mutex_lock(&vgasr_mutex);
if (vgasr_priv.handler) {
}
EXPORT_SYMBOL(vga_switcheroo_register_handler);
+/**
+ * vga_switcheroo_unregister_handler() - unregister handler
+ *
+ * Unregister handler. Disable vga_switcheroo.
+ */
void vga_switcheroo_unregister_handler(void)
{
mutex_lock(&vgasr_mutex);
static int register_client(struct pci_dev *pdev,
const struct vga_switcheroo_client_ops *ops,
- int id, bool active, bool driver_power_control)
+ enum vga_switcheroo_client_id id, bool active,
+ bool driver_power_control)
{
struct vga_switcheroo_client *client;
return 0;
}
+/**
+ * vga_switcheroo_register_client - register vga client
+ * @pdev: client pci device
+ * @ops: client callbacks
+ * @driver_power_control: whether power state is controlled by the driver's
+ * runtime pm
+ *
+ * Register vga client (GPU). Enable vga_switcheroo if another GPU and a
+ * handler have already registered. The power state of the client is assumed
+ * to be ON.
+ *
+ * Return: 0 on success, -ENOMEM on memory allocation error.
+ */
int vga_switcheroo_register_client(struct pci_dev *pdev,
const struct vga_switcheroo_client_ops *ops,
bool driver_power_control)
{
- return register_client(pdev, ops, -1,
+ return register_client(pdev, ops, VGA_SWITCHEROO_UNKNOWN_ID,
pdev == vga_default_device(),
driver_power_control);
}
EXPORT_SYMBOL(vga_switcheroo_register_client);
+/**
+ * vga_switcheroo_register_audio_client - register audio client
+ * @pdev: client pci device
+ * @ops: client callbacks
+ * @id: client identifier
+ *
+ * Register audio client (audio device on a GPU). The power state of the
+ * client is assumed to be ON.
+ *
+ * Return: 0 on success, -ENOMEM on memory allocation error.
+ */
int vga_switcheroo_register_audio_client(struct pci_dev *pdev,
const struct vga_switcheroo_client_ops *ops,
- int id, bool active)
+ enum vga_switcheroo_client_id id)
{
- return register_client(pdev, ops, id | ID_BIT_AUDIO, active, false);
+ return register_client(pdev, ops, id | ID_BIT_AUDIO, false, false);
}
EXPORT_SYMBOL(vga_switcheroo_register_audio_client);
}
static struct vga_switcheroo_client *
-find_client_from_id(struct list_head *head, int client_id)
+find_client_from_id(struct list_head *head,
+ enum vga_switcheroo_client_id client_id)
{
struct vga_switcheroo_client *client;
struct vga_switcheroo_client *client;
list_for_each_entry(client, head, list)
- if (client->active && client_is_vga(client))
+ if (client->active)
return client;
return NULL;
}
-int vga_switcheroo_get_client_state(struct pci_dev *pdev)
+/**
+ * vga_switcheroo_get_client_state() - obtain power state of a given client
+ * @pdev: client pci device
+ *
+ * Obtain power state of a given client as seen from vga_switcheroo.
+ * The function is only called from hda_intel.c.
+ *
+ * Return: Power state.
+ */
+enum vga_switcheroo_state vga_switcheroo_get_client_state(struct pci_dev *pdev)
{
struct vga_switcheroo_client *client;
+ enum vga_switcheroo_state ret;
+ mutex_lock(&vgasr_mutex);
client = find_client_from_pci(&vgasr_priv.clients, pdev);
if (!client)
- return VGA_SWITCHEROO_NOT_FOUND;
- if (!vgasr_priv.active)
- return VGA_SWITCHEROO_INIT;
- return client->pwr_state;
+ ret = VGA_SWITCHEROO_NOT_FOUND;
+ else if (!vgasr_priv.active)
+ ret = VGA_SWITCHEROO_INIT;
+ else
+ ret = client->pwr_state;
+ mutex_unlock(&vgasr_mutex);
+ return ret;
}
EXPORT_SYMBOL(vga_switcheroo_get_client_state);
+/**
+ * vga_switcheroo_unregister_client() - unregister client
+ * @pdev: client pci device
+ *
+ * Unregister client. Disable vga_switcheroo if this is a vga client (GPU).
+ */
void vga_switcheroo_unregister_client(struct pci_dev *pdev)
{
struct vga_switcheroo_client *client;
}
EXPORT_SYMBOL(vga_switcheroo_unregister_client);
+/**
+ * vga_switcheroo_client_fb_set() - set framebuffer of a given client
+ * @pdev: client pci device
+ * @info: framebuffer
+ *
+ * Set framebuffer of a given client. The console will be remapped to this
+ * on switching.
+ */
void vga_switcheroo_client_fb_set(struct pci_dev *pdev,
struct fb_info *info)
{
}
EXPORT_SYMBOL(vga_switcheroo_client_fb_set);
+/**
+ * DOC: Manual switching and manual power control
+ *
+ * In this mode of use, the file /sys/kernel/debug/vgaswitcheroo/switch
+ * can be read to retrieve the current vga_switcheroo state and commands
+ * can be written to it to change the state. The file appears as soon as
+ * two GPU drivers and one handler have registered with vga_switcheroo.
+ * The following commands are understood:
+ *
+ * * OFF: Power off the device not in use.
+ * * ON: Power on the device not in use.
+ * * IGD: Switch to the integrated graphics device.
+ * Power on the integrated GPU if necessary, power off the discrete GPU.
+ * Prerequisite is that no user space processes (e.g. Xorg, alsactl)
+ * have opened device files of the GPUs or the audio client. If the
+ * switch fails, the user may invoke lsof(8) or fuser(1) on /dev/dri/
+ * and /dev/snd/controlC1 to identify processes blocking the switch.
+ * * DIS: Switch to the discrete graphics device.
+ * * DIGD: Delayed switch to the integrated graphics device.
+ * This will perform the switch once the last user space process has
+ * closed the device files of the GPUs and the audio client.
+ * * DDIS: Delayed switch to the discrete graphics device.
+ * * MIGD: Mux-only switch to the integrated graphics device.
+ * Does not remap console or change the power state of either gpu.
+ * If the integrated GPU is currently off, the screen will turn black.
+ * If it is on, the screen will show whatever happens to be in VRAM.
+ * Either way, the user has to blindly enter the command to switch back.
+ * * MDIS: Mux-only switch to the discrete graphics device.
+ *
+ * For GPUs whose power state is controlled by the driver's runtime pm,
+ * the ON and OFF commands are a no-op (see next section).
+ *
+ * For muxless machines, the IGD/DIS, DIGD/DDIS and MIGD/MDIS commands
+ * should not be used.
+ */
+
static int vga_switcheroo_show(struct seq_file *m, void *v)
{
struct vga_switcheroo_client *client;
return 0;
}
-static void set_audio_state(int id, int state)
+static void set_audio_state(enum vga_switcheroo_client_id id,
+ enum vga_switcheroo_state state)
{
struct vga_switcheroo_client *client;
int ret;
bool delay = false, can_switch;
bool just_mux = false;
- int client_id = -1;
+ enum vga_switcheroo_client_id client_id = VGA_SWITCHEROO_UNKNOWN_ID;
struct vga_switcheroo_client *client = NULL;
if (cnt > 63)
client_id = VGA_SWITCHEROO_DIS;
}
- if (client_id == -1)
+ if (client_id == VGA_SWITCHEROO_UNKNOWN_ID)
goto out;
client = find_client_from_id(&vgasr_priv.clients, client_id);
if (!client)
return -1;
}
+/**
+ * vga_switcheroo_process_delayed_switch() - helper for delayed switching
+ *
+ * Process a delayed switch if one is pending. DRM drivers should call this
+ * from their ->lastclose callback.
+ *
+ * Return: 0 on success. -EINVAL if no delayed switch is pending, if the client
+ * has unregistered in the meantime or if there are other clients blocking the
+ * switch. If the actual switch fails, an error is reported and 0 is returned.
+ */
int vga_switcheroo_process_delayed_switch(void)
{
struct vga_switcheroo_client *client;
}
EXPORT_SYMBOL(vga_switcheroo_process_delayed_switch);
+/**
+ * DOC: Driver power control
+ *
+ * In this mode of use, the discrete GPU automatically powers up and down at
+ * the discretion of the driver's runtime pm. On muxed machines, the user may
+ * still influence the muxer state by way of the debugfs interface, however
+ * the ON and OFF commands become a no-op for the discrete GPU.
+ *
+ * This mode is the default on Nvidia HybridPower/Optimus and ATI PowerXpress.
+ * Specifying nouveau.runpm=0, radeon.runpm=0 or amdgpu.runpm=0 on the kernel
+ * command line disables it.
+ *
+ * When the driver decides to power up or down, it notifies vga_switcheroo
+ * thereof so that it can (a) power the audio device on the GPU up or down,
+ * and (b) update its internal power state representation for the device.
+ * This is achieved by vga_switcheroo_set_dynamic_switch().
+ *
+ * After the GPU has been suspended, the handler needs to be called to cut
+ * power to the GPU. Likewise it needs to reinstate power before the GPU
+ * can resume. This is achieved by vga_switcheroo_init_domain_pm_ops(),
+ * which augments the GPU's suspend/resume functions by the requisite
+ * calls to the handler.
+ *
+ * When the audio device resumes, the GPU needs to be woken. This is achieved
+ * by vga_switcheroo_init_domain_pm_optimus_hdmi_audio(), which augments the
+ * audio device's resume function.
+ *
+ * On muxed machines, if the mux is initially switched to the discrete GPU,
+ * the user ends up with a black screen when the GPU powers down after boot.
+ * As a workaround, the mux is forced to the integrated GPU on runtime suspend,
+ * cf. https://bugs.freedesktop.org/show_bug.cgi?id=75917
+ */
+
static void vga_switcheroo_power_switch(struct pci_dev *pdev,
enum vga_switcheroo_state state)
{
vgasr_priv.handler->power_state(client->id, state);
}
-/* force a PCI device to a certain state - mainly to turn off audio clients */
-
+/**
+ * vga_switcheroo_set_dynamic_switch() - helper for driver power control
+ * @pdev: client pci device
+ * @dynamic: new power state
+ *
+ * Helper for GPUs whose power state is controlled by the driver's runtime pm.
+ * When the driver decides to power up or down, it notifies vga_switcheroo
+ * thereof using this helper so that it can (a) power the audio device on
+ * the GPU up or down, and (b) update its internal power state representation
+ * for the device.
+ */
void vga_switcheroo_set_dynamic_switch(struct pci_dev *pdev,
enum vga_switcheroo_state dynamic)
{
struct vga_switcheroo_client *client;
+ mutex_lock(&vgasr_mutex);
client = find_client_from_pci(&vgasr_priv.clients, pdev);
- if (!client)
- return;
-
- if (!client->driver_power_control)
+ if (!client || !client->driver_power_control) {
+ mutex_unlock(&vgasr_mutex);
return;
+ }
client->pwr_state = dynamic;
set_audio_state(client->id, dynamic);
+ mutex_unlock(&vgasr_mutex);
}
EXPORT_SYMBOL(vga_switcheroo_set_dynamic_switch);
ret = dev->bus->pm->runtime_suspend(dev);
if (ret)
return ret;
+ mutex_lock(&vgasr_mutex);
if (vgasr_priv.handler->switchto)
vgasr_priv.handler->switchto(VGA_SWITCHEROO_IGD);
vga_switcheroo_power_switch(pdev, VGA_SWITCHEROO_OFF);
+ mutex_unlock(&vgasr_mutex);
return 0;
}
struct pci_dev *pdev = to_pci_dev(dev);
int ret;
+ mutex_lock(&vgasr_mutex);
vga_switcheroo_power_switch(pdev, VGA_SWITCHEROO_ON);
+ mutex_unlock(&vgasr_mutex);
ret = dev->bus->pm->runtime_resume(dev);
if (ret)
return ret;
return 0;
}
-/* this version is for the case where the power switch is separate
- to the device being powered down. */
+/**
+ * vga_switcheroo_init_domain_pm_ops() - helper for driver power control
+ * @dev: vga client device
+ * @domain: power domain
+ *
+ * Helper for GPUs whose power state is controlled by the driver's runtime pm.
+ * After the GPU has been suspended, the handler needs to be called to cut
+ * power to the GPU. Likewise it needs to reinstate power before the GPU
+ * can resume. To this end, this helper augments the suspend/resume functions
+ * by the requisite calls to the handler. It needs only be called on platforms
+ * where the power switch is separate to the device being powered down.
+ */
int vga_switcheroo_init_domain_pm_ops(struct device *dev,
struct dev_pm_domain *domain)
{
static int vga_switcheroo_runtime_resume_hdmi_audio(struct device *dev)
{
struct pci_dev *pdev = to_pci_dev(dev);
+ struct vga_switcheroo_client *client;
+ struct device *video_dev = NULL;
int ret;
- struct vga_switcheroo_client *client, *found = NULL;
/* we need to check if we have to switch back on the video
device so the audio device can come back */
+ mutex_lock(&vgasr_mutex);
list_for_each_entry(client, &vgasr_priv.clients, list) {
if (PCI_SLOT(client->pdev->devfn) == PCI_SLOT(pdev->devfn) &&
client_is_vga(client)) {
- found = client;
- ret = pm_runtime_get_sync(&client->pdev->dev);
- if (ret) {
- if (ret != 1)
- return ret;
- }
+ video_dev = &client->pdev->dev;
break;
}
}
+ mutex_unlock(&vgasr_mutex);
+
+ if (video_dev) {
+ ret = pm_runtime_get_sync(video_dev);
+ if (ret && ret != 1)
+ return ret;
+ }
ret = dev->bus->pm->runtime_resume(dev);
/* put the reference for the gpu */
- if (found) {
- pm_runtime_mark_last_busy(&found->pdev->dev);
- pm_runtime_put_autosuspend(&found->pdev->dev);
+ if (video_dev) {
+ pm_runtime_mark_last_busy(video_dev);
+ pm_runtime_put_autosuspend(video_dev);
}
return ret;
}
+/**
+ * vga_switcheroo_init_domain_pm_optimus_hdmi_audio() - helper for driver
+ * power control
+ * @dev: audio client device
+ * @domain: power domain
+ *
+ * Helper for GPUs whose power state is controlled by the driver's runtime pm.
+ * When the audio device resumes, the GPU needs to be woken. This helper
+ * augments the audio device's resume function to do that.
+ *
+ * Return: 0 on success, -EINVAL if no power management operations are
+ * defined for this device.
+ */
int
vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev,
struct dev_pm_domain *domain)
return false;
/* Allocate structure */
- vgadev = kmalloc(sizeof(struct vga_device), GFP_KERNEL);
+ vgadev = kzalloc(sizeof(struct vga_device), GFP_KERNEL);
if (vgadev == NULL) {
pr_err("failed to allocate pci device\n");
/*
return false;
}
- memset(vgadev, 0, sizeof(*vgadev));
-
/* Take lock & check for duplicates */
spin_lock_irqsave(&vga_lock, flags);
if (vgadev_find(pdev) != NULL) {
spin_lock_irqsave(&vmbus_connection.channel_lock, flags);
list_del(&channel->listentry);
spin_unlock_irqrestore(&vmbus_connection.channel_lock, flags);
+
+ primary_channel = channel;
} else {
primary_channel = channel->primary_channel;
spin_lock_irqsave(&primary_channel->lock, flags);
primary_channel->num_sc--;
spin_unlock_irqrestore(&primary_channel->lock, flags);
}
+
+ /*
+ * We need to free the bit for init_vp_index() to work in the case
+ * of sub-channel, when we reload drivers like hv_netvsc.
+ */
+ cpumask_clear_cpu(channel->target_cpu,
+ &primary_channel->alloced_cpus_in_node);
+
free_channel(channel);
}
continue;
}
+ /*
+ * NOTE: in the case of sub-channel, we clear the sub-channel
+ * related bit(s) in primary->alloced_cpus_in_node in
+ * hv_process_channel_removal(), so when we reload drivers
+ * like hv_netvsc in SMP guest, here we're able to re-allocate
+ * bit from primary->alloced_cpus_in_node.
+ */
if (!cpumask_test_cpu(cur_cpu,
&primary->alloced_cpus_in_node)) {
cpumask_set_cpu(cur_cpu,
help
If you say yes here you get support for the hardware monitoring
functionality of the Nuvoton NCT6106D, NCT6775F, NCT6776F, NCT6779D,
- NCT6791D, NCT6792D and compatible Super-I/O chips. This driver
- replaces the w83627ehf driver for NCT6775F and NCT6776F.
+ NCT6791D, NCT6792D, NCT6793D, and compatible Super-I/O chips. This
+ driver replaces the w83627ehf driver for NCT6775F and NCT6776F.
This driver can also be built as a module. If so, the module
will be called nct6775.
{ .compatible = "stericsson,abx500-temp" },
{},
};
+MODULE_DEVICE_TABLE(of, abx500_temp_match);
#endif
static struct platform_driver abx500_temp_driver = {
{ .compatible = "gpio-fan", },
{},
};
+MODULE_DEVICE_TABLE(of, of_gpio_fan_match);
#endif /* CONFIG_OF_GPIO */
static int gpio_fan_probe(struct platform_device *pdev)
* nct6779d 15 5 5 2+6 0xc560 0xc1 0x5ca3
* nct6791d 15 6 6 2+6 0xc800 0xc1 0x5ca3
* nct6792d 15 6 6 2+6 0xc910 0xc1 0x5ca3
+ * nct6793d 15 6 6 2+6 0xd120 0xc1 0x5ca3
*
* #temp lists the number of monitored temperature sources (first value) plus
* the number of directly connectable temperature sensors (second value).
#define USE_ALTERNATE
-enum kinds { nct6106, nct6775, nct6776, nct6779, nct6791, nct6792 };
+enum kinds { nct6106, nct6775, nct6776, nct6779, nct6791, nct6792, nct6793 };
/* used to set data->name = nct6775_device_names[data->sio_kind] */
static const char * const nct6775_device_names[] = {
"nct6779",
"nct6791",
"nct6792",
+ "nct6793",
+};
+
+static const char * const nct6775_sio_names[] __initconst = {
+ "NCT6106D",
+ "NCT6775F",
+ "NCT6776D/F",
+ "NCT6779D",
+ "NCT6791D",
+ "NCT6792D",
+ "NCT6793D",
};
static unsigned short force_id;
#define SIO_NCT6779_ID 0xc560
#define SIO_NCT6791_ID 0xc800
#define SIO_NCT6792_ID 0xc910
+#define SIO_NCT6793_ID 0xd120
#define SIO_ID_MASK 0xFFF0
enum pwm_enable { off, manual, thermal_cruise, speed_cruise, sf3, sf4 };
/* NCT6776 specific data */
+/* STEP_UP_TIME and STEP_DOWN_TIME regs are swapped for all chips but NCT6775 */
+#define NCT6776_REG_FAN_STEP_UP_TIME NCT6775_REG_FAN_STEP_DOWN_TIME
+#define NCT6776_REG_FAN_STEP_DOWN_TIME NCT6775_REG_FAN_STEP_UP_TIME
+
static const s8 NCT6776_ALARM_BITS[] = {
0, 1, 2, 3, 8, 21, 20, 16, /* in0.. in7 */
17, -1, -1, -1, -1, -1, -1, /* in8..in14 */
4, 5, 13, -1, -1, -1, /* temp1..temp6 */
12, 9 }; /* intrusion0, intrusion1 */
-/* NCT6792 specific data */
+/* NCT6792/NCT6793 specific data */
static const u16 NCT6792_REG_TEMP_MON[] = {
0x73, 0x75, 0x77, 0x79, 0x7b, 0x7d };
case nct6779:
case nct6791:
case nct6792:
+ case nct6793:
return reg == 0x150 || reg == 0x153 || reg == 0x155 ||
((reg & 0xfff0) == 0x4b0 && (reg & 0x000f) < 0x0b) ||
reg == 0x402 ||
case nct6779:
case nct6791:
case nct6792:
+ case nct6793:
reg = nct6775_read_value(data,
data->REG_CRITICAL_PWM_ENABLE[i]);
if (reg & data->CRITICAL_PWM_ENABLE_MASK)
case nct6779:
case nct6791:
case nct6792:
+ case nct6793:
nct6775_write_value(data, data->REG_CRITICAL_PWM[nr],
val);
reg = nct6775_read_value(data,
pwm4pin = false;
pwm5pin = false;
pwm6pin = false;
- } else { /* NCT6779D, NCT6791D, or NCT6792D */
+ } else { /* NCT6779D, NCT6791D, NCT6792D, or NCT6793D */
regval = superio_inb(sioreg, 0x1c);
fan3pin = !(regval & (1 << 5));
fan4min = fan4pin;
- if (data->kind == nct6791 || data->kind == nct6792) {
+ if (data->kind == nct6791 || data->kind == nct6792 ||
+ data->kind == nct6793) {
regval = superio_inb(sioreg, 0x2d);
fan6pin = (regval & (1 << 1));
pwm6pin = (regval & (1 << 0));
data->REG_FAN_PULSES = NCT6776_REG_FAN_PULSES;
data->FAN_PULSE_SHIFT = NCT6775_FAN_PULSE_SHIFT;
data->REG_FAN_TIME[0] = NCT6775_REG_FAN_STOP_TIME;
- data->REG_FAN_TIME[1] = NCT6775_REG_FAN_STEP_UP_TIME;
- data->REG_FAN_TIME[2] = NCT6775_REG_FAN_STEP_DOWN_TIME;
+ data->REG_FAN_TIME[1] = NCT6776_REG_FAN_STEP_UP_TIME;
+ data->REG_FAN_TIME[2] = NCT6776_REG_FAN_STEP_DOWN_TIME;
data->REG_TOLERANCE_H = NCT6776_REG_TOLERANCE_H;
data->REG_PWM[0] = NCT6775_REG_PWM;
data->REG_PWM[1] = NCT6775_REG_FAN_START_OUTPUT;
data->REG_FAN_PULSES = NCT6779_REG_FAN_PULSES;
data->FAN_PULSE_SHIFT = NCT6775_FAN_PULSE_SHIFT;
data->REG_FAN_TIME[0] = NCT6775_REG_FAN_STOP_TIME;
- data->REG_FAN_TIME[1] = NCT6775_REG_FAN_STEP_UP_TIME;
- data->REG_FAN_TIME[2] = NCT6775_REG_FAN_STEP_DOWN_TIME;
+ data->REG_FAN_TIME[1] = NCT6776_REG_FAN_STEP_UP_TIME;
+ data->REG_FAN_TIME[2] = NCT6776_REG_FAN_STEP_DOWN_TIME;
data->REG_TOLERANCE_H = NCT6776_REG_TOLERANCE_H;
data->REG_PWM[0] = NCT6775_REG_PWM;
data->REG_PWM[1] = NCT6775_REG_FAN_START_OUTPUT;
break;
case nct6791:
case nct6792:
+ case nct6793:
data->in_num = 15;
data->pwm_num = 6;
data->auto_pwm_num = 4;
data->REG_FAN_PULSES = NCT6779_REG_FAN_PULSES;
data->FAN_PULSE_SHIFT = NCT6775_FAN_PULSE_SHIFT;
data->REG_FAN_TIME[0] = NCT6775_REG_FAN_STOP_TIME;
- data->REG_FAN_TIME[1] = NCT6775_REG_FAN_STEP_UP_TIME;
- data->REG_FAN_TIME[2] = NCT6775_REG_FAN_STEP_DOWN_TIME;
+ data->REG_FAN_TIME[1] = NCT6776_REG_FAN_STEP_UP_TIME;
+ data->REG_FAN_TIME[2] = NCT6776_REG_FAN_STEP_DOWN_TIME;
data->REG_TOLERANCE_H = NCT6776_REG_TOLERANCE_H;
data->REG_PWM[0] = NCT6775_REG_PWM;
data->REG_PWM[1] = NCT6775_REG_FAN_START_OUTPUT;
case nct6779:
case nct6791:
case nct6792:
+ case nct6793:
break;
}
break;
case nct6791:
case nct6792:
+ case nct6793:
tmp |= 0x7e;
break;
}
if (reg != data->sio_reg_enable)
superio_outb(sioreg, SIO_REG_ENABLE, data->sio_reg_enable);
- if (data->kind == nct6791 || data->kind == nct6792)
+ if (data->kind == nct6791 || data->kind == nct6792 ||
+ data->kind == nct6793)
nct6791_enable_io_mapping(sioreg);
superio_exit(sioreg);
.probe = nct6775_probe,
};
-static const char * const nct6775_sio_names[] __initconst = {
- "NCT6106D",
- "NCT6775F",
- "NCT6776D/F",
- "NCT6779D",
- "NCT6791D",
- "NCT6792D",
-};
-
/* nct6775_find() looks for a '627 in the Super-I/O config space */
static int __init nct6775_find(int sioaddr, struct nct6775_sio_data *sio_data)
{
case SIO_NCT6792_ID:
sio_data->kind = nct6792;
break;
+ case SIO_NCT6793_ID:
+ sio_data->kind = nct6793;
+ break;
default:
if (val != 0xffff)
pr_debug("unsupported chip ID: 0x%04x\n", val);
superio_outb(sioaddr, SIO_REG_ENABLE, val | 0x01);
}
- if (sio_data->kind == nct6791 || sio_data->kind == nct6792)
+ if (sio_data->kind == nct6791 || sio_data->kind == nct6792 ||
+ sio_data->kind == nct6793)
nct6791_enable_io_mapping(sioaddr);
superio_exit(sioaddr);
}
MODULE_AUTHOR("Guenter Roeck <linux@roeck-us.net>");
-MODULE_DESCRIPTION("NCT6106D/NCT6775F/NCT6776F/NCT6779D/NCT6791D/NCT6792D driver");
+MODULE_DESCRIPTION("Driver for NCT6775F and compatible chips");
MODULE_LICENSE("GPL");
module_init(sensors_nct6775_init);
{ .compatible = "pwm-fan", },
{},
};
+MODULE_DEVICE_TABLE(of, of_pwm_fan_match);
static struct platform_driver pwm_fan_driver = {
.probe = pwm_fan_probe,
.name = "C6-SKL",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
- .exit_latency = 75,
+ .exit_latency = 85,
.target_residency = 200,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
.name = "C8-SKL",
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
- .exit_latency = 174,
+ .exit_latency = 200,
.target_residency = 800,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
+ {
+ .name = "C9-SKL",
+ .desc = "MWAIT 0x50",
+ .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 480,
+ .target_residency = 5000,
+ .enter = &intel_idle,
+ .enter_freeze = intel_idle_freeze, },
{
.name = "C10-SKL",
.desc = "MWAIT 0x60",
source "drivers/infiniband/hw/mthca/Kconfig"
source "drivers/infiniband/hw/qib/Kconfig"
-source "drivers/infiniband/hw/ehca/Kconfig"
source "drivers/infiniband/hw/cxgb3/Kconfig"
source "drivers/infiniband/hw/cxgb4/Kconfig"
source "drivers/infiniband/hw/mlx4/Kconfig"
obj-$(CONFIG_INFINIBAND_MTHCA) += mthca/
obj-$(CONFIG_INFINIBAND_QIB) += qib/
-obj-$(CONFIG_INFINIBAND_EHCA) += ehca/
obj-$(CONFIG_INFINIBAND_CXGB3) += cxgb3/
obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
+++ /dev/null
-config INFINIBAND_EHCA
- tristate "eHCA support"
- depends on IBMEBUS
- ---help---
- This driver supports the IBM pSeries eHCA InfiniBand adapter.
-
- To compile the driver as a module, choose M here. The module
- will be called ib_ehca.
-
+++ /dev/null
-# Authors: Heiko J Schick <schickhj@de.ibm.com>
-# Christoph Raisch <raisch@de.ibm.com>
-# Joachim Fenkes <fenkes@de.ibm.com>
-#
-# Copyright (c) 2005 IBM Corporation
-#
-# All rights reserved.
-#
-# This source code is distributed under a dual license of GPL v2.0 and OpenIB BSD.
-
-obj-$(CONFIG_INFINIBAND_EHCA) += ib_ehca.o
-
-ib_ehca-objs = ehca_main.o ehca_hca.o ehca_mcast.o ehca_pd.o ehca_av.o ehca_eq.o \
- ehca_cq.o ehca_qp.o ehca_sqp.o ehca_mrmw.o ehca_reqs.o ehca_irq.o \
- ehca_uverbs.o ipz_pt_fn.o hcp_if.o hcp_phyp.o
-
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * address vector functions
- *
- * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Khadija Souissi <souissik@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-static struct kmem_cache *av_cache;
-
-int ehca_calc_ipd(struct ehca_shca *shca, int port,
- enum ib_rate path_rate, u32 *ipd)
-{
- int path = ib_rate_to_mult(path_rate);
- int link, ret;
- struct ib_port_attr pa;
-
- if (path_rate == IB_RATE_PORT_CURRENT) {
- *ipd = 0;
- return 0;
- }
-
- if (unlikely(path < 0)) {
- ehca_err(&shca->ib_device, "Invalid static rate! path_rate=%x",
- path_rate);
- return -EINVAL;
- }
-
- ret = ehca_query_port(&shca->ib_device, port, &pa);
- if (unlikely(ret < 0)) {
- ehca_err(&shca->ib_device, "Failed to query port ret=%i", ret);
- return ret;
- }
-
- link = ib_width_enum_to_int(pa.active_width) * pa.active_speed;
-
- if (path >= link)
- /* no need to throttle if path faster than link */
- *ipd = 0;
- else
- /* IPD = round((link / path) - 1) */
- *ipd = ((link + (path >> 1)) / path) - 1;
-
- return 0;
-}
-
-struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
-{
- int ret;
- struct ehca_av *av;
- struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
- ib_device);
-
- av = kmem_cache_alloc(av_cache, GFP_KERNEL);
- if (!av) {
- ehca_err(pd->device, "Out of memory pd=%p ah_attr=%p",
- pd, ah_attr);
- return ERR_PTR(-ENOMEM);
- }
-
- av->av.sl = ah_attr->sl;
- av->av.dlid = ah_attr->dlid;
- av->av.slid_path_bits = ah_attr->src_path_bits;
-
- if (ehca_static_rate < 0) {
- u32 ipd;
- if (ehca_calc_ipd(shca, ah_attr->port_num,
- ah_attr->static_rate, &ipd)) {
- ret = -EINVAL;
- goto create_ah_exit1;
- }
- av->av.ipd = ipd;
- } else
- av->av.ipd = ehca_static_rate;
-
- av->av.lnh = ah_attr->ah_flags;
- av->av.grh.word_0 = EHCA_BMASK_SET(GRH_IPVERSION_MASK, 6);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_TCLASS_MASK,
- ah_attr->grh.traffic_class);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
- ah_attr->grh.flow_label);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
- ah_attr->grh.hop_limit);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1B);
- /* set sgid in grh.word_1 */
- if (ah_attr->ah_flags & IB_AH_GRH) {
- int rc;
- struct ib_port_attr port_attr;
- union ib_gid gid;
- memset(&port_attr, 0, sizeof(port_attr));
- rc = ehca_query_port(pd->device, ah_attr->port_num,
- &port_attr);
- if (rc) { /* invalid port number */
- ret = -EINVAL;
- ehca_err(pd->device, "Invalid port number "
- "ehca_query_port() returned %x "
- "pd=%p ah_attr=%p", rc, pd, ah_attr);
- goto create_ah_exit1;
- }
- memset(&gid, 0, sizeof(gid));
- rc = ehca_query_gid(pd->device,
- ah_attr->port_num,
- ah_attr->grh.sgid_index, &gid);
- if (rc) {
- ret = -EINVAL;
- ehca_err(pd->device, "Failed to retrieve sgid "
- "ehca_query_gid() returned %x "
- "pd=%p ah_attr=%p", rc, pd, ah_attr);
- goto create_ah_exit1;
- }
- memcpy(&av->av.grh.word_1, &gid, sizeof(gid));
- }
- av->av.pmtu = shca->max_mtu;
-
- /* dgid comes in grh.word_3 */
- memcpy(&av->av.grh.word_3, &ah_attr->grh.dgid,
- sizeof(ah_attr->grh.dgid));
-
- return &av->ib_ah;
-
-create_ah_exit1:
- kmem_cache_free(av_cache, av);
-
- return ERR_PTR(ret);
-}
-
-int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
-{
- struct ehca_av *av;
- struct ehca_ud_av new_ehca_av;
- struct ehca_shca *shca = container_of(ah->pd->device, struct ehca_shca,
- ib_device);
-
- memset(&new_ehca_av, 0, sizeof(new_ehca_av));
- new_ehca_av.sl = ah_attr->sl;
- new_ehca_av.dlid = ah_attr->dlid;
- new_ehca_av.slid_path_bits = ah_attr->src_path_bits;
- new_ehca_av.ipd = ah_attr->static_rate;
- new_ehca_av.lnh = EHCA_BMASK_SET(GRH_FLAG_MASK,
- (ah_attr->ah_flags & IB_AH_GRH) > 0);
- new_ehca_av.grh.word_0 = EHCA_BMASK_SET(GRH_TCLASS_MASK,
- ah_attr->grh.traffic_class);
- new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
- ah_attr->grh.flow_label);
- new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
- ah_attr->grh.hop_limit);
- new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1b);
-
- /* set sgid in grh.word_1 */
- if (ah_attr->ah_flags & IB_AH_GRH) {
- int rc;
- struct ib_port_attr port_attr;
- union ib_gid gid;
- memset(&port_attr, 0, sizeof(port_attr));
- rc = ehca_query_port(ah->device, ah_attr->port_num,
- &port_attr);
- if (rc) { /* invalid port number */
- ehca_err(ah->device, "Invalid port number "
- "ehca_query_port() returned %x "
- "ah=%p ah_attr=%p port_num=%x",
- rc, ah, ah_attr, ah_attr->port_num);
- return -EINVAL;
- }
- memset(&gid, 0, sizeof(gid));
- rc = ehca_query_gid(ah->device,
- ah_attr->port_num,
- ah_attr->grh.sgid_index, &gid);
- if (rc) {
- ehca_err(ah->device, "Failed to retrieve sgid "
- "ehca_query_gid() returned %x "
- "ah=%p ah_attr=%p port_num=%x "
- "sgid_index=%x",
- rc, ah, ah_attr, ah_attr->port_num,
- ah_attr->grh.sgid_index);
- return -EINVAL;
- }
- memcpy(&new_ehca_av.grh.word_1, &gid, sizeof(gid));
- }
-
- new_ehca_av.pmtu = shca->max_mtu;
-
- memcpy(&new_ehca_av.grh.word_3, &ah_attr->grh.dgid,
- sizeof(ah_attr->grh.dgid));
-
- av = container_of(ah, struct ehca_av, ib_ah);
- av->av = new_ehca_av;
-
- return 0;
-}
-
-int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
-{
- struct ehca_av *av = container_of(ah, struct ehca_av, ib_ah);
-
- memcpy(&ah_attr->grh.dgid, &av->av.grh.word_3,
- sizeof(ah_attr->grh.dgid));
- ah_attr->sl = av->av.sl;
-
- ah_attr->dlid = av->av.dlid;
-
- ah_attr->src_path_bits = av->av.slid_path_bits;
- ah_attr->static_rate = av->av.ipd;
- ah_attr->ah_flags = EHCA_BMASK_GET(GRH_FLAG_MASK, av->av.lnh);
- ah_attr->grh.traffic_class = EHCA_BMASK_GET(GRH_TCLASS_MASK,
- av->av.grh.word_0);
- ah_attr->grh.hop_limit = EHCA_BMASK_GET(GRH_HOPLIMIT_MASK,
- av->av.grh.word_0);
- ah_attr->grh.flow_label = EHCA_BMASK_GET(GRH_FLOWLABEL_MASK,
- av->av.grh.word_0);
-
- return 0;
-}
-
-int ehca_destroy_ah(struct ib_ah *ah)
-{
- kmem_cache_free(av_cache, container_of(ah, struct ehca_av, ib_ah));
-
- return 0;
-}
-
-int ehca_init_av_cache(void)
-{
- av_cache = kmem_cache_create("ehca_cache_av",
- sizeof(struct ehca_av), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!av_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_av_cache(void)
-{
- if (av_cache)
- kmem_cache_destroy(av_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Struct definition for eHCA internal structures
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_CLASSES_H__
-#define __EHCA_CLASSES_H__
-
-struct ehca_module;
-struct ehca_qp;
-struct ehca_cq;
-struct ehca_eq;
-struct ehca_mr;
-struct ehca_mw;
-struct ehca_pd;
-struct ehca_av;
-
-#include <linux/wait.h>
-#include <linux/mutex.h>
-
-#include <rdma/ib_verbs.h>
-#include <rdma/ib_user_verbs.h>
-
-#ifdef CONFIG_PPC64
-#include "ehca_classes_pSeries.h"
-#endif
-#include "ipz_pt_fn.h"
-#include "ehca_qes.h"
-#include "ehca_irq.h"
-
-#define EHCA_EQE_CACHE_SIZE 20
-#define EHCA_MAX_NUM_QUEUES 0xffff
-
-struct ehca_eqe_cache_entry {
- struct ehca_eqe *eqe;
- struct ehca_cq *cq;
-};
-
-struct ehca_eq {
- u32 length;
- struct ipz_queue ipz_queue;
- struct ipz_eq_handle ipz_eq_handle;
- struct work_struct work;
- struct h_galpas galpas;
- int is_initialized;
- struct ehca_pfeq pf;
- spinlock_t spinlock;
- struct tasklet_struct interrupt_task;
- u32 ist;
- spinlock_t irq_spinlock;
- struct ehca_eqe_cache_entry eqe_cache[EHCA_EQE_CACHE_SIZE];
-};
-
-struct ehca_sma_attr {
- u16 lid, lmc, sm_sl, sm_lid;
- u16 pkey_tbl_len, pkeys[16];
-};
-
-struct ehca_sport {
- struct ib_cq *ibcq_aqp1;
- struct ib_qp *ibqp_sqp[2];
- /* lock to serialze modify_qp() calls for sqp in normal
- * and irq path (when event PORT_ACTIVE is received first time)
- */
- spinlock_t mod_sqp_lock;
- enum ib_port_state port_state;
- struct ehca_sma_attr saved_attr;
- u32 pma_qp_nr;
-};
-
-#define HCA_CAP_MR_PGSIZE_4K 0x80000000
-#define HCA_CAP_MR_PGSIZE_64K 0x40000000
-#define HCA_CAP_MR_PGSIZE_1M 0x20000000
-#define HCA_CAP_MR_PGSIZE_16M 0x10000000
-
-struct ehca_shca {
- struct ib_device ib_device;
- struct platform_device *ofdev;
- u8 num_ports;
- int hw_level;
- struct list_head shca_list;
- struct ipz_adapter_handle ipz_hca_handle;
- struct ehca_sport sport[2];
- struct ehca_eq eq;
- struct ehca_eq neq;
- struct ehca_mr *maxmr;
- struct ehca_pd *pd;
- struct h_galpas galpas;
- struct mutex modify_mutex;
- u64 hca_cap;
- /* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
- u32 hca_cap_mr_pgsize;
- int max_mtu;
- int max_num_qps;
- int max_num_cqs;
- atomic_t num_cqs;
- atomic_t num_qps;
-};
-
-struct ehca_pd {
- struct ib_pd ib_pd;
- struct ipz_pd fw_pd;
- /* small queue mgmt */
- struct mutex lock;
- struct list_head free[2];
- struct list_head full[2];
-};
-
-enum ehca_ext_qp_type {
- EQPT_NORMAL = 0,
- EQPT_LLQP = 1,
- EQPT_SRQBASE = 2,
- EQPT_SRQ = 3,
-};
-
-/* struct to cache modify_qp()'s parms for GSI/SMI qp */
-struct ehca_mod_qp_parm {
- int mask;
- struct ib_qp_attr attr;
-};
-
-#define EHCA_MOD_QP_PARM_MAX 4
-
-#define QMAP_IDX_MASK 0xFFFFULL
-
-/* struct for tracking if cqes have been reported to the application */
-struct ehca_qmap_entry {
- u16 app_wr_id;
- u8 reported;
- u8 cqe_req;
-};
-
-struct ehca_queue_map {
- struct ehca_qmap_entry *map;
- unsigned int entries;
- unsigned int tail;
- unsigned int left_to_poll;
- unsigned int next_wqe_idx; /* Idx to first wqe to be flushed */
-};
-
-/* function to calculate the next index for the qmap */
-static inline unsigned int next_index(unsigned int cur_index, unsigned int limit)
-{
- unsigned int temp = cur_index + 1;
- return (temp == limit) ? 0 : temp;
-}
-
-struct ehca_qp {
- union {
- struct ib_qp ib_qp;
- struct ib_srq ib_srq;
- };
- u32 qp_type;
- enum ehca_ext_qp_type ext_type;
- enum ib_qp_state state;
- struct ipz_queue ipz_squeue;
- struct ehca_queue_map sq_map;
- struct ipz_queue ipz_rqueue;
- struct ehca_queue_map rq_map;
- struct h_galpas galpas;
- u32 qkey;
- u32 real_qp_num;
- u32 token;
- spinlock_t spinlock_s;
- spinlock_t spinlock_r;
- u32 sq_max_inline_data_size;
- struct ipz_qp_handle ipz_qp_handle;
- struct ehca_pfqp pf;
- struct ib_qp_init_attr init_attr;
- struct ehca_cq *send_cq;
- struct ehca_cq *recv_cq;
- unsigned int sqerr_purgeflag;
- struct hlist_node list_entries;
- /* array to cache modify_qp()'s parms for GSI/SMI qp */
- struct ehca_mod_qp_parm *mod_qp_parm;
- int mod_qp_parm_idx;
- /* mmap counter for resources mapped into user space */
- u32 mm_count_squeue;
- u32 mm_count_rqueue;
- u32 mm_count_galpa;
- /* unsolicited ack circumvention */
- int unsol_ack_circ;
- int mtu_shift;
- u32 message_count;
- u32 packet_count;
- atomic_t nr_events; /* events seen */
- wait_queue_head_t wait_completion;
- int mig_armed;
- struct list_head sq_err_node;
- struct list_head rq_err_node;
-};
-
-#define IS_SRQ(qp) (qp->ext_type == EQPT_SRQ)
-#define HAS_SQ(qp) (qp->ext_type != EQPT_SRQ)
-#define HAS_RQ(qp) (qp->ext_type != EQPT_SRQBASE)
-
-/* must be power of 2 */
-#define QP_HASHTAB_LEN 8
-
-struct ehca_cq {
- struct ib_cq ib_cq;
- struct ipz_queue ipz_queue;
- struct h_galpas galpas;
- spinlock_t spinlock;
- u32 cq_number;
- u32 token;
- u32 nr_of_entries;
- struct ipz_cq_handle ipz_cq_handle;
- struct ehca_pfcq pf;
- spinlock_t cb_lock;
- struct hlist_head qp_hashtab[QP_HASHTAB_LEN];
- struct list_head entry;
- u32 nr_callbacks; /* #events assigned to cpu by scaling code */
- atomic_t nr_events; /* #events seen */
- wait_queue_head_t wait_completion;
- spinlock_t task_lock;
- /* mmap counter for resources mapped into user space */
- u32 mm_count_queue;
- u32 mm_count_galpa;
- struct list_head sqp_err_list;
- struct list_head rqp_err_list;
-};
-
-enum ehca_mr_flag {
- EHCA_MR_FLAG_FMR = 0x80000000, /* FMR, created with ehca_alloc_fmr */
- EHCA_MR_FLAG_MAXMR = 0x40000000, /* max-MR */
-};
-
-struct ehca_mr {
- union {
- struct ib_mr ib_mr; /* must always be first in ehca_mr */
- struct ib_fmr ib_fmr; /* must always be first in ehca_mr */
- } ib;
- struct ib_umem *umem;
- spinlock_t mrlock;
-
- enum ehca_mr_flag flags;
- u32 num_kpages; /* number of kernel pages */
- u32 num_hwpages; /* number of hw pages to form MR */
- u64 hwpage_size; /* hw page size used for this MR */
- int acl; /* ACL (stored here for usage in reregister) */
- u64 *start; /* virtual start address (stored here for */
- /* usage in reregister) */
- u64 size; /* size (stored here for usage in reregister) */
- u32 fmr_page_size; /* page size for FMR */
- u32 fmr_max_pages; /* max pages for FMR */
- u32 fmr_max_maps; /* max outstanding maps for FMR */
- u32 fmr_map_cnt; /* map counter for FMR */
- /* fw specific data */
- struct ipz_mrmw_handle ipz_mr_handle; /* MR handle for h-calls */
- struct h_galpas galpas;
-};
-
-struct ehca_mw {
- struct ib_mw ib_mw; /* gen2 mw, must always be first in ehca_mw */
- spinlock_t mwlock;
-
- u8 never_bound; /* indication MW was never bound */
- struct ipz_mrmw_handle ipz_mw_handle; /* MW handle for h-calls */
- struct h_galpas galpas;
-};
-
-enum ehca_mr_pgi_type {
- EHCA_MR_PGI_PHYS = 1, /* type of ehca_reg_phys_mr,
- * ehca_rereg_phys_mr,
- * ehca_reg_internal_maxmr */
- EHCA_MR_PGI_USER = 2, /* type of ehca_reg_user_mr */
- EHCA_MR_PGI_FMR = 3 /* type of ehca_map_phys_fmr */
-};
-
-struct ehca_mr_pginfo {
- enum ehca_mr_pgi_type type;
- u64 num_kpages;
- u64 kpage_cnt;
- u64 hwpage_size; /* hw page size used for this MR */
- u64 num_hwpages; /* number of hw pages */
- u64 hwpage_cnt; /* counter for hw pages */
- u64 next_hwpage; /* next hw page in buffer/chunk/listelem */
-
- union {
- struct { /* type EHCA_MR_PGI_PHYS section */
- int num_phys_buf;
- struct ib_phys_buf *phys_buf_array;
- u64 next_buf;
- } phy;
- struct { /* type EHCA_MR_PGI_USER section */
- struct ib_umem *region;
- struct scatterlist *next_sg;
- u64 next_nmap;
- } usr;
- struct { /* type EHCA_MR_PGI_FMR section */
- u64 fmr_pgsize;
- u64 *page_list;
- u64 next_listelem;
- } fmr;
- } u;
-};
-
-/* output parameters for MR/FMR hipz calls */
-struct ehca_mr_hipzout_parms {
- struct ipz_mrmw_handle handle;
- u32 lkey;
- u32 rkey;
- u64 len;
- u64 vaddr;
- u32 acl;
-};
-
-/* output parameters for MW hipz calls */
-struct ehca_mw_hipzout_parms {
- struct ipz_mrmw_handle handle;
- u32 rkey;
-};
-
-struct ehca_av {
- struct ib_ah ib_ah;
- struct ehca_ud_av av;
-};
-
-struct ehca_ucontext {
- struct ib_ucontext ib_ucontext;
-};
-
-int ehca_init_pd_cache(void);
-void ehca_cleanup_pd_cache(void);
-int ehca_init_cq_cache(void);
-void ehca_cleanup_cq_cache(void);
-int ehca_init_qp_cache(void);
-void ehca_cleanup_qp_cache(void);
-int ehca_init_av_cache(void);
-void ehca_cleanup_av_cache(void);
-int ehca_init_mrmw_cache(void);
-void ehca_cleanup_mrmw_cache(void);
-int ehca_init_small_qp_cache(void);
-void ehca_cleanup_small_qp_cache(void);
-
-extern rwlock_t ehca_qp_idr_lock;
-extern rwlock_t ehca_cq_idr_lock;
-extern struct idr ehca_qp_idr;
-extern struct idr ehca_cq_idr;
-extern spinlock_t shca_list_lock;
-
-extern int ehca_static_rate;
-extern int ehca_port_act_time;
-extern bool ehca_use_hp_mr;
-extern bool ehca_scaling_code;
-extern int ehca_lock_hcalls;
-extern int ehca_nr_ports;
-extern int ehca_max_cq;
-extern int ehca_max_qp;
-
-struct ipzu_queue_resp {
- u32 qe_size; /* queue entry size */
- u32 act_nr_of_sg;
- u32 queue_length; /* queue length allocated in bytes */
- u32 pagesize;
- u32 toggle_state;
- u32 offset; /* save offset within a page for small_qp */
-};
-
-struct ehca_create_cq_resp {
- u32 cq_number;
- u32 token;
- struct ipzu_queue_resp ipz_queue;
- u32 fw_handle_ofs;
- u32 dummy;
-};
-
-struct ehca_create_qp_resp {
- u32 qp_num;
- u32 token;
- u32 qp_type;
- u32 ext_type;
- u32 qkey;
- /* qp_num assigned by ehca: sqp0/1 may have got different numbers */
- u32 real_qp_num;
- u32 fw_handle_ofs;
- u32 dummy;
- struct ipzu_queue_resp ipz_squeue;
- struct ipzu_queue_resp ipz_rqueue;
-};
-
-struct ehca_alloc_cq_parms {
- u32 nr_cqe;
- u32 act_nr_of_entries;
- u32 act_pages;
- struct ipz_eq_handle eq_handle;
-};
-
-enum ehca_service_type {
- ST_RC = 0,
- ST_UC = 1,
- ST_RD = 2,
- ST_UD = 3,
-};
-
-enum ehca_ll_comp_flags {
- LLQP_SEND_COMP = 0x20,
- LLQP_RECV_COMP = 0x40,
- LLQP_COMP_MASK = 0x60,
-};
-
-struct ehca_alloc_queue_parms {
- /* input parameters */
- int max_wr;
- int max_sge;
- int page_size;
- int is_small;
-
- /* output parameters */
- u16 act_nr_wqes;
- u8 act_nr_sges;
- u32 queue_size; /* bytes for small queues, pages otherwise */
-};
-
-struct ehca_alloc_qp_parms {
- struct ehca_alloc_queue_parms squeue;
- struct ehca_alloc_queue_parms rqueue;
-
- /* input parameters */
- enum ehca_service_type servicetype;
- int qp_storage;
- int sigtype;
- enum ehca_ext_qp_type ext_type;
- enum ehca_ll_comp_flags ll_comp_flags;
- int ud_av_l_key_ctl;
-
- u32 token;
- struct ipz_eq_handle eq_handle;
- struct ipz_pd pd;
- struct ipz_cq_handle send_cq_handle, recv_cq_handle;
-
- u32 srq_qpn, srq_token, srq_limit;
-
- /* output parameters */
- u32 real_qp_num;
- struct ipz_qp_handle qp_handle;
- struct h_galpas galpas;
-};
-
-int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
-int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int qp_num);
-struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int qp_num);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * pSeries interface definitions
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_CLASSES_PSERIES_H__
-#define __EHCA_CLASSES_PSERIES_H__
-
-#include "hcp_phyp.h"
-#include "ipz_pt_fn.h"
-
-
-struct ehca_pfqp {
- struct ipz_qpt sqpt;
- struct ipz_qpt rqpt;
-};
-
-struct ehca_pfcq {
- struct ipz_qpt qpt;
- u32 cqnr;
-};
-
-struct ehca_pfeq {
- struct ipz_qpt qpt;
- struct h_galpa galpa;
- u32 eqnr;
-};
-
-struct ipz_adapter_handle {
- u64 handle;
-};
-
-struct ipz_cq_handle {
- u64 handle;
-};
-
-struct ipz_eq_handle {
- u64 handle;
-};
-
-struct ipz_qp_handle {
- u64 handle;
-};
-struct ipz_mrmw_handle {
- u64 handle;
-};
-
-struct ipz_pd {
- u32 value;
-};
-
-struct hcp_modify_qp_control_block {
- u32 qkey; /* 00 */
- u32 rdd; /* reliable datagram domain */
- u32 send_psn; /* 02 */
- u32 receive_psn; /* 03 */
- u32 prim_phys_port; /* 04 */
- u32 alt_phys_port; /* 05 */
- u32 prim_p_key_idx; /* 06 */
- u32 alt_p_key_idx; /* 07 */
- u32 rdma_atomic_ctrl; /* 08 */
- u32 qp_state; /* 09 */
- u32 reserved_10; /* 10 */
- u32 rdma_nr_atomic_resp_res; /* 11 */
- u32 path_migration_state; /* 12 */
- u32 rdma_atomic_outst_dest_qp; /* 13 */
- u32 dest_qp_nr; /* 14 */
- u32 min_rnr_nak_timer_field; /* 15 */
- u32 service_level; /* 16 */
- u32 send_grh_flag; /* 17 */
- u32 retry_count; /* 18 */
- u32 timeout; /* 19 */
- u32 path_mtu; /* 20 */
- u32 max_static_rate; /* 21 */
- u32 dlid; /* 22 */
- u32 rnr_retry_count; /* 23 */
- u32 source_path_bits; /* 24 */
- u32 traffic_class; /* 25 */
- u32 hop_limit; /* 26 */
- u32 source_gid_idx; /* 27 */
- u32 flow_label; /* 28 */
- u32 reserved_29; /* 29 */
- union { /* 30 */
- u64 dw[2];
- u8 byte[16];
- } dest_gid;
- u32 service_level_al; /* 34 */
- u32 send_grh_flag_al; /* 35 */
- u32 retry_count_al; /* 36 */
- u32 timeout_al; /* 37 */
- u32 max_static_rate_al; /* 38 */
- u32 dlid_al; /* 39 */
- u32 rnr_retry_count_al; /* 40 */
- u32 source_path_bits_al; /* 41 */
- u32 traffic_class_al; /* 42 */
- u32 hop_limit_al; /* 43 */
- u32 source_gid_idx_al; /* 44 */
- u32 flow_label_al; /* 45 */
- u32 reserved_46; /* 46 */
- u32 reserved_47; /* 47 */
- union { /* 48 */
- u64 dw[2];
- u8 byte[16];
- } dest_gid_al;
- u32 max_nr_outst_send_wr; /* 52 */
- u32 max_nr_outst_recv_wr; /* 53 */
- u32 disable_ete_credit_check; /* 54 */
- u32 qp_number; /* 55 */
- u64 send_queue_handle; /* 56 */
- u64 recv_queue_handle; /* 58 */
- u32 actual_nr_sges_in_sq_wqe; /* 60 */
- u32 actual_nr_sges_in_rq_wqe; /* 61 */
- u32 qp_enable; /* 62 */
- u32 curr_srq_limit; /* 63 */
- u64 qp_aff_asyn_ev_log_reg; /* 64 */
- u64 shared_rq_hndl; /* 66 */
- u64 trigg_doorbell_qp_hndl; /* 68 */
- u32 reserved_70_127[58]; /* 70 */
-};
-
-#define MQPCB_MASK_QKEY EHCA_BMASK_IBM( 0, 0)
-#define MQPCB_MASK_SEND_PSN EHCA_BMASK_IBM( 2, 2)
-#define MQPCB_MASK_RECEIVE_PSN EHCA_BMASK_IBM( 3, 3)
-#define MQPCB_MASK_PRIM_PHYS_PORT EHCA_BMASK_IBM( 4, 4)
-#define MQPCB_PRIM_PHYS_PORT EHCA_BMASK_IBM(24, 31)
-#define MQPCB_MASK_ALT_PHYS_PORT EHCA_BMASK_IBM( 5, 5)
-#define MQPCB_MASK_PRIM_P_KEY_IDX EHCA_BMASK_IBM( 6, 6)
-#define MQPCB_PRIM_P_KEY_IDX EHCA_BMASK_IBM(24, 31)
-#define MQPCB_MASK_ALT_P_KEY_IDX EHCA_BMASK_IBM( 7, 7)
-#define MQPCB_MASK_RDMA_ATOMIC_CTRL EHCA_BMASK_IBM( 8, 8)
-#define MQPCB_MASK_QP_STATE EHCA_BMASK_IBM( 9, 9)
-#define MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES EHCA_BMASK_IBM(11, 11)
-#define MQPCB_MASK_PATH_MIGRATION_STATE EHCA_BMASK_IBM(12, 12)
-#define MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP EHCA_BMASK_IBM(13, 13)
-#define MQPCB_MASK_DEST_QP_NR EHCA_BMASK_IBM(14, 14)
-#define MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD EHCA_BMASK_IBM(15, 15)
-#define MQPCB_MASK_SERVICE_LEVEL EHCA_BMASK_IBM(16, 16)
-#define MQPCB_MASK_SEND_GRH_FLAG EHCA_BMASK_IBM(17, 17)
-#define MQPCB_MASK_RETRY_COUNT EHCA_BMASK_IBM(18, 18)
-#define MQPCB_MASK_TIMEOUT EHCA_BMASK_IBM(19, 19)
-#define MQPCB_MASK_PATH_MTU EHCA_BMASK_IBM(20, 20)
-#define MQPCB_MASK_MAX_STATIC_RATE EHCA_BMASK_IBM(21, 21)
-#define MQPCB_MASK_DLID EHCA_BMASK_IBM(22, 22)
-#define MQPCB_MASK_RNR_RETRY_COUNT EHCA_BMASK_IBM(23, 23)
-#define MQPCB_MASK_SOURCE_PATH_BITS EHCA_BMASK_IBM(24, 24)
-#define MQPCB_MASK_TRAFFIC_CLASS EHCA_BMASK_IBM(25, 25)
-#define MQPCB_MASK_HOP_LIMIT EHCA_BMASK_IBM(26, 26)
-#define MQPCB_MASK_SOURCE_GID_IDX EHCA_BMASK_IBM(27, 27)
-#define MQPCB_MASK_FLOW_LABEL EHCA_BMASK_IBM(28, 28)
-#define MQPCB_MASK_DEST_GID EHCA_BMASK_IBM(30, 30)
-#define MQPCB_MASK_SERVICE_LEVEL_AL EHCA_BMASK_IBM(31, 31)
-#define MQPCB_MASK_SEND_GRH_FLAG_AL EHCA_BMASK_IBM(32, 32)
-#define MQPCB_MASK_RETRY_COUNT_AL EHCA_BMASK_IBM(33, 33)
-#define MQPCB_MASK_TIMEOUT_AL EHCA_BMASK_IBM(34, 34)
-#define MQPCB_MASK_MAX_STATIC_RATE_AL EHCA_BMASK_IBM(35, 35)
-#define MQPCB_MASK_DLID_AL EHCA_BMASK_IBM(36, 36)
-#define MQPCB_MASK_RNR_RETRY_COUNT_AL EHCA_BMASK_IBM(37, 37)
-#define MQPCB_MASK_SOURCE_PATH_BITS_AL EHCA_BMASK_IBM(38, 38)
-#define MQPCB_MASK_TRAFFIC_CLASS_AL EHCA_BMASK_IBM(39, 39)
-#define MQPCB_MASK_HOP_LIMIT_AL EHCA_BMASK_IBM(40, 40)
-#define MQPCB_MASK_SOURCE_GID_IDX_AL EHCA_BMASK_IBM(41, 41)
-#define MQPCB_MASK_FLOW_LABEL_AL EHCA_BMASK_IBM(42, 42)
-#define MQPCB_MASK_DEST_GID_AL EHCA_BMASK_IBM(44, 44)
-#define MQPCB_MASK_MAX_NR_OUTST_SEND_WR EHCA_BMASK_IBM(45, 45)
-#define MQPCB_MASK_MAX_NR_OUTST_RECV_WR EHCA_BMASK_IBM(46, 46)
-#define MQPCB_MASK_DISABLE_ETE_CREDIT_CHECK EHCA_BMASK_IBM(47, 47)
-#define MQPCB_MASK_QP_ENABLE EHCA_BMASK_IBM(48, 48)
-#define MQPCB_MASK_CURR_SRQ_LIMIT EHCA_BMASK_IBM(49, 49)
-#define MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG EHCA_BMASK_IBM(50, 50)
-#define MQPCB_MASK_SHARED_RQ_HNDL EHCA_BMASK_IBM(51, 51)
-
-#endif /* __EHCA_CLASSES_PSERIES_H__ */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Completion queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_iverbs.h"
-#include "ehca_classes.h"
-#include "ehca_irq.h"
-#include "hcp_if.h"
-
-static struct kmem_cache *cq_cache;
-
-int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
-{
- unsigned int qp_num = qp->real_qp_num;
- unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
- unsigned long flags;
-
- spin_lock_irqsave(&cq->spinlock, flags);
- hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
- spin_unlock_irqrestore(&cq->spinlock, flags);
-
- ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
- cq->cq_number, qp_num);
-
- return 0;
-}
-
-int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
-{
- int ret = -EINVAL;
- unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
- struct hlist_node *iter;
- struct ehca_qp *qp;
- unsigned long flags;
-
- spin_lock_irqsave(&cq->spinlock, flags);
- hlist_for_each(iter, &cq->qp_hashtab[key]) {
- qp = hlist_entry(iter, struct ehca_qp, list_entries);
- if (qp->real_qp_num == real_qp_num) {
- hlist_del(iter);
- ehca_dbg(cq->ib_cq.device,
- "removed qp from cq .cq_num=%x real_qp_num=%x",
- cq->cq_number, real_qp_num);
- ret = 0;
- break;
- }
- }
- spin_unlock_irqrestore(&cq->spinlock, flags);
- if (ret)
- ehca_err(cq->ib_cq.device,
- "qp not found cq_num=%x real_qp_num=%x",
- cq->cq_number, real_qp_num);
-
- return ret;
-}
-
-struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
-{
- struct ehca_qp *ret = NULL;
- unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
- struct hlist_node *iter;
- struct ehca_qp *qp;
- hlist_for_each(iter, &cq->qp_hashtab[key]) {
- qp = hlist_entry(iter, struct ehca_qp, list_entries);
- if (qp->real_qp_num == real_qp_num) {
- ret = qp;
- break;
- }
- }
- return ret;
-}
-
-struct ib_cq *ehca_create_cq(struct ib_device *device,
- const struct ib_cq_init_attr *attr,
- struct ib_ucontext *context,
- struct ib_udata *udata)
-{
- int cqe = attr->cqe;
- static const u32 additional_cqe = 20;
- struct ib_cq *cq;
- struct ehca_cq *my_cq;
- struct ehca_shca *shca =
- container_of(device, struct ehca_shca, ib_device);
- struct ipz_adapter_handle adapter_handle;
- struct ehca_alloc_cq_parms param; /* h_call's out parameters */
- struct h_galpa gal;
- void *vpage;
- u32 counter;
- u64 rpage, cqx_fec, h_ret;
- int ipz_rc, i;
- unsigned long flags;
-
- if (attr->flags)
- return ERR_PTR(-EINVAL);
-
- if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
- return ERR_PTR(-EINVAL);
-
- if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
- ehca_err(device, "Unable to create CQ, max number of %i "
- "CQs reached.", shca->max_num_cqs);
- ehca_err(device, "To increase the maximum number of CQs "
- "use the number_of_cqs module parameter.\n");
- return ERR_PTR(-ENOSPC);
- }
-
- my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
- if (!my_cq) {
- ehca_err(device, "Out of memory for ehca_cq struct device=%p",
- device);
- atomic_dec(&shca->num_cqs);
- return ERR_PTR(-ENOMEM);
- }
-
- memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms));
-
- spin_lock_init(&my_cq->spinlock);
- spin_lock_init(&my_cq->cb_lock);
- spin_lock_init(&my_cq->task_lock);
- atomic_set(&my_cq->nr_events, 0);
- init_waitqueue_head(&my_cq->wait_completion);
-
- cq = &my_cq->ib_cq;
-
- adapter_handle = shca->ipz_hca_handle;
- param.eq_handle = shca->eq.ipz_eq_handle;
-
- idr_preload(GFP_KERNEL);
- write_lock_irqsave(&ehca_cq_idr_lock, flags);
- my_cq->token = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT);
- write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
- idr_preload_end();
-
- if (my_cq->token < 0) {
- cq = ERR_PTR(-ENOMEM);
- ehca_err(device, "Can't allocate new idr entry. device=%p",
- device);
- goto create_cq_exit1;
- }
-
- /*
- * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
- * for receiving errors CQEs.
- */
- param.nr_cqe = cqe + additional_cqe;
- h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m);
-
- if (h_ret != H_SUCCESS) {
- ehca_err(device, "hipz_h_alloc_resource_cq() failed "
- "h_ret=%lli device=%p", h_ret, device);
- cq = ERR_PTR(ehca2ib_return_code(h_ret));
- goto create_cq_exit2;
- }
-
- ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
- EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
- if (!ipz_rc) {
- ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
- ipz_rc, device);
- cq = ERR_PTR(-EINVAL);
- goto create_cq_exit3;
- }
-
- for (counter = 0; counter < param.act_pages; counter++) {
- vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
- if (!vpage) {
- ehca_err(device, "ipz_qpageit_get_inc() "
- "returns NULL device=%p", device);
- cq = ERR_PTR(-EAGAIN);
- goto create_cq_exit4;
- }
- rpage = __pa(vpage);
-
- h_ret = hipz_h_register_rpage_cq(adapter_handle,
- my_cq->ipz_cq_handle,
- &my_cq->pf,
- 0,
- 0,
- rpage,
- 1,
- my_cq->galpas.
- kernel);
-
- if (h_ret < H_SUCCESS) {
- ehca_err(device, "hipz_h_register_rpage_cq() failed "
- "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i "
- "act_pages=%i", my_cq, my_cq->cq_number,
- h_ret, counter, param.act_pages);
- cq = ERR_PTR(-EINVAL);
- goto create_cq_exit4;
- }
-
- if (counter == (param.act_pages - 1)) {
- vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
- if ((h_ret != H_SUCCESS) || vpage) {
- ehca_err(device, "Registration of pages not "
- "complete ehca_cq=%p cq_num=%x "
- "h_ret=%lli", my_cq, my_cq->cq_number,
- h_ret);
- cq = ERR_PTR(-EAGAIN);
- goto create_cq_exit4;
- }
- } else {
- if (h_ret != H_PAGE_REGISTERED) {
- ehca_err(device, "Registration of page failed "
- "ehca_cq=%p cq_num=%x h_ret=%lli "
- "counter=%i act_pages=%i",
- my_cq, my_cq->cq_number,
- h_ret, counter, param.act_pages);
- cq = ERR_PTR(-ENOMEM);
- goto create_cq_exit4;
- }
- }
- }
-
- ipz_qeit_reset(&my_cq->ipz_queue);
-
- gal = my_cq->galpas.kernel;
- cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
- ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx",
- my_cq, my_cq->cq_number, cqx_fec);
-
- my_cq->ib_cq.cqe = my_cq->nr_of_entries =
- param.act_nr_of_entries - additional_cqe;
- my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
-
- for (i = 0; i < QP_HASHTAB_LEN; i++)
- INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
-
- INIT_LIST_HEAD(&my_cq->sqp_err_list);
- INIT_LIST_HEAD(&my_cq->rqp_err_list);
-
- if (context) {
- struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
- struct ehca_create_cq_resp resp;
- memset(&resp, 0, sizeof(resp));
- resp.cq_number = my_cq->cq_number;
- resp.token = my_cq->token;
- resp.ipz_queue.qe_size = ipz_queue->qe_size;
- resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
- resp.ipz_queue.queue_length = ipz_queue->queue_length;
- resp.ipz_queue.pagesize = ipz_queue->pagesize;
- resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
- resp.fw_handle_ofs = (u32)
- (my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
- if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
- ehca_err(device, "Copy to udata failed.");
- cq = ERR_PTR(-EFAULT);
- goto create_cq_exit4;
- }
- }
-
- return cq;
-
-create_cq_exit4:
- ipz_queue_dtor(NULL, &my_cq->ipz_queue);
-
-create_cq_exit3:
- h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
- if (h_ret != H_SUCCESS)
- ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
- "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret);
-
-create_cq_exit2:
- write_lock_irqsave(&ehca_cq_idr_lock, flags);
- idr_remove(&ehca_cq_idr, my_cq->token);
- write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
-
-create_cq_exit1:
- kmem_cache_free(cq_cache, my_cq);
-
- atomic_dec(&shca->num_cqs);
- return cq;
-}
-
-int ehca_destroy_cq(struct ib_cq *cq)
-{
- u64 h_ret;
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- int cq_num = my_cq->cq_number;
- struct ib_device *device = cq->device;
- struct ehca_shca *shca = container_of(device, struct ehca_shca,
- ib_device);
- struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
- unsigned long flags;
-
- if (cq->uobject) {
- if (my_cq->mm_count_galpa || my_cq->mm_count_queue) {
- ehca_err(device, "Resources still referenced in "
- "user space cq_num=%x", my_cq->cq_number);
- return -EINVAL;
- }
- }
-
- /*
- * remove the CQ from the idr first to make sure
- * no more interrupt tasklets will touch this CQ
- */
- write_lock_irqsave(&ehca_cq_idr_lock, flags);
- idr_remove(&ehca_cq_idr, my_cq->token);
- write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
-
- /* now wait until all pending events have completed */
- wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
-
- /* nobody's using our CQ any longer -- we can destroy it */
- h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
- if (h_ret == H_R_STATE) {
- /* cq in err: read err data and destroy it forcibly */
- ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err "
- "state. Try to delete it forcibly.",
- my_cq, cq_num, my_cq->ipz_cq_handle.handle);
- ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
- h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
- if (h_ret == H_SUCCESS)
- ehca_dbg(device, "cq_num=%x deleted successfully.",
- cq_num);
- }
- if (h_ret != H_SUCCESS) {
- ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli "
- "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
- return ehca2ib_return_code(h_ret);
- }
- ipz_queue_dtor(NULL, &my_cq->ipz_queue);
- kmem_cache_free(cq_cache, my_cq);
-
- atomic_dec(&shca->num_cqs);
- return 0;
-}
-
-int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
-{
- /* TODO: proper resize needs to be done */
- ehca_err(cq->device, "not implemented yet");
-
- return -EFAULT;
-}
-
-int ehca_init_cq_cache(void)
-{
- cq_cache = kmem_cache_create("ehca_cache_cq",
- sizeof(struct ehca_cq), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!cq_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_cq_cache(void)
-{
- if (cq_cache)
- kmem_cache_destroy(cq_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Event queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "ehca_classes.h"
-#include "ehca_irq.h"
-#include "ehca_iverbs.h"
-#include "ehca_qes.h"
-#include "hcp_if.h"
-#include "ipz_pt_fn.h"
-
-int ehca_create_eq(struct ehca_shca *shca,
- struct ehca_eq *eq,
- const enum ehca_eq_type type, const u32 length)
-{
- int ret;
- u64 h_ret;
- u32 nr_pages;
- u32 i;
- void *vpage;
- struct ib_device *ib_dev = &shca->ib_device;
-
- spin_lock_init(&eq->spinlock);
- spin_lock_init(&eq->irq_spinlock);
- eq->is_initialized = 0;
-
- if (type != EHCA_EQ && type != EHCA_NEQ) {
- ehca_err(ib_dev, "Invalid EQ type %x. eq=%p", type, eq);
- return -EINVAL;
- }
- if (!length) {
- ehca_err(ib_dev, "EQ length must not be zero. eq=%p", eq);
- return -EINVAL;
- }
-
- h_ret = hipz_h_alloc_resource_eq(shca->ipz_hca_handle,
- &eq->pf,
- type,
- length,
- &eq->ipz_eq_handle,
- &eq->length,
- &nr_pages, &eq->ist);
-
- if (h_ret != H_SUCCESS) {
- ehca_err(ib_dev, "Can't allocate EQ/NEQ. eq=%p", eq);
- return -EINVAL;
- }
-
- ret = ipz_queue_ctor(NULL, &eq->ipz_queue, nr_pages,
- EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0, 0);
- if (!ret) {
- ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq);
- goto create_eq_exit1;
- }
-
- for (i = 0; i < nr_pages; i++) {
- u64 rpage;
-
- vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
- if (!vpage)
- goto create_eq_exit2;
-
- rpage = __pa(vpage);
- h_ret = hipz_h_register_rpage_eq(shca->ipz_hca_handle,
- eq->ipz_eq_handle,
- &eq->pf,
- 0, 0, rpage, 1);
-
- if (i == (nr_pages - 1)) {
- /* last page */
- vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
- if (h_ret != H_SUCCESS || vpage)
- goto create_eq_exit2;
- } else {
- if (h_ret != H_PAGE_REGISTERED)
- goto create_eq_exit2;
- }
- }
-
- ipz_qeit_reset(&eq->ipz_queue);
-
- /* register interrupt handlers and initialize work queues */
- if (type == EHCA_EQ) {
- tasklet_init(&eq->interrupt_task, ehca_tasklet_eq, (long)shca);
-
- ret = ibmebus_request_irq(eq->ist, ehca_interrupt_eq,
- 0, "ehca_eq",
- (void *)shca);
- if (ret < 0)
- ehca_err(ib_dev, "Can't map interrupt handler.");
- } else if (type == EHCA_NEQ) {
- tasklet_init(&eq->interrupt_task, ehca_tasklet_neq, (long)shca);
-
- ret = ibmebus_request_irq(eq->ist, ehca_interrupt_neq,
- 0, "ehca_neq",
- (void *)shca);
- if (ret < 0)
- ehca_err(ib_dev, "Can't map interrupt handler.");
- }
-
- eq->is_initialized = 1;
-
- return 0;
-
-create_eq_exit2:
- ipz_queue_dtor(NULL, &eq->ipz_queue);
-
-create_eq_exit1:
- hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
-
- return -EINVAL;
-}
-
-void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq)
-{
- unsigned long flags;
- void *eqe;
-
- spin_lock_irqsave(&eq->spinlock, flags);
- eqe = ipz_eqit_eq_get_inc_valid(&eq->ipz_queue);
- spin_unlock_irqrestore(&eq->spinlock, flags);
-
- return eqe;
-}
-
-int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq)
-{
- unsigned long flags;
- u64 h_ret;
-
- ibmebus_free_irq(eq->ist, (void *)shca);
-
- spin_lock_irqsave(&shca_list_lock, flags);
- eq->is_initialized = 0;
- spin_unlock_irqrestore(&shca_list_lock, flags);
-
- tasklet_kill(&eq->interrupt_task);
-
- h_ret = hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
-
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't free EQ resources.");
- return -EINVAL;
- }
- ipz_queue_dtor(NULL, &eq->ipz_queue);
-
- return 0;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * HCA query functions
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/gfp.h>
-
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-static unsigned int limit_uint(unsigned int value)
-{
- return min_t(unsigned int, value, INT_MAX);
-}
-
-int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
- struct ib_udata *uhw)
-{
- int i, ret = 0;
- struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
- ib_device);
- struct hipz_query_hca *rblock;
-
- static const u32 cap_mapping[] = {
- IB_DEVICE_RESIZE_MAX_WR, HCA_CAP_WQE_RESIZE,
- IB_DEVICE_BAD_PKEY_CNTR, HCA_CAP_BAD_P_KEY_CTR,
- IB_DEVICE_BAD_QKEY_CNTR, HCA_CAP_Q_KEY_VIOL_CTR,
- IB_DEVICE_RAW_MULTI, HCA_CAP_RAW_PACKET_MCAST,
- IB_DEVICE_AUTO_PATH_MIG, HCA_CAP_AUTO_PATH_MIG,
- IB_DEVICE_CHANGE_PHY_PORT, HCA_CAP_SQD_RTS_PORT_CHANGE,
- IB_DEVICE_UD_AV_PORT_ENFORCE, HCA_CAP_AH_PORT_NR_CHECK,
- IB_DEVICE_CURR_QP_STATE_MOD, HCA_CAP_CUR_QP_STATE_MOD,
- IB_DEVICE_SHUTDOWN_PORT, HCA_CAP_SHUTDOWN_PORT,
- IB_DEVICE_INIT_TYPE, HCA_CAP_INIT_TYPE,
- IB_DEVICE_PORT_ACTIVE_EVENT, HCA_CAP_PORT_ACTIVE_EVENT,
- };
-
- if (uhw->inlen || uhw->outlen)
- return -EINVAL;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query device properties");
- ret = -EINVAL;
- goto query_device1;
- }
-
- memset(props, 0, sizeof(struct ib_device_attr));
- props->page_size_cap = shca->hca_cap_mr_pgsize;
- props->fw_ver = rblock->hw_ver;
- props->max_mr_size = rblock->max_mr_size;
- props->vendor_id = rblock->vendor_id >> 8;
- props->vendor_part_id = rblock->vendor_part_id >> 16;
- props->hw_ver = rblock->hw_ver;
- props->max_qp = limit_uint(rblock->max_qp);
- props->max_qp_wr = limit_uint(rblock->max_wqes_wq);
- props->max_sge = limit_uint(rblock->max_sge);
- props->max_sge_rd = limit_uint(rblock->max_sge_rd);
- props->max_cq = limit_uint(rblock->max_cq);
- props->max_cqe = limit_uint(rblock->max_cqe);
- props->max_mr = limit_uint(rblock->max_mr);
- props->max_mw = limit_uint(rblock->max_mw);
- props->max_pd = limit_uint(rblock->max_pd);
- props->max_ah = limit_uint(rblock->max_ah);
- props->max_ee = limit_uint(rblock->max_rd_ee_context);
- props->max_rdd = limit_uint(rblock->max_rd_domain);
- props->max_fmr = limit_uint(rblock->max_mr);
- props->max_qp_rd_atom = limit_uint(rblock->max_rr_qp);
- props->max_ee_rd_atom = limit_uint(rblock->max_rr_ee_context);
- props->max_res_rd_atom = limit_uint(rblock->max_rr_hca);
- props->max_qp_init_rd_atom = limit_uint(rblock->max_act_wqs_qp);
- props->max_ee_init_rd_atom = limit_uint(rblock->max_act_wqs_ee_context);
-
- if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
- props->max_srq = limit_uint(props->max_qp);
- props->max_srq_wr = limit_uint(props->max_qp_wr);
- props->max_srq_sge = 3;
- }
-
- props->max_pkeys = 16;
- /* Some FW versions say 0 here; insert sensible value in that case */
- props->local_ca_ack_delay = rblock->local_ca_ack_delay ?
- min_t(u8, rblock->local_ca_ack_delay, 255) : 12;
- props->max_raw_ipv6_qp = limit_uint(rblock->max_raw_ipv6_qp);
- props->max_raw_ethy_qp = limit_uint(rblock->max_raw_ethy_qp);
- props->max_mcast_grp = limit_uint(rblock->max_mcast_grp);
- props->max_mcast_qp_attach = limit_uint(rblock->max_mcast_qp_attach);
- props->max_total_mcast_qp_attach
- = limit_uint(rblock->max_total_mcast_qp_attach);
-
- /* translate device capabilities */
- props->device_cap_flags = IB_DEVICE_SYS_IMAGE_GUID |
- IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_N_NOTIFY_CQ;
- for (i = 0; i < ARRAY_SIZE(cap_mapping); i += 2)
- if (rblock->hca_cap_indicators & cap_mapping[i + 1])
- props->device_cap_flags |= cap_mapping[i];
-
-query_device1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-static enum ib_mtu map_mtu(struct ehca_shca *shca, u32 fw_mtu)
-{
- switch (fw_mtu) {
- case 0x1:
- return IB_MTU_256;
- case 0x2:
- return IB_MTU_512;
- case 0x3:
- return IB_MTU_1024;
- case 0x4:
- return IB_MTU_2048;
- case 0x5:
- return IB_MTU_4096;
- default:
- ehca_err(&shca->ib_device, "Unknown MTU size: %x.",
- fw_mtu);
- return 0;
- }
-}
-
-static u8 map_number_of_vls(struct ehca_shca *shca, u32 vl_cap)
-{
- switch (vl_cap) {
- case 0x1:
- return 1;
- case 0x2:
- return 2;
- case 0x3:
- return 4;
- case 0x4:
- return 8;
- case 0x5:
- return 15;
- default:
- ehca_err(&shca->ib_device, "invalid Vl Capability: %x.",
- vl_cap);
- return 0;
- }
-}
-
-int ehca_query_port(struct ib_device *ibdev,
- u8 port, struct ib_port_attr *props)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
- ib_device);
- struct hipz_query_port *rblock;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_port1;
- }
-
- memset(props, 0, sizeof(struct ib_port_attr));
-
- props->active_mtu = props->max_mtu = map_mtu(shca, rblock->max_mtu);
- props->port_cap_flags = rblock->capability_mask;
- props->gid_tbl_len = rblock->gid_tbl_len;
- if (rblock->max_msg_sz)
- props->max_msg_sz = rblock->max_msg_sz;
- else
- props->max_msg_sz = 0x1 << 31;
- props->bad_pkey_cntr = rblock->bad_pkey_cntr;
- props->qkey_viol_cntr = rblock->qkey_viol_cntr;
- props->pkey_tbl_len = rblock->pkey_tbl_len;
- props->lid = rblock->lid;
- props->sm_lid = rblock->sm_lid;
- props->lmc = rblock->lmc;
- props->sm_sl = rblock->sm_sl;
- props->subnet_timeout = rblock->subnet_timeout;
- props->init_type_reply = rblock->init_type_reply;
- props->max_vl_num = map_number_of_vls(shca, rblock->vl_cap);
-
- if (rblock->state && rblock->phys_width) {
- props->phys_state = rblock->phys_pstate;
- props->state = rblock->phys_state;
- props->active_width = rblock->phys_width;
- props->active_speed = rblock->phys_speed;
- } else {
- /* old firmware releases don't report physical
- * port info, so use default values
- */
- props->phys_state = 5;
- props->state = rblock->state;
- props->active_width = IB_WIDTH_12X;
- props->active_speed = IB_SPEED_SDR;
- }
-
-query_port1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-int ehca_query_sma_attr(struct ehca_shca *shca,
- u8 port, struct ehca_sma_attr *attr)
-{
- int ret = 0;
- u64 h_ret;
- struct hipz_query_port *rblock;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_sma_attr1;
- }
-
- memset(attr, 0, sizeof(struct ehca_sma_attr));
-
- attr->lid = rblock->lid;
- attr->lmc = rblock->lmc;
- attr->sm_sl = rblock->sm_sl;
- attr->sm_lid = rblock->sm_lid;
-
- attr->pkey_tbl_len = rblock->pkey_tbl_len;
- memcpy(attr->pkeys, rblock->pkey_entries, sizeof(attr->pkeys));
-
-query_sma_attr1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca;
- struct hipz_query_port *rblock;
-
- shca = container_of(ibdev, struct ehca_shca, ib_device);
- if (index > 16) {
- ehca_err(&shca->ib_device, "Invalid index: %x.", index);
- return -EINVAL;
- }
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_pkey1;
- }
-
- memcpy(pkey, &rblock->pkey_entries + index, sizeof(u16));
-
-query_pkey1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-int ehca_query_gid(struct ib_device *ibdev, u8 port,
- int index, union ib_gid *gid)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
- ib_device);
- struct hipz_query_port *rblock;
-
- if (index < 0 || index > 255) {
- ehca_err(&shca->ib_device, "Invalid index: %x.", index);
- return -EINVAL;
- }
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_gid1;
- }
-
- memcpy(&gid->raw[0], &rblock->gid_prefix, sizeof(u64));
- memcpy(&gid->raw[8], &rblock->guid_entries[index], sizeof(u64));
-
-query_gid1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-static const u32 allowed_port_caps = (
- IB_PORT_SM | IB_PORT_LED_INFO_SUP | IB_PORT_CM_SUP |
- IB_PORT_SNMP_TUNNEL_SUP | IB_PORT_DEVICE_MGMT_SUP |
- IB_PORT_VENDOR_CLASS_SUP);
-
-int ehca_modify_port(struct ib_device *ibdev,
- u8 port, int port_modify_mask,
- struct ib_port_modify *props)
-{
- int ret = 0;
- struct ehca_shca *shca;
- struct hipz_query_port *rblock;
- u32 cap;
- u64 hret;
-
- shca = container_of(ibdev, struct ehca_shca, ib_device);
- if ((props->set_port_cap_mask | props->clr_port_cap_mask)
- & ~allowed_port_caps) {
- ehca_err(&shca->ib_device, "Non-changeable bits set in masks "
- "set=%x clr=%x allowed=%x", props->set_port_cap_mask,
- props->clr_port_cap_mask, allowed_port_caps);
- return -EINVAL;
- }
-
- if (mutex_lock_interruptible(&shca->modify_mutex))
- return -ERESTARTSYS;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- ret = -ENOMEM;
- goto modify_port1;
- }
-
- hret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (hret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto modify_port2;
- }
-
- cap = (rblock->capability_mask | props->set_port_cap_mask)
- & ~props->clr_port_cap_mask;
-
- hret = hipz_h_modify_port(shca->ipz_hca_handle, port,
- cap, props->init_type, port_modify_mask);
- if (hret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Modify port failed h_ret=%lli",
- hret);
- ret = -EINVAL;
- }
-
-modify_port2:
- ehca_free_fw_ctrlblock(rblock);
-
-modify_port1:
- mutex_unlock(&shca->modify_mutex);
-
- return ret;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Functions for EQs, NEQs and interrupts
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-#include <linux/smpboot.h>
-
-#include "ehca_classes.h"
-#include "ehca_irq.h"
-#include "ehca_iverbs.h"
-#include "ehca_tools.h"
-#include "hcp_if.h"
-#include "hipz_fns.h"
-#include "ipz_pt_fn.h"
-
-#define EQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
-#define EQE_CQ_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
-#define EQE_EE_IDENTIFIER EHCA_BMASK_IBM( 2, 7)
-#define EQE_CQ_NUMBER EHCA_BMASK_IBM( 8, 31)
-#define EQE_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
-#define EQE_QP_TOKEN EHCA_BMASK_IBM(32, 63)
-#define EQE_CQ_TOKEN EHCA_BMASK_IBM(32, 63)
-
-#define NEQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
-#define NEQE_EVENT_CODE EHCA_BMASK_IBM( 2, 7)
-#define NEQE_PORT_NUMBER EHCA_BMASK_IBM( 8, 15)
-#define NEQE_PORT_AVAILABILITY EHCA_BMASK_IBM(16, 16)
-#define NEQE_DISRUPTIVE EHCA_BMASK_IBM(16, 16)
-#define NEQE_SPECIFIC_EVENT EHCA_BMASK_IBM(16, 23)
-
-#define ERROR_DATA_LENGTH EHCA_BMASK_IBM(52, 63)
-#define ERROR_DATA_TYPE EHCA_BMASK_IBM( 0, 7)
-
-static void queue_comp_task(struct ehca_cq *__cq);
-
-static struct ehca_comp_pool *pool;
-
-static inline void comp_event_callback(struct ehca_cq *cq)
-{
- if (!cq->ib_cq.comp_handler)
- return;
-
- spin_lock(&cq->cb_lock);
- cq->ib_cq.comp_handler(&cq->ib_cq, cq->ib_cq.cq_context);
- spin_unlock(&cq->cb_lock);
-
- return;
-}
-
-static void print_error_data(struct ehca_shca *shca, void *data,
- u64 *rblock, int length)
-{
- u64 type = EHCA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]);
- u64 resource = rblock[1];
-
- switch (type) {
- case 0x1: /* Queue Pair */
- {
- struct ehca_qp *qp = (struct ehca_qp *)data;
-
- /* only print error data if AER is set */
- if (rblock[6] == 0)
- return;
-
- ehca_err(&shca->ib_device,
- "QP 0x%x (resource=%llx) has errors.",
- qp->ib_qp.qp_num, resource);
- break;
- }
- case 0x4: /* Completion Queue */
- {
- struct ehca_cq *cq = (struct ehca_cq *)data;
-
- ehca_err(&shca->ib_device,
- "CQ 0x%x (resource=%llx) has errors.",
- cq->cq_number, resource);
- break;
- }
- default:
- ehca_err(&shca->ib_device,
- "Unknown error type: %llx on %s.",
- type, shca->ib_device.name);
- break;
- }
-
- ehca_err(&shca->ib_device, "Error data is available: %llx.", resource);
- ehca_err(&shca->ib_device, "EHCA ----- error data begin "
- "---------------------------------------------------");
- ehca_dmp(rblock, length, "resource=%llx", resource);
- ehca_err(&shca->ib_device, "EHCA ----- error data end "
- "----------------------------------------------------");
-
- return;
-}
-
-int ehca_error_data(struct ehca_shca *shca, void *data,
- u64 resource)
-{
-
- unsigned long ret;
- u64 *rblock;
- unsigned long block_count;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Cannot allocate rblock memory.");
- ret = -ENOMEM;
- goto error_data1;
- }
-
- /* rblock must be 4K aligned and should be 4K large */
- ret = hipz_h_error_data(shca->ipz_hca_handle,
- resource,
- rblock,
- &block_count);
-
- if (ret == H_R_STATE)
- ehca_err(&shca->ib_device,
- "No error data is available: %llx.", resource);
- else if (ret == H_SUCCESS) {
- int length;
-
- length = EHCA_BMASK_GET(ERROR_DATA_LENGTH, rblock[0]);
-
- if (length > EHCA_PAGESIZE)
- length = EHCA_PAGESIZE;
-
- print_error_data(shca, data, rblock, length);
- } else
- ehca_err(&shca->ib_device,
- "Error data could not be fetched: %llx", resource);
-
- ehca_free_fw_ctrlblock(rblock);
-
-error_data1:
- return ret;
-
-}
-
-static void dispatch_qp_event(struct ehca_shca *shca, struct ehca_qp *qp,
- enum ib_event_type event_type)
-{
- struct ib_event event;
-
- /* PATH_MIG without the QP ever having been armed is false alarm */
- if (event_type == IB_EVENT_PATH_MIG && !qp->mig_armed)
- return;
-
- event.device = &shca->ib_device;
- event.event = event_type;
-
- if (qp->ext_type == EQPT_SRQ) {
- if (!qp->ib_srq.event_handler)
- return;
-
- event.element.srq = &qp->ib_srq;
- qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context);
- } else {
- if (!qp->ib_qp.event_handler)
- return;
-
- event.element.qp = &qp->ib_qp;
- qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
- }
-}
-
-static void qp_event_callback(struct ehca_shca *shca, u64 eqe,
- enum ib_event_type event_type, int fatal)
-{
- struct ehca_qp *qp;
- u32 token = EHCA_BMASK_GET(EQE_QP_TOKEN, eqe);
-
- read_lock(&ehca_qp_idr_lock);
- qp = idr_find(&ehca_qp_idr, token);
- if (qp)
- atomic_inc(&qp->nr_events);
- read_unlock(&ehca_qp_idr_lock);
-
- if (!qp)
- return;
-
- if (fatal)
- ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
-
- dispatch_qp_event(shca, qp, fatal && qp->ext_type == EQPT_SRQ ?
- IB_EVENT_SRQ_ERR : event_type);
-
- /*
- * eHCA only processes one WQE at a time for SRQ base QPs,
- * so the last WQE has been processed as soon as the QP enters
- * error state.
- */
- if (fatal && qp->ext_type == EQPT_SRQBASE)
- dispatch_qp_event(shca, qp, IB_EVENT_QP_LAST_WQE_REACHED);
-
- if (atomic_dec_and_test(&qp->nr_events))
- wake_up(&qp->wait_completion);
- return;
-}
-
-static void cq_event_callback(struct ehca_shca *shca,
- u64 eqe)
-{
- struct ehca_cq *cq;
- u32 token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe);
-
- read_lock(&ehca_cq_idr_lock);
- cq = idr_find(&ehca_cq_idr, token);
- if (cq)
- atomic_inc(&cq->nr_events);
- read_unlock(&ehca_cq_idr_lock);
-
- if (!cq)
- return;
-
- ehca_error_data(shca, cq, cq->ipz_cq_handle.handle);
-
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
-
- return;
-}
-
-static void parse_identifier(struct ehca_shca *shca, u64 eqe)
-{
- u8 identifier = EHCA_BMASK_GET(EQE_EE_IDENTIFIER, eqe);
-
- switch (identifier) {
- case 0x02: /* path migrated */
- qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0);
- break;
- case 0x03: /* communication established */
- qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0);
- break;
- case 0x04: /* send queue drained */
- qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0);
- break;
- case 0x05: /* QP error */
- case 0x06: /* QP error */
- qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1);
- break;
- case 0x07: /* CQ error */
- case 0x08: /* CQ error */
- cq_event_callback(shca, eqe);
- break;
- case 0x09: /* MRMWPTE error */
- ehca_err(&shca->ib_device, "MRMWPTE error.");
- break;
- case 0x0A: /* port event */
- ehca_err(&shca->ib_device, "Port event.");
- break;
- case 0x0B: /* MR access error */
- ehca_err(&shca->ib_device, "MR access error.");
- break;
- case 0x0C: /* EQ error */
- ehca_err(&shca->ib_device, "EQ error.");
- break;
- case 0x0D: /* P/Q_Key mismatch */
- ehca_err(&shca->ib_device, "P/Q_Key mismatch.");
- break;
- case 0x10: /* sampling complete */
- ehca_err(&shca->ib_device, "Sampling complete.");
- break;
- case 0x11: /* unaffiliated access error */
- ehca_err(&shca->ib_device, "Unaffiliated access error.");
- break;
- case 0x12: /* path migrating */
- ehca_err(&shca->ib_device, "Path migrating.");
- break;
- case 0x13: /* interface trace stopped */
- ehca_err(&shca->ib_device, "Interface trace stopped.");
- break;
- case 0x14: /* first error capture info available */
- ehca_info(&shca->ib_device, "First error capture available");
- break;
- case 0x15: /* SRQ limit reached */
- qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0);
- break;
- default:
- ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.",
- identifier, shca->ib_device.name);
- break;
- }
-
- return;
-}
-
-static void dispatch_port_event(struct ehca_shca *shca, int port_num,
- enum ib_event_type type, const char *msg)
-{
- struct ib_event event;
-
- ehca_info(&shca->ib_device, "port %d %s.", port_num, msg);
- event.device = &shca->ib_device;
- event.event = type;
- event.element.port_num = port_num;
- ib_dispatch_event(&event);
-}
-
-static void notify_port_conf_change(struct ehca_shca *shca, int port_num)
-{
- struct ehca_sma_attr new_attr;
- struct ehca_sma_attr *old_attr = &shca->sport[port_num - 1].saved_attr;
-
- ehca_query_sma_attr(shca, port_num, &new_attr);
-
- if (new_attr.sm_sl != old_attr->sm_sl ||
- new_attr.sm_lid != old_attr->sm_lid)
- dispatch_port_event(shca, port_num, IB_EVENT_SM_CHANGE,
- "SM changed");
-
- if (new_attr.lid != old_attr->lid ||
- new_attr.lmc != old_attr->lmc)
- dispatch_port_event(shca, port_num, IB_EVENT_LID_CHANGE,
- "LID changed");
-
- if (new_attr.pkey_tbl_len != old_attr->pkey_tbl_len ||
- memcmp(new_attr.pkeys, old_attr->pkeys,
- sizeof(u16) * new_attr.pkey_tbl_len))
- dispatch_port_event(shca, port_num, IB_EVENT_PKEY_CHANGE,
- "P_Key changed");
-
- *old_attr = new_attr;
-}
-
-/* replay modify_qp for sqps -- return 0 if all is well, 1 if AQP1 destroyed */
-static int replay_modify_qp(struct ehca_sport *sport)
-{
- int aqp1_destroyed;
- unsigned long flags;
-
- spin_lock_irqsave(&sport->mod_sqp_lock, flags);
-
- aqp1_destroyed = !sport->ibqp_sqp[IB_QPT_GSI];
-
- if (sport->ibqp_sqp[IB_QPT_SMI])
- ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_SMI]);
- if (!aqp1_destroyed)
- ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_GSI]);
-
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
-
- return aqp1_destroyed;
-}
-
-static void parse_ec(struct ehca_shca *shca, u64 eqe)
-{
- u8 ec = EHCA_BMASK_GET(NEQE_EVENT_CODE, eqe);
- u8 port = EHCA_BMASK_GET(NEQE_PORT_NUMBER, eqe);
- u8 spec_event;
- struct ehca_sport *sport = &shca->sport[port - 1];
-
- switch (ec) {
- case 0x30: /* port availability change */
- if (EHCA_BMASK_GET(NEQE_PORT_AVAILABILITY, eqe)) {
- /* only replay modify_qp calls in autodetect mode;
- * if AQP1 was destroyed, the port is already down
- * again and we can drop the event.
- */
- if (ehca_nr_ports < 0)
- if (replay_modify_qp(sport))
- break;
-
- sport->port_state = IB_PORT_ACTIVE;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
- "is active");
- ehca_query_sma_attr(shca, port, &sport->saved_attr);
- } else {
- sport->port_state = IB_PORT_DOWN;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
- "is inactive");
- }
- break;
- case 0x31:
- /* port configuration change
- * disruptive change is caused by
- * LID, PKEY or SM change
- */
- if (EHCA_BMASK_GET(NEQE_DISRUPTIVE, eqe)) {
- ehca_warn(&shca->ib_device, "disruptive port "
- "%d configuration change", port);
-
- sport->port_state = IB_PORT_DOWN;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
- "is inactive");
-
- sport->port_state = IB_PORT_ACTIVE;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
- "is active");
- ehca_query_sma_attr(shca, port,
- &sport->saved_attr);
- } else
- notify_port_conf_change(shca, port);
- break;
- case 0x32: /* adapter malfunction */
- ehca_err(&shca->ib_device, "Adapter malfunction.");
- break;
- case 0x33: /* trace stopped */
- ehca_err(&shca->ib_device, "Traced stopped.");
- break;
- case 0x34: /* util async event */
- spec_event = EHCA_BMASK_GET(NEQE_SPECIFIC_EVENT, eqe);
- if (spec_event == 0x80) /* client reregister required */
- dispatch_port_event(shca, port,
- IB_EVENT_CLIENT_REREGISTER,
- "client reregister req.");
- else
- ehca_warn(&shca->ib_device, "Unknown util async "
- "event %x on port %x", spec_event, port);
- break;
- default:
- ehca_err(&shca->ib_device, "Unknown event code: %x on %s.",
- ec, shca->ib_device.name);
- break;
- }
-
- return;
-}
-
-static inline void reset_eq_pending(struct ehca_cq *cq)
-{
- u64 CQx_EP;
- struct h_galpa gal = cq->galpas.kernel;
-
- hipz_galpa_store_cq(gal, cqx_ep, 0x0);
- CQx_EP = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_ep));
-
- return;
-}
-
-irqreturn_t ehca_interrupt_neq(int irq, void *dev_id)
-{
- struct ehca_shca *shca = (struct ehca_shca*)dev_id;
-
- tasklet_hi_schedule(&shca->neq.interrupt_task);
-
- return IRQ_HANDLED;
-}
-
-void ehca_tasklet_neq(unsigned long data)
-{
- struct ehca_shca *shca = (struct ehca_shca*)data;
- struct ehca_eqe *eqe;
- u64 ret;
-
- eqe = ehca_poll_eq(shca, &shca->neq);
-
- while (eqe) {
- if (!EHCA_BMASK_GET(NEQE_COMPLETION_EVENT, eqe->entry))
- parse_ec(shca, eqe->entry);
-
- eqe = ehca_poll_eq(shca, &shca->neq);
- }
-
- ret = hipz_h_reset_event(shca->ipz_hca_handle,
- shca->neq.ipz_eq_handle, 0xFFFFFFFFFFFFFFFFL);
-
- if (ret != H_SUCCESS)
- ehca_err(&shca->ib_device, "Can't clear notification events.");
-
- return;
-}
-
-irqreturn_t ehca_interrupt_eq(int irq, void *dev_id)
-{
- struct ehca_shca *shca = (struct ehca_shca*)dev_id;
-
- tasklet_hi_schedule(&shca->eq.interrupt_task);
-
- return IRQ_HANDLED;
-}
-
-
-static inline void process_eqe(struct ehca_shca *shca, struct ehca_eqe *eqe)
-{
- u64 eqe_value;
- u32 token;
- struct ehca_cq *cq;
-
- eqe_value = eqe->entry;
- ehca_dbg(&shca->ib_device, "eqe_value=%llx", eqe_value);
- if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
- ehca_dbg(&shca->ib_device, "Got completion event");
- token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
- read_lock(&ehca_cq_idr_lock);
- cq = idr_find(&ehca_cq_idr, token);
- if (cq)
- atomic_inc(&cq->nr_events);
- read_unlock(&ehca_cq_idr_lock);
- if (cq == NULL) {
- ehca_err(&shca->ib_device,
- "Invalid eqe for non-existing cq token=%x",
- token);
- return;
- }
- reset_eq_pending(cq);
- if (ehca_scaling_code)
- queue_comp_task(cq);
- else {
- comp_event_callback(cq);
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
- }
- } else {
- ehca_dbg(&shca->ib_device, "Got non completion event");
- parse_identifier(shca, eqe_value);
- }
-}
-
-void ehca_process_eq(struct ehca_shca *shca, int is_irq)
-{
- struct ehca_eq *eq = &shca->eq;
- struct ehca_eqe_cache_entry *eqe_cache = eq->eqe_cache;
- u64 eqe_value, ret;
- int eqe_cnt, i;
- int eq_empty = 0;
-
- spin_lock(&eq->irq_spinlock);
- if (is_irq) {
- const int max_query_cnt = 100;
- int query_cnt = 0;
- int int_state = 1;
- do {
- int_state = hipz_h_query_int_state(
- shca->ipz_hca_handle, eq->ist);
- query_cnt++;
- iosync();
- } while (int_state && query_cnt < max_query_cnt);
- if (unlikely((query_cnt == max_query_cnt)))
- ehca_dbg(&shca->ib_device, "int_state=%x query_cnt=%x",
- int_state, query_cnt);
- }
-
- /* read out all eqes */
- eqe_cnt = 0;
- do {
- u32 token;
- eqe_cache[eqe_cnt].eqe = ehca_poll_eq(shca, eq);
- if (!eqe_cache[eqe_cnt].eqe)
- break;
- eqe_value = eqe_cache[eqe_cnt].eqe->entry;
- if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
- token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
- read_lock(&ehca_cq_idr_lock);
- eqe_cache[eqe_cnt].cq = idr_find(&ehca_cq_idr, token);
- if (eqe_cache[eqe_cnt].cq)
- atomic_inc(&eqe_cache[eqe_cnt].cq->nr_events);
- read_unlock(&ehca_cq_idr_lock);
- if (!eqe_cache[eqe_cnt].cq) {
- ehca_err(&shca->ib_device,
- "Invalid eqe for non-existing cq "
- "token=%x", token);
- continue;
- }
- } else
- eqe_cache[eqe_cnt].cq = NULL;
- eqe_cnt++;
- } while (eqe_cnt < EHCA_EQE_CACHE_SIZE);
- if (!eqe_cnt) {
- if (is_irq)
- ehca_dbg(&shca->ib_device,
- "No eqe found for irq event");
- goto unlock_irq_spinlock;
- } else if (!is_irq) {
- ret = hipz_h_eoi(eq->ist);
- if (ret != H_SUCCESS)
- ehca_err(&shca->ib_device,
- "bad return code EOI -rc = %lld\n", ret);
- ehca_dbg(&shca->ib_device, "deadman found %x eqe", eqe_cnt);
- }
- if (unlikely(eqe_cnt == EHCA_EQE_CACHE_SIZE))
- ehca_dbg(&shca->ib_device, "too many eqes for one irq event");
- /* enable irq for new packets */
- for (i = 0; i < eqe_cnt; i++) {
- if (eq->eqe_cache[i].cq)
- reset_eq_pending(eq->eqe_cache[i].cq);
- }
- /* check eq */
- spin_lock(&eq->spinlock);
- eq_empty = (!ipz_eqit_eq_peek_valid(&shca->eq.ipz_queue));
- spin_unlock(&eq->spinlock);
- /* call completion handler for cached eqes */
- for (i = 0; i < eqe_cnt; i++)
- if (eq->eqe_cache[i].cq) {
- if (ehca_scaling_code)
- queue_comp_task(eq->eqe_cache[i].cq);
- else {
- struct ehca_cq *cq = eq->eqe_cache[i].cq;
- comp_event_callback(cq);
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
- }
- } else {
- ehca_dbg(&shca->ib_device, "Got non completion event");
- parse_identifier(shca, eq->eqe_cache[i].eqe->entry);
- }
- /* poll eq if not empty */
- if (eq_empty)
- goto unlock_irq_spinlock;
- do {
- struct ehca_eqe *eqe;
- eqe = ehca_poll_eq(shca, &shca->eq);
- if (!eqe)
- break;
- process_eqe(shca, eqe);
- } while (1);
-
-unlock_irq_spinlock:
- spin_unlock(&eq->irq_spinlock);
-}
-
-void ehca_tasklet_eq(unsigned long data)
-{
- ehca_process_eq((struct ehca_shca*)data, 1);
-}
-
-static int find_next_online_cpu(struct ehca_comp_pool *pool)
-{
- int cpu;
- unsigned long flags;
-
- WARN_ON_ONCE(!in_interrupt());
- if (ehca_debug_level >= 3)
- ehca_dmp(cpu_online_mask, cpumask_size(), "");
-
- spin_lock_irqsave(&pool->last_cpu_lock, flags);
- do {
- cpu = cpumask_next(pool->last_cpu, cpu_online_mask);
- if (cpu >= nr_cpu_ids)
- cpu = cpumask_first(cpu_online_mask);
- pool->last_cpu = cpu;
- } while (!per_cpu_ptr(pool->cpu_comp_tasks, cpu)->active);
- spin_unlock_irqrestore(&pool->last_cpu_lock, flags);
-
- return cpu;
-}
-
-static void __queue_comp_task(struct ehca_cq *__cq,
- struct ehca_cpu_comp_task *cct,
- struct task_struct *thread)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&cct->task_lock, flags);
- spin_lock(&__cq->task_lock);
-
- if (__cq->nr_callbacks == 0) {
- __cq->nr_callbacks++;
- list_add_tail(&__cq->entry, &cct->cq_list);
- cct->cq_jobs++;
- wake_up_process(thread);
- } else
- __cq->nr_callbacks++;
-
- spin_unlock(&__cq->task_lock);
- spin_unlock_irqrestore(&cct->task_lock, flags);
-}
-
-static void queue_comp_task(struct ehca_cq *__cq)
-{
- int cpu_id;
- struct ehca_cpu_comp_task *cct;
- struct task_struct *thread;
- int cq_jobs;
- unsigned long flags;
-
- cpu_id = find_next_online_cpu(pool);
- BUG_ON(!cpu_online(cpu_id));
-
- cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
- thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
- BUG_ON(!cct || !thread);
-
- spin_lock_irqsave(&cct->task_lock, flags);
- cq_jobs = cct->cq_jobs;
- spin_unlock_irqrestore(&cct->task_lock, flags);
- if (cq_jobs > 0) {
- cpu_id = find_next_online_cpu(pool);
- cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
- thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
- BUG_ON(!cct || !thread);
- }
- __queue_comp_task(__cq, cct, thread);
-}
-
-static void run_comp_task(struct ehca_cpu_comp_task *cct)
-{
- struct ehca_cq *cq;
-
- while (!list_empty(&cct->cq_list)) {
- cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
- spin_unlock_irq(&cct->task_lock);
-
- comp_event_callback(cq);
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
-
- spin_lock_irq(&cct->task_lock);
- spin_lock(&cq->task_lock);
- cq->nr_callbacks--;
- if (!cq->nr_callbacks) {
- list_del_init(cct->cq_list.next);
- cct->cq_jobs--;
- }
- spin_unlock(&cq->task_lock);
- }
-}
-
-static void comp_task_park(unsigned int cpu)
-{
- struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
- struct ehca_cpu_comp_task *target;
- struct task_struct *thread;
- struct ehca_cq *cq, *tmp;
- LIST_HEAD(list);
-
- spin_lock_irq(&cct->task_lock);
- cct->cq_jobs = 0;
- cct->active = 0;
- list_splice_init(&cct->cq_list, &list);
- spin_unlock_irq(&cct->task_lock);
-
- cpu = find_next_online_cpu(pool);
- target = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
- thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu);
- spin_lock_irq(&target->task_lock);
- list_for_each_entry_safe(cq, tmp, &list, entry) {
- list_del(&cq->entry);
- __queue_comp_task(cq, target, thread);
- }
- spin_unlock_irq(&target->task_lock);
-}
-
-static void comp_task_stop(unsigned int cpu, bool online)
-{
- struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
-
- spin_lock_irq(&cct->task_lock);
- cct->cq_jobs = 0;
- cct->active = 0;
- WARN_ON(!list_empty(&cct->cq_list));
- spin_unlock_irq(&cct->task_lock);
-}
-
-static int comp_task_should_run(unsigned int cpu)
-{
- struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
-
- return cct->cq_jobs;
-}
-
-static void comp_task(unsigned int cpu)
-{
- struct ehca_cpu_comp_task *cct = this_cpu_ptr(pool->cpu_comp_tasks);
- int cql_empty;
-
- spin_lock_irq(&cct->task_lock);
- cql_empty = list_empty(&cct->cq_list);
- if (!cql_empty) {
- __set_current_state(TASK_RUNNING);
- run_comp_task(cct);
- }
- spin_unlock_irq(&cct->task_lock);
-}
-
-static struct smp_hotplug_thread comp_pool_threads = {
- .thread_should_run = comp_task_should_run,
- .thread_fn = comp_task,
- .thread_comm = "ehca_comp/%u",
- .cleanup = comp_task_stop,
- .park = comp_task_park,
-};
-
-int ehca_create_comp_pool(void)
-{
- int cpu, ret = -ENOMEM;
-
- if (!ehca_scaling_code)
- return 0;
-
- pool = kzalloc(sizeof(struct ehca_comp_pool), GFP_KERNEL);
- if (pool == NULL)
- return -ENOMEM;
-
- spin_lock_init(&pool->last_cpu_lock);
- pool->last_cpu = cpumask_any(cpu_online_mask);
-
- pool->cpu_comp_tasks = alloc_percpu(struct ehca_cpu_comp_task);
- if (!pool->cpu_comp_tasks)
- goto out_pool;
-
- pool->cpu_comp_threads = alloc_percpu(struct task_struct *);
- if (!pool->cpu_comp_threads)
- goto out_tasks;
-
- for_each_present_cpu(cpu) {
- struct ehca_cpu_comp_task *cct;
-
- cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
- spin_lock_init(&cct->task_lock);
- INIT_LIST_HEAD(&cct->cq_list);
- }
-
- comp_pool_threads.store = pool->cpu_comp_threads;
- ret = smpboot_register_percpu_thread(&comp_pool_threads);
- if (ret)
- goto out_threads;
-
- pr_info("eHCA scaling code enabled\n");
- return ret;
-
-out_threads:
- free_percpu(pool->cpu_comp_threads);
-out_tasks:
- free_percpu(pool->cpu_comp_tasks);
-out_pool:
- kfree(pool);
- return ret;
-}
-
-void ehca_destroy_comp_pool(void)
-{
- if (!ehca_scaling_code)
- return;
-
- smpboot_unregister_percpu_thread(&comp_pool_threads);
-
- free_percpu(pool->cpu_comp_threads);
- free_percpu(pool->cpu_comp_tasks);
- kfree(pool);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Function definitions and structs for EQs, NEQs and interrupts
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_IRQ_H
-#define __EHCA_IRQ_H
-
-
-struct ehca_shca;
-
-#include <linux/interrupt.h>
-#include <linux/types.h>
-
-int ehca_error_data(struct ehca_shca *shca, void *data, u64 resource);
-
-irqreturn_t ehca_interrupt_neq(int irq, void *dev_id);
-void ehca_tasklet_neq(unsigned long data);
-
-irqreturn_t ehca_interrupt_eq(int irq, void *dev_id);
-void ehca_tasklet_eq(unsigned long data);
-void ehca_process_eq(struct ehca_shca *shca, int is_irq);
-
-struct ehca_cpu_comp_task {
- struct list_head cq_list;
- spinlock_t task_lock;
- int cq_jobs;
- int active;
-};
-
-struct ehca_comp_pool {
- struct ehca_cpu_comp_task __percpu *cpu_comp_tasks;
- struct task_struct * __percpu *cpu_comp_threads;
- int last_cpu;
- spinlock_t last_cpu_lock;
-};
-
-int ehca_create_comp_pool(void);
-void ehca_destroy_comp_pool(void);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Function definitions for internal functions
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Dietmar Decker <ddecker@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_IVERBS_H__
-#define __EHCA_IVERBS_H__
-
-#include "ehca_classes.h"
-
-int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
- struct ib_udata *uhw);
-
-int ehca_query_port(struct ib_device *ibdev, u8 port,
- struct ib_port_attr *props);
-
-enum rdma_protocol_type
-ehca_query_protocol(struct ib_device *device, u8 port_num);
-
-int ehca_query_sma_attr(struct ehca_shca *shca, u8 port,
- struct ehca_sma_attr *attr);
-
-int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 * pkey);
-
-int ehca_query_gid(struct ib_device *ibdev, u8 port, int index,
- union ib_gid *gid);
-
-int ehca_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask,
- struct ib_port_modify *props);
-
-struct ib_pd *ehca_alloc_pd(struct ib_device *device,
- struct ib_ucontext *context,
- struct ib_udata *udata);
-
-int ehca_dealloc_pd(struct ib_pd *pd);
-
-struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
-
-int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
-
-int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
-
-int ehca_destroy_ah(struct ib_ah *ah);
-
-struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
-
-struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- int mr_access_flags, u64 *iova_start);
-
-struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
- u64 virt, int mr_access_flags,
- struct ib_udata *udata);
-
-int ehca_rereg_phys_mr(struct ib_mr *mr,
- int mr_rereg_mask,
- struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf, int mr_access_flags, u64 *iova_start);
-
-int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr);
-
-int ehca_dereg_mr(struct ib_mr *mr);
-
-struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type);
-
-int ehca_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
- struct ib_mw_bind *mw_bind);
-
-int ehca_dealloc_mw(struct ib_mw *mw);
-
-struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
- int mr_access_flags,
- struct ib_fmr_attr *fmr_attr);
-
-int ehca_map_phys_fmr(struct ib_fmr *fmr,
- u64 *page_list, int list_len, u64 iova);
-
-int ehca_unmap_fmr(struct list_head *fmr_list);
-
-int ehca_dealloc_fmr(struct ib_fmr *fmr);
-
-enum ehca_eq_type {
- EHCA_EQ = 0, /* Event Queue */
- EHCA_NEQ /* Notification Event Queue */
-};
-
-int ehca_create_eq(struct ehca_shca *shca, struct ehca_eq *eq,
- enum ehca_eq_type type, const u32 length);
-
-int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq);
-
-void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq);
-
-
-struct ib_cq *ehca_create_cq(struct ib_device *device,
- const struct ib_cq_init_attr *attr,
- struct ib_ucontext *context,
- struct ib_udata *udata);
-
-int ehca_destroy_cq(struct ib_cq *cq);
-
-int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
-
-int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
-
-int ehca_peek_cq(struct ib_cq *cq, int wc_cnt);
-
-int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags);
-
-struct ib_qp *ehca_create_qp(struct ib_pd *pd,
- struct ib_qp_init_attr *init_attr,
- struct ib_udata *udata);
-
-int ehca_destroy_qp(struct ib_qp *qp);
-
-int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
- struct ib_udata *udata);
-
-int ehca_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
- int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
-
-int ehca_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr,
- struct ib_send_wr **bad_send_wr);
-
-int ehca_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr);
-
-int ehca_post_srq_recv(struct ib_srq *srq,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr);
-
-struct ib_srq *ehca_create_srq(struct ib_pd *pd,
- struct ib_srq_init_attr *init_attr,
- struct ib_udata *udata);
-
-int ehca_modify_srq(struct ib_srq *srq, struct ib_srq_attr *attr,
- enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
-
-int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
-
-int ehca_destroy_srq(struct ib_srq *srq);
-
-u64 ehca_define_sqp(struct ehca_shca *shca, struct ehca_qp *ibqp,
- struct ib_qp_init_attr *qp_init_attr);
-
-int ehca_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
-
-int ehca_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
-
-struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
- struct ib_udata *udata);
-
-int ehca_dealloc_ucontext(struct ib_ucontext *context);
-
-int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
-
-int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
- const struct ib_wc *in_wc, const struct ib_grh *in_grh,
- const struct ib_mad_hdr *in, size_t in_mad_size,
- struct ib_mad_hdr *out, size_t *out_mad_size,
- u16 *out_mad_pkey_index);
-
-void ehca_poll_eqs(unsigned long data);
-
-int ehca_calc_ipd(struct ehca_shca *shca, int port,
- enum ib_rate path_rate, u32 *ipd);
-
-void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq);
-
-#ifdef CONFIG_PPC_64K_PAGES
-void *ehca_alloc_fw_ctrlblock(gfp_t flags);
-void ehca_free_fw_ctrlblock(void *ptr);
-#else
-#define ehca_alloc_fw_ctrlblock(flags) ((void *)get_zeroed_page(flags))
-#define ehca_free_fw_ctrlblock(ptr) free_page((unsigned long)(ptr))
-#endif
-
-void ehca_recover_sqp(struct ib_qp *sqp);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * module start stop, hca detection
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifdef CONFIG_PPC_64K_PAGES
-#include <linux/slab.h>
-#endif
-
-#include <linux/notifier.h>
-#include <linux/memory.h>
-#include <rdma/ib_mad.h>
-#include "ehca_classes.h"
-#include "ehca_iverbs.h"
-#include "ehca_mrmw.h"
-#include "ehca_tools.h"
-#include "hcp_if.h"
-
-#define HCAD_VERSION "0029"
-
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
-MODULE_DESCRIPTION("IBM eServer HCA InfiniBand Device Driver");
-MODULE_VERSION(HCAD_VERSION);
-
-static bool ehca_open_aqp1 = 0;
-static int ehca_hw_level = 0;
-static bool ehca_poll_all_eqs = 1;
-
-int ehca_debug_level = 0;
-int ehca_nr_ports = -1;
-bool ehca_use_hp_mr = 0;
-int ehca_port_act_time = 30;
-int ehca_static_rate = -1;
-bool ehca_scaling_code = 0;
-int ehca_lock_hcalls = -1;
-int ehca_max_cq = -1;
-int ehca_max_qp = -1;
-
-module_param_named(open_aqp1, ehca_open_aqp1, bool, S_IRUGO);
-module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
-module_param_named(hw_level, ehca_hw_level, int, S_IRUGO);
-module_param_named(nr_ports, ehca_nr_ports, int, S_IRUGO);
-module_param_named(use_hp_mr, ehca_use_hp_mr, bool, S_IRUGO);
-module_param_named(port_act_time, ehca_port_act_time, int, S_IRUGO);
-module_param_named(poll_all_eqs, ehca_poll_all_eqs, bool, S_IRUGO);
-module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
-module_param_named(scaling_code, ehca_scaling_code, bool, S_IRUGO);
-module_param_named(lock_hcalls, ehca_lock_hcalls, bint, S_IRUGO);
-module_param_named(number_of_cqs, ehca_max_cq, int, S_IRUGO);
-module_param_named(number_of_qps, ehca_max_qp, int, S_IRUGO);
-
-MODULE_PARM_DESC(open_aqp1,
- "Open AQP1 on startup (default: no)");
-MODULE_PARM_DESC(debug_level,
- "Amount of debug output (0: none (default), 1: traces, "
- "2: some dumps, 3: lots)");
-MODULE_PARM_DESC(hw_level,
- "Hardware level (0: autosensing (default), "
- "0x10..0x14: eHCA, 0x20..0x23: eHCA2)");
-MODULE_PARM_DESC(nr_ports,
- "number of connected ports (-1: autodetect (default), "
- "1: port one only, 2: two ports)");
-MODULE_PARM_DESC(use_hp_mr,
- "Use high performance MRs (default: no)");
-MODULE_PARM_DESC(port_act_time,
- "Time to wait for port activation (default: 30 sec)");
-MODULE_PARM_DESC(poll_all_eqs,
- "Poll all event queues periodically (default: yes)");
-MODULE_PARM_DESC(static_rate,
- "Set permanent static rate (default: no static rate)");
-MODULE_PARM_DESC(scaling_code,
- "Enable scaling code (default: no)");
-MODULE_PARM_DESC(lock_hcalls,
- "Serialize all hCalls made by the driver "
- "(default: autodetect)");
-MODULE_PARM_DESC(number_of_cqs,
- "Max number of CQs which can be allocated "
- "(default: autodetect)");
-MODULE_PARM_DESC(number_of_qps,
- "Max number of QPs which can be allocated "
- "(default: autodetect)");
-
-DEFINE_RWLOCK(ehca_qp_idr_lock);
-DEFINE_RWLOCK(ehca_cq_idr_lock);
-DEFINE_IDR(ehca_qp_idr);
-DEFINE_IDR(ehca_cq_idr);
-
-static LIST_HEAD(shca_list); /* list of all registered ehcas */
-DEFINE_SPINLOCK(shca_list_lock);
-
-static struct timer_list poll_eqs_timer;
-
-#ifdef CONFIG_PPC_64K_PAGES
-static struct kmem_cache *ctblk_cache;
-
-void *ehca_alloc_fw_ctrlblock(gfp_t flags)
-{
- void *ret = kmem_cache_zalloc(ctblk_cache, flags);
- if (!ret)
- ehca_gen_err("Out of memory for ctblk");
- return ret;
-}
-
-void ehca_free_fw_ctrlblock(void *ptr)
-{
- if (ptr)
- kmem_cache_free(ctblk_cache, ptr);
-
-}
-#endif
-
-int ehca2ib_return_code(u64 ehca_rc)
-{
- switch (ehca_rc) {
- case H_SUCCESS:
- return 0;
- case H_RESOURCE: /* Resource in use */
- case H_BUSY:
- return -EBUSY;
- case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
- case H_CONSTRAINED: /* resource constraint */
- case H_NO_MEM:
- return -ENOMEM;
- default:
- return -EINVAL;
- }
-}
-
-static int ehca_create_slab_caches(void)
-{
- int ret;
-
- ret = ehca_init_pd_cache();
- if (ret) {
- ehca_gen_err("Cannot create PD SLAB cache.");
- return ret;
- }
-
- ret = ehca_init_cq_cache();
- if (ret) {
- ehca_gen_err("Cannot create CQ SLAB cache.");
- goto create_slab_caches2;
- }
-
- ret = ehca_init_qp_cache();
- if (ret) {
- ehca_gen_err("Cannot create QP SLAB cache.");
- goto create_slab_caches3;
- }
-
- ret = ehca_init_av_cache();
- if (ret) {
- ehca_gen_err("Cannot create AV SLAB cache.");
- goto create_slab_caches4;
- }
-
- ret = ehca_init_mrmw_cache();
- if (ret) {
- ehca_gen_err("Cannot create MR&MW SLAB cache.");
- goto create_slab_caches5;
- }
-
- ret = ehca_init_small_qp_cache();
- if (ret) {
- ehca_gen_err("Cannot create small queue SLAB cache.");
- goto create_slab_caches6;
- }
-
-#ifdef CONFIG_PPC_64K_PAGES
- ctblk_cache = kmem_cache_create("ehca_cache_ctblk",
- EHCA_PAGESIZE, H_CB_ALIGNMENT,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!ctblk_cache) {
- ehca_gen_err("Cannot create ctblk SLAB cache.");
- ehca_cleanup_small_qp_cache();
- ret = -ENOMEM;
- goto create_slab_caches6;
- }
-#endif
- return 0;
-
-create_slab_caches6:
- ehca_cleanup_mrmw_cache();
-
-create_slab_caches5:
- ehca_cleanup_av_cache();
-
-create_slab_caches4:
- ehca_cleanup_qp_cache();
-
-create_slab_caches3:
- ehca_cleanup_cq_cache();
-
-create_slab_caches2:
- ehca_cleanup_pd_cache();
-
- return ret;
-}
-
-static void ehca_destroy_slab_caches(void)
-{
- ehca_cleanup_small_qp_cache();
- ehca_cleanup_mrmw_cache();
- ehca_cleanup_av_cache();
- ehca_cleanup_qp_cache();
- ehca_cleanup_cq_cache();
- ehca_cleanup_pd_cache();
-#ifdef CONFIG_PPC_64K_PAGES
- if (ctblk_cache)
- kmem_cache_destroy(ctblk_cache);
-#endif
-}
-
-#define EHCA_HCAAVER EHCA_BMASK_IBM(32, 39)
-#define EHCA_REVID EHCA_BMASK_IBM(40, 63)
-
-static struct cap_descr {
- u64 mask;
- char *descr;
-} hca_cap_descr[] = {
- { HCA_CAP_AH_PORT_NR_CHECK, "HCA_CAP_AH_PORT_NR_CHECK" },
- { HCA_CAP_ATOMIC, "HCA_CAP_ATOMIC" },
- { HCA_CAP_AUTO_PATH_MIG, "HCA_CAP_AUTO_PATH_MIG" },
- { HCA_CAP_BAD_P_KEY_CTR, "HCA_CAP_BAD_P_KEY_CTR" },
- { HCA_CAP_SQD_RTS_PORT_CHANGE, "HCA_CAP_SQD_RTS_PORT_CHANGE" },
- { HCA_CAP_CUR_QP_STATE_MOD, "HCA_CAP_CUR_QP_STATE_MOD" },
- { HCA_CAP_INIT_TYPE, "HCA_CAP_INIT_TYPE" },
- { HCA_CAP_PORT_ACTIVE_EVENT, "HCA_CAP_PORT_ACTIVE_EVENT" },
- { HCA_CAP_Q_KEY_VIOL_CTR, "HCA_CAP_Q_KEY_VIOL_CTR" },
- { HCA_CAP_WQE_RESIZE, "HCA_CAP_WQE_RESIZE" },
- { HCA_CAP_RAW_PACKET_MCAST, "HCA_CAP_RAW_PACKET_MCAST" },
- { HCA_CAP_SHUTDOWN_PORT, "HCA_CAP_SHUTDOWN_PORT" },
- { HCA_CAP_RC_LL_QP, "HCA_CAP_RC_LL_QP" },
- { HCA_CAP_SRQ, "HCA_CAP_SRQ" },
- { HCA_CAP_UD_LL_QP, "HCA_CAP_UD_LL_QP" },
- { HCA_CAP_RESIZE_MR, "HCA_CAP_RESIZE_MR" },
- { HCA_CAP_MINI_QP, "HCA_CAP_MINI_QP" },
- { HCA_CAP_H_ALLOC_RES_SYNC, "HCA_CAP_H_ALLOC_RES_SYNC" },
-};
-
-static int ehca_sense_attributes(struct ehca_shca *shca)
-{
- int i, ret = 0;
- u64 h_ret;
- struct hipz_query_hca *rblock;
- struct hipz_query_port *port;
- const char *loc_code;
-
- static const u32 pgsize_map[] = {
- HCA_CAP_MR_PGSIZE_4K, 0x1000,
- HCA_CAP_MR_PGSIZE_64K, 0x10000,
- HCA_CAP_MR_PGSIZE_1M, 0x100000,
- HCA_CAP_MR_PGSIZE_16M, 0x1000000,
- };
-
- ehca_gen_dbg("Probing adapter %s...",
- shca->ofdev->dev.of_node->full_name);
- loc_code = of_get_property(shca->ofdev->dev.of_node, "ibm,loc-code",
- NULL);
- if (loc_code)
- ehca_gen_dbg(" ... location lode=%s", loc_code);
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_gen_err("Cannot allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_hca(shca->ipz_hca_handle, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_gen_err("Cannot query device properties. h_ret=%lli",
- h_ret);
- ret = -EPERM;
- goto sense_attributes1;
- }
-
- if (ehca_nr_ports == 1)
- shca->num_ports = 1;
- else
- shca->num_ports = (u8)rblock->num_ports;
-
- ehca_gen_dbg(" ... found %x ports", rblock->num_ports);
-
- if (ehca_hw_level == 0) {
- u32 hcaaver;
- u32 revid;
-
- hcaaver = EHCA_BMASK_GET(EHCA_HCAAVER, rblock->hw_ver);
- revid = EHCA_BMASK_GET(EHCA_REVID, rblock->hw_ver);
-
- ehca_gen_dbg(" ... hardware version=%x:%x", hcaaver, revid);
-
- if (hcaaver == 1) {
- if (revid <= 3)
- shca->hw_level = 0x10 | (revid + 1);
- else
- shca->hw_level = 0x14;
- } else if (hcaaver == 2) {
- if (revid == 0)
- shca->hw_level = 0x21;
- else if (revid == 0x10)
- shca->hw_level = 0x22;
- else if (revid == 0x20 || revid == 0x21)
- shca->hw_level = 0x23;
- }
-
- if (!shca->hw_level) {
- ehca_gen_warn("unknown hardware version"
- " - assuming default level");
- shca->hw_level = 0x22;
- }
- } else
- shca->hw_level = ehca_hw_level;
- ehca_gen_dbg(" ... hardware level=%x", shca->hw_level);
-
- shca->hca_cap = rblock->hca_cap_indicators;
- ehca_gen_dbg(" ... HCA capabilities:");
- for (i = 0; i < ARRAY_SIZE(hca_cap_descr); i++)
- if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap))
- ehca_gen_dbg(" %s", hca_cap_descr[i].descr);
-
- /* Autodetect hCall locking -- the "H_ALLOC_RESOURCE synced" flag is
- * a firmware property, so it's valid across all adapters
- */
- if (ehca_lock_hcalls == -1)
- ehca_lock_hcalls = !EHCA_BMASK_GET(HCA_CAP_H_ALLOC_RES_SYNC,
- shca->hca_cap);
-
- /* translate supported MR page sizes; always support 4K */
- shca->hca_cap_mr_pgsize = EHCA_PAGESIZE;
- for (i = 0; i < ARRAY_SIZE(pgsize_map); i += 2)
- if (rblock->memory_page_size_supported & pgsize_map[i])
- shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];
-
- /* Set maximum number of CQs and QPs to calculate EQ size */
- if (shca->max_num_qps == -1)
- shca->max_num_qps = min_t(int, rblock->max_qp,
- EHCA_MAX_NUM_QUEUES);
- else if (shca->max_num_qps < 1 || shca->max_num_qps > rblock->max_qp) {
- ehca_gen_warn("The requested number of QPs is out of range "
- "(1 - %i) specified by HW. Value is set to %i",
- rblock->max_qp, rblock->max_qp);
- shca->max_num_qps = rblock->max_qp;
- }
-
- if (shca->max_num_cqs == -1)
- shca->max_num_cqs = min_t(int, rblock->max_cq,
- EHCA_MAX_NUM_QUEUES);
- else if (shca->max_num_cqs < 1 || shca->max_num_cqs > rblock->max_cq) {
- ehca_gen_warn("The requested number of CQs is out of range "
- "(1 - %i) specified by HW. Value is set to %i",
- rblock->max_cq, rblock->max_cq);
- }
-
- /* query max MTU from first port -- it's the same for all ports */
- port = (struct hipz_query_port *)rblock;
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
- if (h_ret != H_SUCCESS) {
- ehca_gen_err("Cannot query port properties. h_ret=%lli",
- h_ret);
- ret = -EPERM;
- goto sense_attributes1;
- }
-
- shca->max_mtu = port->max_mtu;
-
-sense_attributes1:
- ehca_free_fw_ctrlblock(rblock);
- return ret;
-}
-
-static int init_node_guid(struct ehca_shca *shca)
-{
- int ret = 0;
- struct hipz_query_hca *rblock;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query device properties");
- ret = -EINVAL;
- goto init_node_guid1;
- }
-
- memcpy(&shca->ib_device.node_guid, &rblock->node_guid, sizeof(u64));
-
-init_node_guid1:
- ehca_free_fw_ctrlblock(rblock);
- return ret;
-}
-
-static int ehca_port_immutable(struct ib_device *ibdev, u8 port_num,
- struct ib_port_immutable *immutable)
-{
- struct ib_port_attr attr;
- int err;
-
- err = ehca_query_port(ibdev, port_num, &attr);
- if (err)
- return err;
-
- immutable->pkey_tbl_len = attr.pkey_tbl_len;
- immutable->gid_tbl_len = attr.gid_tbl_len;
- immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
- immutable->max_mad_size = IB_MGMT_MAD_SIZE;
-
- return 0;
-}
-
-static int ehca_init_device(struct ehca_shca *shca)
-{
- int ret;
-
- ret = init_node_guid(shca);
- if (ret)
- return ret;
-
- strlcpy(shca->ib_device.name, "ehca%d", IB_DEVICE_NAME_MAX);
- shca->ib_device.owner = THIS_MODULE;
-
- shca->ib_device.uverbs_abi_ver = 8;
- shca->ib_device.uverbs_cmd_mask =
- (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
- (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
- (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
- (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_REG_MR) |
- (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
- (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
- (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
- (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
- (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
- (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
- (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
- (1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
-
- shca->ib_device.node_type = RDMA_NODE_IB_CA;
- shca->ib_device.phys_port_cnt = shca->num_ports;
- shca->ib_device.num_comp_vectors = 1;
- shca->ib_device.dma_device = &shca->ofdev->dev;
- shca->ib_device.query_device = ehca_query_device;
- shca->ib_device.query_port = ehca_query_port;
- shca->ib_device.query_gid = ehca_query_gid;
- shca->ib_device.query_pkey = ehca_query_pkey;
- /* shca->in_device.modify_device = ehca_modify_device */
- shca->ib_device.modify_port = ehca_modify_port;
- shca->ib_device.alloc_ucontext = ehca_alloc_ucontext;
- shca->ib_device.dealloc_ucontext = ehca_dealloc_ucontext;
- shca->ib_device.alloc_pd = ehca_alloc_pd;
- shca->ib_device.dealloc_pd = ehca_dealloc_pd;
- shca->ib_device.create_ah = ehca_create_ah;
- /* shca->ib_device.modify_ah = ehca_modify_ah; */
- shca->ib_device.query_ah = ehca_query_ah;
- shca->ib_device.destroy_ah = ehca_destroy_ah;
- shca->ib_device.create_qp = ehca_create_qp;
- shca->ib_device.modify_qp = ehca_modify_qp;
- shca->ib_device.query_qp = ehca_query_qp;
- shca->ib_device.destroy_qp = ehca_destroy_qp;
- shca->ib_device.post_send = ehca_post_send;
- shca->ib_device.post_recv = ehca_post_recv;
- shca->ib_device.create_cq = ehca_create_cq;
- shca->ib_device.destroy_cq = ehca_destroy_cq;
- shca->ib_device.resize_cq = ehca_resize_cq;
- shca->ib_device.poll_cq = ehca_poll_cq;
- /* shca->ib_device.peek_cq = ehca_peek_cq; */
- shca->ib_device.req_notify_cq = ehca_req_notify_cq;
- /* shca->ib_device.req_ncomp_notif = ehca_req_ncomp_notif; */
- shca->ib_device.get_dma_mr = ehca_get_dma_mr;
- shca->ib_device.reg_phys_mr = ehca_reg_phys_mr;
- shca->ib_device.reg_user_mr = ehca_reg_user_mr;
- shca->ib_device.query_mr = ehca_query_mr;
- shca->ib_device.dereg_mr = ehca_dereg_mr;
- shca->ib_device.rereg_phys_mr = ehca_rereg_phys_mr;
- shca->ib_device.alloc_mw = ehca_alloc_mw;
- shca->ib_device.bind_mw = ehca_bind_mw;
- shca->ib_device.dealloc_mw = ehca_dealloc_mw;
- shca->ib_device.alloc_fmr = ehca_alloc_fmr;
- shca->ib_device.map_phys_fmr = ehca_map_phys_fmr;
- shca->ib_device.unmap_fmr = ehca_unmap_fmr;
- shca->ib_device.dealloc_fmr = ehca_dealloc_fmr;
- shca->ib_device.attach_mcast = ehca_attach_mcast;
- shca->ib_device.detach_mcast = ehca_detach_mcast;
- shca->ib_device.process_mad = ehca_process_mad;
- shca->ib_device.mmap = ehca_mmap;
- shca->ib_device.dma_ops = &ehca_dma_mapping_ops;
- shca->ib_device.get_port_immutable = ehca_port_immutable;
-
- if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
- shca->ib_device.uverbs_cmd_mask |=
- (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
- (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
- (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
-
- shca->ib_device.create_srq = ehca_create_srq;
- shca->ib_device.modify_srq = ehca_modify_srq;
- shca->ib_device.query_srq = ehca_query_srq;
- shca->ib_device.destroy_srq = ehca_destroy_srq;
- shca->ib_device.post_srq_recv = ehca_post_srq_recv;
- }
-
- return ret;
-}
-
-static int ehca_create_aqp1(struct ehca_shca *shca, u32 port)
-{
- struct ehca_sport *sport = &shca->sport[port - 1];
- struct ib_cq *ibcq;
- struct ib_qp *ibqp;
- struct ib_qp_init_attr qp_init_attr;
- struct ib_cq_init_attr cq_attr = {};
- int ret;
-
- if (sport->ibcq_aqp1) {
- ehca_err(&shca->ib_device, "AQP1 CQ is already created.");
- return -EPERM;
- }
-
- cq_attr.cqe = 10;
- ibcq = ib_create_cq(&shca->ib_device, NULL, NULL, (void *)(-1),
- &cq_attr);
- if (IS_ERR(ibcq)) {
- ehca_err(&shca->ib_device, "Cannot create AQP1 CQ.");
- return PTR_ERR(ibcq);
- }
- sport->ibcq_aqp1 = ibcq;
-
- if (sport->ibqp_sqp[IB_QPT_GSI]) {
- ehca_err(&shca->ib_device, "AQP1 QP is already created.");
- ret = -EPERM;
- goto create_aqp1;
- }
-
- memset(&qp_init_attr, 0, sizeof(struct ib_qp_init_attr));
- qp_init_attr.send_cq = ibcq;
- qp_init_attr.recv_cq = ibcq;
- qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
- qp_init_attr.cap.max_send_wr = 100;
- qp_init_attr.cap.max_recv_wr = 100;
- qp_init_attr.cap.max_send_sge = 2;
- qp_init_attr.cap.max_recv_sge = 1;
- qp_init_attr.qp_type = IB_QPT_GSI;
- qp_init_attr.port_num = port;
- qp_init_attr.qp_context = NULL;
- qp_init_attr.event_handler = NULL;
- qp_init_attr.srq = NULL;
-
- ibqp = ib_create_qp(&shca->pd->ib_pd, &qp_init_attr);
- if (IS_ERR(ibqp)) {
- ehca_err(&shca->ib_device, "Cannot create AQP1 QP.");
- ret = PTR_ERR(ibqp);
- goto create_aqp1;
- }
- sport->ibqp_sqp[IB_QPT_GSI] = ibqp;
-
- return 0;
-
-create_aqp1:
- ib_destroy_cq(sport->ibcq_aqp1);
- return ret;
-}
-
-static int ehca_destroy_aqp1(struct ehca_sport *sport)
-{
- int ret;
-
- ret = ib_destroy_qp(sport->ibqp_sqp[IB_QPT_GSI]);
- if (ret) {
- ehca_gen_err("Cannot destroy AQP1 QP. ret=%i", ret);
- return ret;
- }
-
- ret = ib_destroy_cq(sport->ibcq_aqp1);
- if (ret)
- ehca_gen_err("Cannot destroy AQP1 CQ. ret=%i", ret);
-
- return ret;
-}
-
-static ssize_t ehca_show_debug_level(struct device_driver *ddp, char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "%d\n", ehca_debug_level);
-}
-
-static ssize_t ehca_store_debug_level(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- int value = (*buf) - '0';
- if (value >= 0 && value <= 9)
- ehca_debug_level = value;
- return 1;
-}
-
-static DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
- ehca_show_debug_level, ehca_store_debug_level);
-
-static struct attribute *ehca_drv_attrs[] = {
- &driver_attr_debug_level.attr,
- NULL
-};
-
-static struct attribute_group ehca_drv_attr_grp = {
- .attrs = ehca_drv_attrs
-};
-
-static const struct attribute_group *ehca_drv_attr_groups[] = {
- &ehca_drv_attr_grp,
- NULL,
-};
-
-#define EHCA_RESOURCE_ATTR(name) \
-static ssize_t ehca_show_##name(struct device *dev, \
- struct device_attribute *attr, \
- char *buf) \
-{ \
- struct ehca_shca *shca; \
- struct hipz_query_hca *rblock; \
- int data; \
- \
- shca = dev_get_drvdata(dev); \
- \
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL); \
- if (!rblock) { \
- dev_err(dev, "Can't allocate rblock memory.\n"); \
- return 0; \
- } \
- \
- if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) { \
- dev_err(dev, "Can't query device properties\n"); \
- ehca_free_fw_ctrlblock(rblock); \
- return 0; \
- } \
- \
- data = rblock->name; \
- ehca_free_fw_ctrlblock(rblock); \
- \
- if ((strcmp(#name, "num_ports") == 0) && (ehca_nr_ports == 1)) \
- return snprintf(buf, 256, "1\n"); \
- else \
- return snprintf(buf, 256, "%d\n", data); \
- \
-} \
-static DEVICE_ATTR(name, S_IRUGO, ehca_show_##name, NULL);
-
-EHCA_RESOURCE_ATTR(num_ports);
-EHCA_RESOURCE_ATTR(hw_ver);
-EHCA_RESOURCE_ATTR(max_eq);
-EHCA_RESOURCE_ATTR(cur_eq);
-EHCA_RESOURCE_ATTR(max_cq);
-EHCA_RESOURCE_ATTR(cur_cq);
-EHCA_RESOURCE_ATTR(max_qp);
-EHCA_RESOURCE_ATTR(cur_qp);
-EHCA_RESOURCE_ATTR(max_mr);
-EHCA_RESOURCE_ATTR(cur_mr);
-EHCA_RESOURCE_ATTR(max_mw);
-EHCA_RESOURCE_ATTR(cur_mw);
-EHCA_RESOURCE_ATTR(max_pd);
-EHCA_RESOURCE_ATTR(max_ah);
-
-static ssize_t ehca_show_adapter_handle(struct device *dev,
- struct device_attribute *attr,
- char *buf)
-{
- struct ehca_shca *shca = dev_get_drvdata(dev);
-
- return sprintf(buf, "%llx\n", shca->ipz_hca_handle.handle);
-
-}
-static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
-
-static struct attribute *ehca_dev_attrs[] = {
- &dev_attr_adapter_handle.attr,
- &dev_attr_num_ports.attr,
- &dev_attr_hw_ver.attr,
- &dev_attr_max_eq.attr,
- &dev_attr_cur_eq.attr,
- &dev_attr_max_cq.attr,
- &dev_attr_cur_cq.attr,
- &dev_attr_max_qp.attr,
- &dev_attr_cur_qp.attr,
- &dev_attr_max_mr.attr,
- &dev_attr_cur_mr.attr,
- &dev_attr_max_mw.attr,
- &dev_attr_cur_mw.attr,
- &dev_attr_max_pd.attr,
- &dev_attr_max_ah.attr,
- NULL
-};
-
-static struct attribute_group ehca_dev_attr_grp = {
- .attrs = ehca_dev_attrs
-};
-
-static int ehca_probe(struct platform_device *dev)
-{
- struct ehca_shca *shca;
- const u64 *handle;
- struct ib_pd *ibpd;
- int ret, i, eq_size;
- unsigned long flags;
-
- handle = of_get_property(dev->dev.of_node, "ibm,hca-handle", NULL);
- if (!handle) {
- ehca_gen_err("Cannot get eHCA handle for adapter: %s.",
- dev->dev.of_node->full_name);
- return -ENODEV;
- }
-
- if (!(*handle)) {
- ehca_gen_err("Wrong eHCA handle for adapter: %s.",
- dev->dev.of_node->full_name);
- return -ENODEV;
- }
-
- shca = (struct ehca_shca *)ib_alloc_device(sizeof(*shca));
- if (!shca) {
- ehca_gen_err("Cannot allocate shca memory.");
- return -ENOMEM;
- }
-
- mutex_init(&shca->modify_mutex);
- atomic_set(&shca->num_cqs, 0);
- atomic_set(&shca->num_qps, 0);
- shca->max_num_qps = ehca_max_qp;
- shca->max_num_cqs = ehca_max_cq;
-
- for (i = 0; i < ARRAY_SIZE(shca->sport); i++)
- spin_lock_init(&shca->sport[i].mod_sqp_lock);
-
- shca->ofdev = dev;
- shca->ipz_hca_handle.handle = *handle;
- dev_set_drvdata(&dev->dev, shca);
-
- ret = ehca_sense_attributes(shca);
- if (ret < 0) {
- ehca_gen_err("Cannot sense eHCA attributes.");
- goto probe1;
- }
-
- ret = ehca_init_device(shca);
- if (ret) {
- ehca_gen_err("Cannot init ehca device struct");
- goto probe1;
- }
-
- eq_size = 2 * shca->max_num_cqs + 4 * shca->max_num_qps;
- /* create event queues */
- ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, eq_size);
- if (ret) {
- ehca_err(&shca->ib_device, "Cannot create EQ.");
- goto probe1;
- }
-
- ret = ehca_create_eq(shca, &shca->neq, EHCA_NEQ, 513);
- if (ret) {
- ehca_err(&shca->ib_device, "Cannot create NEQ.");
- goto probe3;
- }
-
- /* create internal protection domain */
- ibpd = ehca_alloc_pd(&shca->ib_device, (void *)(-1), NULL);
- if (IS_ERR(ibpd)) {
- ehca_err(&shca->ib_device, "Cannot create internal PD.");
- ret = PTR_ERR(ibpd);
- goto probe4;
- }
-
- shca->pd = container_of(ibpd, struct ehca_pd, ib_pd);
- shca->pd->ib_pd.device = &shca->ib_device;
-
- /* create internal max MR */
- ret = ehca_reg_internal_maxmr(shca, shca->pd, &shca->maxmr);
-
- if (ret) {
- ehca_err(&shca->ib_device, "Cannot create internal MR ret=%i",
- ret);
- goto probe5;
- }
-
- ret = ib_register_device(&shca->ib_device, NULL);
- if (ret) {
- ehca_err(&shca->ib_device,
- "ib_register_device() failed ret=%i", ret);
- goto probe6;
- }
-
- /* create AQP1 for port 1 */
- if (ehca_open_aqp1 == 1) {
- shca->sport[0].port_state = IB_PORT_DOWN;
- ret = ehca_create_aqp1(shca, 1);
- if (ret) {
- ehca_err(&shca->ib_device,
- "Cannot create AQP1 for port 1.");
- goto probe7;
- }
- }
-
- /* create AQP1 for port 2 */
- if ((ehca_open_aqp1 == 1) && (shca->num_ports == 2)) {
- shca->sport[1].port_state = IB_PORT_DOWN;
- ret = ehca_create_aqp1(shca, 2);
- if (ret) {
- ehca_err(&shca->ib_device,
- "Cannot create AQP1 for port 2.");
- goto probe8;
- }
- }
-
- ret = sysfs_create_group(&dev->dev.kobj, &ehca_dev_attr_grp);
- if (ret) /* only complain; we can live without attributes */
- ehca_err(&shca->ib_device,
- "Cannot create device attributes ret=%d", ret);
-
- spin_lock_irqsave(&shca_list_lock, flags);
- list_add(&shca->shca_list, &shca_list);
- spin_unlock_irqrestore(&shca_list_lock, flags);
-
- return 0;
-
-probe8:
- ret = ehca_destroy_aqp1(&shca->sport[0]);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy AQP1 for port 1. ret=%i", ret);
-
-probe7:
- ib_unregister_device(&shca->ib_device);
-
-probe6:
- ret = ehca_dereg_internal_maxmr(shca);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal MR. ret=%x", ret);
-
-probe5:
- ret = ehca_dealloc_pd(&shca->pd->ib_pd);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal PD. ret=%x", ret);
-
-probe4:
- ret = ehca_destroy_eq(shca, &shca->neq);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy NEQ. ret=%x", ret);
-
-probe3:
- ret = ehca_destroy_eq(shca, &shca->eq);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy EQ. ret=%x", ret);
-
-probe1:
- ib_dealloc_device(&shca->ib_device);
-
- return -EINVAL;
-}
-
-static int ehca_remove(struct platform_device *dev)
-{
- struct ehca_shca *shca = dev_get_drvdata(&dev->dev);
- unsigned long flags;
- int ret;
-
- sysfs_remove_group(&dev->dev.kobj, &ehca_dev_attr_grp);
-
- if (ehca_open_aqp1 == 1) {
- int i;
- for (i = 0; i < shca->num_ports; i++) {
- ret = ehca_destroy_aqp1(&shca->sport[i]);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy AQP1 for port %x "
- "ret=%i", ret, i);
- }
- }
-
- ib_unregister_device(&shca->ib_device);
-
- ret = ehca_dereg_internal_maxmr(shca);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal MR. ret=%i", ret);
-
- ret = ehca_dealloc_pd(&shca->pd->ib_pd);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal PD. ret=%i", ret);
-
- ret = ehca_destroy_eq(shca, &shca->eq);
- if (ret)
- ehca_err(&shca->ib_device, "Cannot destroy EQ. ret=%i", ret);
-
- ret = ehca_destroy_eq(shca, &shca->neq);
- if (ret)
- ehca_err(&shca->ib_device, "Canot destroy NEQ. ret=%i", ret);
-
- ib_dealloc_device(&shca->ib_device);
-
- spin_lock_irqsave(&shca_list_lock, flags);
- list_del(&shca->shca_list);
- spin_unlock_irqrestore(&shca_list_lock, flags);
-
- return ret;
-}
-
-static struct of_device_id ehca_device_table[] =
-{
- {
- .name = "lhca",
- .compatible = "IBM,lhca",
- },
- {},
-};
-MODULE_DEVICE_TABLE(of, ehca_device_table);
-
-static struct platform_driver ehca_driver = {
- .probe = ehca_probe,
- .remove = ehca_remove,
- .driver = {
- .name = "ehca",
- .owner = THIS_MODULE,
- .groups = ehca_drv_attr_groups,
- .of_match_table = ehca_device_table,
- },
-};
-
-void ehca_poll_eqs(unsigned long data)
-{
- struct ehca_shca *shca;
-
- spin_lock(&shca_list_lock);
- list_for_each_entry(shca, &shca_list, shca_list) {
- if (shca->eq.is_initialized) {
- /* call deadman proc only if eq ptr does not change */
- struct ehca_eq *eq = &shca->eq;
- int max = 3;
- volatile u64 q_ofs, q_ofs2;
- unsigned long flags;
- spin_lock_irqsave(&eq->spinlock, flags);
- q_ofs = eq->ipz_queue.current_q_offset;
- spin_unlock_irqrestore(&eq->spinlock, flags);
- do {
- spin_lock_irqsave(&eq->spinlock, flags);
- q_ofs2 = eq->ipz_queue.current_q_offset;
- spin_unlock_irqrestore(&eq->spinlock, flags);
- max--;
- } while (q_ofs == q_ofs2 && max > 0);
- if (q_ofs == q_ofs2)
- ehca_process_eq(shca, 0);
- }
- }
- mod_timer(&poll_eqs_timer, round_jiffies(jiffies + HZ));
- spin_unlock(&shca_list_lock);
-}
-
-static int ehca_mem_notifier(struct notifier_block *nb,
- unsigned long action, void *data)
-{
- static unsigned long ehca_dmem_warn_time;
- unsigned long flags;
-
- switch (action) {
- case MEM_CANCEL_OFFLINE:
- case MEM_CANCEL_ONLINE:
- case MEM_ONLINE:
- case MEM_OFFLINE:
- return NOTIFY_OK;
- case MEM_GOING_ONLINE:
- case MEM_GOING_OFFLINE:
- /* only ok if no hca is attached to the lpar */
- spin_lock_irqsave(&shca_list_lock, flags);
- if (list_empty(&shca_list)) {
- spin_unlock_irqrestore(&shca_list_lock, flags);
- return NOTIFY_OK;
- } else {
- spin_unlock_irqrestore(&shca_list_lock, flags);
- if (printk_timed_ratelimit(&ehca_dmem_warn_time,
- 30 * 1000))
- ehca_gen_err("DMEM operations are not allowed"
- "in conjunction with eHCA");
- return NOTIFY_BAD;
- }
- }
- return NOTIFY_OK;
-}
-
-static struct notifier_block ehca_mem_nb = {
- .notifier_call = ehca_mem_notifier,
-};
-
-static int __init ehca_module_init(void)
-{
- int ret;
-
- printk(KERN_INFO "eHCA Infiniband Device Driver "
- "(Version " HCAD_VERSION ")\n");
-
- ret = ehca_create_comp_pool();
- if (ret) {
- ehca_gen_err("Cannot create comp pool.");
- return ret;
- }
-
- ret = ehca_create_slab_caches();
- if (ret) {
- ehca_gen_err("Cannot create SLAB caches");
- ret = -ENOMEM;
- goto module_init1;
- }
-
- ret = ehca_create_busmap();
- if (ret) {
- ehca_gen_err("Cannot create busmap.");
- goto module_init2;
- }
-
- ret = ibmebus_register_driver(&ehca_driver);
- if (ret) {
- ehca_gen_err("Cannot register eHCA device driver");
- ret = -EINVAL;
- goto module_init3;
- }
-
- ret = register_memory_notifier(&ehca_mem_nb);
- if (ret) {
- ehca_gen_err("Failed registering memory add/remove notifier");
- goto module_init4;
- }
-
- if (ehca_poll_all_eqs != 1) {
- ehca_gen_err("WARNING!!!");
- ehca_gen_err("It is possible to lose interrupts.");
- } else {
- init_timer(&poll_eqs_timer);
- poll_eqs_timer.function = ehca_poll_eqs;
- poll_eqs_timer.expires = jiffies + HZ;
- add_timer(&poll_eqs_timer);
- }
-
- return 0;
-
-module_init4:
- ibmebus_unregister_driver(&ehca_driver);
-
-module_init3:
- ehca_destroy_busmap();
-
-module_init2:
- ehca_destroy_slab_caches();
-
-module_init1:
- ehca_destroy_comp_pool();
- return ret;
-};
-
-static void __exit ehca_module_exit(void)
-{
- if (ehca_poll_all_eqs == 1)
- del_timer_sync(&poll_eqs_timer);
-
- ibmebus_unregister_driver(&ehca_driver);
-
- unregister_memory_notifier(&ehca_mem_nb);
-
- ehca_destroy_busmap();
-
- ehca_destroy_slab_caches();
-
- ehca_destroy_comp_pool();
-
- idr_destroy(&ehca_cq_idr);
- idr_destroy(&ehca_qp_idr);
-};
-
-module_init(ehca_module_init);
-module_exit(ehca_module_exit);
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * mcast functions
- *
- * Authors: Khadija Souissi <souissik@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/module.h>
-#include <linux/err.h>
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-#define MAX_MC_LID 0xFFFE
-#define MIN_MC_LID 0xC000 /* Multicast limits */
-#define EHCA_VALID_MULTICAST_GID(gid) ((gid)[0] == 0xFF)
-#define EHCA_VALID_MULTICAST_LID(lid) \
- (((lid) >= MIN_MC_LID) && ((lid) <= MAX_MC_LID))
-
-int ehca_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
-{
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
- ib_device);
- union ib_gid my_gid;
- u64 subnet_prefix, interface_id, h_ret;
-
- if (ibqp->qp_type != IB_QPT_UD) {
- ehca_err(ibqp->device, "invalid qp_type=%x", ibqp->qp_type);
- return -EINVAL;
- }
-
- if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
- ehca_err(ibqp->device, "invalid mulitcast gid");
- return -EINVAL;
- } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
- ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
- return -EINVAL;
- }
-
- memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
-
- subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
- interface_id = be64_to_cpu(my_gid.global.interface_id);
- h_ret = hipz_h_attach_mcqp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- my_qp->galpas.kernel,
- lid, subnet_prefix, interface_id);
- if (h_ret != H_SUCCESS)
- ehca_err(ibqp->device,
- "ehca_qp=%p qp_num=%x hipz_h_attach_mcqp() failed "
- "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
-
- return ehca2ib_return_code(h_ret);
-}
-
-int ehca_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
-{
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca = container_of(ibqp->pd->device,
- struct ehca_shca, ib_device);
- union ib_gid my_gid;
- u64 subnet_prefix, interface_id, h_ret;
-
- if (ibqp->qp_type != IB_QPT_UD) {
- ehca_err(ibqp->device, "invalid qp_type %x", ibqp->qp_type);
- return -EINVAL;
- }
-
- if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
- ehca_err(ibqp->device, "invalid mulitcast gid");
- return -EINVAL;
- } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
- ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
- return -EINVAL;
- }
-
- memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
-
- subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
- interface_id = be64_to_cpu(my_gid.global.interface_id);
- h_ret = hipz_h_detach_mcqp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- my_qp->galpas.kernel,
- lid, subnet_prefix, interface_id);
- if (h_ret != H_SUCCESS)
- ehca_err(ibqp->device,
- "ehca_qp=%p qp_num=%x hipz_h_detach_mcqp() failed "
- "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
-
- return ehca2ib_return_code(h_ret);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * MR/MW functions
- *
- * Authors: Dietmar Decker <ddecker@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-#include <rdma/ib_umem.h>
-
-#include "ehca_iverbs.h"
-#include "ehca_mrmw.h"
-#include "hcp_if.h"
-#include "hipz_hw.h"
-
-#define NUM_CHUNKS(length, chunk_size) \
- (((length) + (chunk_size - 1)) / (chunk_size))
-
-/* max number of rpages (per hcall register_rpages) */
-#define MAX_RPAGES 512
-
-/* DMEM toleration management */
-#define EHCA_SECTSHIFT SECTION_SIZE_BITS
-#define EHCA_SECTSIZE (1UL << EHCA_SECTSHIFT)
-#define EHCA_HUGEPAGESHIFT 34
-#define EHCA_HUGEPAGE_SIZE (1UL << EHCA_HUGEPAGESHIFT)
-#define EHCA_HUGEPAGE_PFN_MASK ((EHCA_HUGEPAGE_SIZE - 1) >> PAGE_SHIFT)
-#define EHCA_INVAL_ADDR 0xFFFFFFFFFFFFFFFFULL
-#define EHCA_DIR_INDEX_SHIFT 13 /* 8k Entries in 64k block */
-#define EHCA_TOP_INDEX_SHIFT (EHCA_DIR_INDEX_SHIFT * 2)
-#define EHCA_MAP_ENTRIES (1 << EHCA_DIR_INDEX_SHIFT)
-#define EHCA_TOP_MAP_SIZE (0x10000) /* currently fixed map size */
-#define EHCA_DIR_MAP_SIZE (0x10000)
-#define EHCA_ENT_MAP_SIZE (0x10000)
-#define EHCA_INDEX_MASK (EHCA_MAP_ENTRIES - 1)
-
-static unsigned long ehca_mr_len;
-
-/*
- * Memory map data structures
- */
-struct ehca_dir_bmap {
- u64 ent[EHCA_MAP_ENTRIES];
-};
-struct ehca_top_bmap {
- struct ehca_dir_bmap *dir[EHCA_MAP_ENTRIES];
-};
-struct ehca_bmap {
- struct ehca_top_bmap *top[EHCA_MAP_ENTRIES];
-};
-
-static struct ehca_bmap *ehca_bmap;
-
-static struct kmem_cache *mr_cache;
-static struct kmem_cache *mw_cache;
-
-enum ehca_mr_pgsize {
- EHCA_MR_PGSIZE4K = 0x1000L,
- EHCA_MR_PGSIZE64K = 0x10000L,
- EHCA_MR_PGSIZE1M = 0x100000L,
- EHCA_MR_PGSIZE16M = 0x1000000L
-};
-
-#define EHCA_MR_PGSHIFT4K 12
-#define EHCA_MR_PGSHIFT64K 16
-#define EHCA_MR_PGSHIFT1M 20
-#define EHCA_MR_PGSHIFT16M 24
-
-static u64 ehca_map_vaddr(void *caddr);
-
-static u32 ehca_encode_hwpage_size(u32 pgsize)
-{
- int log = ilog2(pgsize);
- WARN_ON(log < 12 || log > 24 || log & 3);
- return (log - 12) / 4;
-}
-
-static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca)
-{
- return rounddown_pow_of_two(shca->hca_cap_mr_pgsize);
-}
-
-static struct ehca_mr *ehca_mr_new(void)
-{
- struct ehca_mr *me;
-
- me = kmem_cache_zalloc(mr_cache, GFP_KERNEL);
- if (me)
- spin_lock_init(&me->mrlock);
- else
- ehca_gen_err("alloc failed");
-
- return me;
-}
-
-static void ehca_mr_delete(struct ehca_mr *me)
-{
- kmem_cache_free(mr_cache, me);
-}
-
-static struct ehca_mw *ehca_mw_new(void)
-{
- struct ehca_mw *me;
-
- me = kmem_cache_zalloc(mw_cache, GFP_KERNEL);
- if (me)
- spin_lock_init(&me->mwlock);
- else
- ehca_gen_err("alloc failed");
-
- return me;
-}
-
-static void ehca_mw_delete(struct ehca_mw *me)
-{
- kmem_cache_free(mw_cache, me);
-}
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
-{
- struct ib_mr *ib_mr;
- int ret;
- struct ehca_mr *e_maxmr;
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
-
- if (shca->maxmr) {
- e_maxmr = ehca_mr_new();
- if (!e_maxmr) {
- ehca_err(&shca->ib_device, "out of memory");
- ib_mr = ERR_PTR(-ENOMEM);
- goto get_dma_mr_exit0;
- }
-
- ret = ehca_reg_maxmr(shca, e_maxmr,
- (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)),
- mr_access_flags, e_pd,
- &e_maxmr->ib.ib_mr.lkey,
- &e_maxmr->ib.ib_mr.rkey);
- if (ret) {
- ehca_mr_delete(e_maxmr);
- ib_mr = ERR_PTR(ret);
- goto get_dma_mr_exit0;
- }
- ib_mr = &e_maxmr->ib.ib_mr;
- } else {
- ehca_err(&shca->ib_device, "no internal max-MR exist!");
- ib_mr = ERR_PTR(-EINVAL);
- goto get_dma_mr_exit0;
- }
-
-get_dma_mr_exit0:
- if (IS_ERR(ib_mr))
- ehca_err(&shca->ib_device, "h_ret=%li pd=%p mr_access_flags=%x",
- PTR_ERR(ib_mr), pd, mr_access_flags);
- return ib_mr;
-} /* end ehca_get_dma_mr() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- int mr_access_flags,
- u64 *iova_start)
-{
- struct ib_mr *ib_mr;
- int ret;
- struct ehca_mr *e_mr;
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
-
- u64 size;
-
- if ((num_phys_buf <= 0) || !phys_buf_array) {
- ehca_err(pd->device, "bad input values: num_phys_buf=%x "
- "phys_buf_array=%p", num_phys_buf, phys_buf_array);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_phys_mr_exit0;
- }
- if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_phys_mr_exit0;
- }
-
- /* check physical buffer list and calculate size */
- ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
- iova_start, &size);
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_phys_mr_exit0;
- }
- if ((size == 0) ||
- (((u64)iova_start + size) < (u64)iova_start)) {
- ehca_err(pd->device, "bad input values: size=%llx iova_start=%p",
- size, iova_start);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_phys_mr_exit0;
- }
-
- e_mr = ehca_mr_new();
- if (!e_mr) {
- ehca_err(pd->device, "out of memory");
- ib_mr = ERR_PTR(-ENOMEM);
- goto reg_phys_mr_exit0;
- }
-
- /* register MR on HCA */
- if (ehca_mr_is_maxmr(size, iova_start)) {
- e_mr->flags |= EHCA_MR_FLAG_MAXMR;
- ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
- e_pd, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey);
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_phys_mr_exit1;
- }
- } else {
- struct ehca_mr_pginfo pginfo;
- u32 num_kpages;
- u32 num_hwpages;
- u64 hw_pgsize;
-
- num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
- PAGE_SIZE);
- /* for kernel space we try most possible pgsize */
- hw_pgsize = ehca_get_max_hwpage_size(shca);
- num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size,
- hw_pgsize);
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_PHYS;
- pginfo.num_kpages = num_kpages;
- pginfo.hwpage_size = hw_pgsize;
- pginfo.num_hwpages = num_hwpages;
- pginfo.u.phy.num_phys_buf = num_phys_buf;
- pginfo.u.phy.phys_buf_array = phys_buf_array;
- pginfo.next_hwpage =
- ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
-
- ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
- e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_phys_mr_exit1;
- }
- }
-
- /* successful registration of all pages */
- return &e_mr->ib.ib_mr;
-
-reg_phys_mr_exit1:
- ehca_mr_delete(e_mr);
-reg_phys_mr_exit0:
- if (IS_ERR(ib_mr))
- ehca_err(pd->device, "h_ret=%li pd=%p phys_buf_array=%p "
- "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
- PTR_ERR(ib_mr), pd, phys_buf_array,
- num_phys_buf, mr_access_flags, iova_start);
- return ib_mr;
-} /* end ehca_reg_phys_mr() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
- u64 virt, int mr_access_flags,
- struct ib_udata *udata)
-{
- struct ib_mr *ib_mr;
- struct ehca_mr *e_mr;
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_mr_pginfo pginfo;
- int ret, page_shift;
- u32 num_kpages;
- u32 num_hwpages;
- u64 hwpage_size;
-
- if (!pd) {
- ehca_gen_err("bad pd=%p", pd);
- return ERR_PTR(-EFAULT);
- }
-
- if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_user_mr_exit0;
- }
-
- if (length == 0 || virt + length < virt) {
- ehca_err(pd->device, "bad input values: length=%llx "
- "virt_base=%llx", length, virt);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_user_mr_exit0;
- }
-
- e_mr = ehca_mr_new();
- if (!e_mr) {
- ehca_err(pd->device, "out of memory");
- ib_mr = ERR_PTR(-ENOMEM);
- goto reg_user_mr_exit0;
- }
-
- e_mr->umem = ib_umem_get(pd->uobject->context, start, length,
- mr_access_flags, 0);
- if (IS_ERR(e_mr->umem)) {
- ib_mr = (void *)e_mr->umem;
- goto reg_user_mr_exit1;
- }
-
- if (e_mr->umem->page_size != PAGE_SIZE) {
- ehca_err(pd->device, "page size not supported, "
- "e_mr->umem->page_size=%x", e_mr->umem->page_size);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_user_mr_exit2;
- }
-
- /* determine number of MR pages */
- num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
- /* select proper hw_pgsize */
- page_shift = PAGE_SHIFT;
- if (e_mr->umem->hugetlb) {
- /* determine page_shift, clamp between 4K and 16M */
- page_shift = (fls64(length - 1) + 3) & ~3;
- page_shift = min(max(page_shift, EHCA_MR_PGSHIFT4K),
- EHCA_MR_PGSHIFT16M);
- }
- hwpage_size = 1UL << page_shift;
-
- /* now that we have the desired page size, shift until it's
- * supported, too. 4K is always supported, so this terminates.
- */
- while (!(hwpage_size & shca->hca_cap_mr_pgsize))
- hwpage_size >>= 4;
-
-reg_user_mr_fallback:
- num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size);
- /* register MR on HCA */
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_USER;
- pginfo.hwpage_size = hwpage_size;
- pginfo.num_kpages = num_kpages;
- pginfo.num_hwpages = num_hwpages;
- pginfo.u.usr.region = e_mr->umem;
- pginfo.next_hwpage = ib_umem_offset(e_mr->umem) / hwpage_size;
- pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl;
- ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
- e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
- if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) {
- ehca_warn(pd->device, "failed to register mr "
- "with hwpage_size=%llx", hwpage_size);
- ehca_info(pd->device, "try to register mr with "
- "kpage_size=%lx", PAGE_SIZE);
- /*
- * this means kpages are not contiguous for a hw page
- * try kernel page size as fallback solution
- */
- hwpage_size = PAGE_SIZE;
- goto reg_user_mr_fallback;
- }
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_user_mr_exit2;
- }
-
- /* successful registration of all pages */
- return &e_mr->ib.ib_mr;
-
-reg_user_mr_exit2:
- ib_umem_release(e_mr->umem);
-reg_user_mr_exit1:
- ehca_mr_delete(e_mr);
-reg_user_mr_exit0:
- if (IS_ERR(ib_mr))
- ehca_err(pd->device, "rc=%li pd=%p mr_access_flags=%x udata=%p",
- PTR_ERR(ib_mr), pd, mr_access_flags, udata);
- return ib_mr;
-} /* end ehca_reg_user_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_rereg_phys_mr(struct ib_mr *mr,
- int mr_rereg_mask,
- struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- int mr_access_flags,
- u64 *iova_start)
-{
- int ret;
-
- struct ehca_shca *shca =
- container_of(mr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
- u64 new_size;
- u64 *new_start;
- u32 new_acl;
- struct ehca_pd *new_pd;
- u32 tmp_lkey, tmp_rkey;
- unsigned long sl_flags;
- u32 num_kpages = 0;
- u32 num_hwpages = 0;
- struct ehca_mr_pginfo pginfo;
-
- if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
- /* TODO not supported, because PHYP rereg hCall needs pages */
- ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
- "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
-
- if (mr_rereg_mask & IB_MR_REREG_PD) {
- if (!pd) {
- ehca_err(mr->device, "rereg with bad pd, pd=%p "
- "mr_rereg_mask=%x", pd, mr_rereg_mask);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- }
-
- if ((mr_rereg_mask &
- ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
- (mr_rereg_mask == 0)) {
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
-
- /* check other parameters */
- if (e_mr == shca->maxmr) {
- /* should be impossible, however reject to be sure */
- ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
- "shca->maxmr=%p mr->lkey=%x",
- mr, shca->maxmr, mr->lkey);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
- if (e_mr->flags & EHCA_MR_FLAG_FMR) {
- ehca_err(mr->device, "not supported for FMR, mr=%p "
- "flags=%x", mr, e_mr->flags);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- if (!phys_buf_array || num_phys_buf <= 0) {
- ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
- " phys_buf_array=%p num_phys_buf=%x",
- mr_rereg_mask, phys_buf_array, num_phys_buf);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- }
- if ((mr_rereg_mask & IB_MR_REREG_ACCESS) && /* change ACL */
- (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
- "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
-
- /* set requested values dependent on rereg request */
- spin_lock_irqsave(&e_mr->mrlock, sl_flags);
- new_start = e_mr->start;
- new_size = e_mr->size;
- new_acl = e_mr->acl;
- new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
-
- if (mr_rereg_mask & IB_MR_REREG_TRANS) {
- u64 hw_pgsize = ehca_get_max_hwpage_size(shca);
-
- new_start = iova_start; /* change address */
- /* check physical buffer list and calculate size */
- ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
- num_phys_buf, iova_start,
- &new_size);
- if (ret)
- goto rereg_phys_mr_exit1;
- if ((new_size == 0) ||
- (((u64)iova_start + new_size) < (u64)iova_start)) {
- ehca_err(mr->device, "bad input values: new_size=%llx "
- "iova_start=%p", new_size, iova_start);
- ret = -EINVAL;
- goto rereg_phys_mr_exit1;
- }
- num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
- new_size, PAGE_SIZE);
- num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) +
- new_size, hw_pgsize);
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_PHYS;
- pginfo.num_kpages = num_kpages;
- pginfo.hwpage_size = hw_pgsize;
- pginfo.num_hwpages = num_hwpages;
- pginfo.u.phy.num_phys_buf = num_phys_buf;
- pginfo.u.phy.phys_buf_array = phys_buf_array;
- pginfo.next_hwpage =
- ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
- }
- if (mr_rereg_mask & IB_MR_REREG_ACCESS)
- new_acl = mr_access_flags;
- if (mr_rereg_mask & IB_MR_REREG_PD)
- new_pd = container_of(pd, struct ehca_pd, ib_pd);
-
- ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
- new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
- if (ret)
- goto rereg_phys_mr_exit1;
-
- /* successful reregistration */
- if (mr_rereg_mask & IB_MR_REREG_PD)
- mr->pd = pd;
- mr->lkey = tmp_lkey;
- mr->rkey = tmp_rkey;
-
-rereg_phys_mr_exit1:
- spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
-rereg_phys_mr_exit0:
- if (ret)
- ehca_err(mr->device, "ret=%i mr=%p mr_rereg_mask=%x pd=%p "
- "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
- "iova_start=%p",
- ret, mr, mr_rereg_mask, pd, phys_buf_array,
- num_phys_buf, mr_access_flags, iova_start);
- return ret;
-} /* end ehca_rereg_phys_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(mr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
- unsigned long sl_flags;
- struct ehca_mr_hipzout_parms hipzout;
-
- if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
- "e_mr->flags=%x", mr, e_mr, e_mr->flags);
- ret = -EINVAL;
- goto query_mr_exit0;
- }
-
- memset(mr_attr, 0, sizeof(struct ib_mr_attr));
- spin_lock_irqsave(&e_mr->mrlock, sl_flags);
-
- h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lli mr=%p "
- "hca_hndl=%llx mr_hndl=%llx lkey=%x",
- h_ret, mr, shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle, mr->lkey);
- ret = ehca2ib_return_code(h_ret);
- goto query_mr_exit1;
- }
- mr_attr->pd = mr->pd;
- mr_attr->device_virt_addr = hipzout.vaddr;
- mr_attr->size = hipzout.len;
- mr_attr->lkey = hipzout.lkey;
- mr_attr->rkey = hipzout.rkey;
- ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
-
-query_mr_exit1:
- spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
-query_mr_exit0:
- if (ret)
- ehca_err(mr->device, "ret=%i mr=%p mr_attr=%p",
- ret, mr, mr_attr);
- return ret;
-} /* end ehca_query_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dereg_mr(struct ib_mr *mr)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(mr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
-
- if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
- "e_mr->flags=%x", mr, e_mr, e_mr->flags);
- ret = -EINVAL;
- goto dereg_mr_exit0;
- } else if (e_mr == shca->maxmr) {
- /* should be impossible, however reject to be sure */
- ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
- "shca->maxmr=%p mr->lkey=%x",
- mr, shca->maxmr, mr->lkey);
- ret = -EINVAL;
- goto dereg_mr_exit0;
- }
-
- /* TODO: BUSY: MR still has bound window(s) */
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
- if (h_ret != H_SUCCESS) {
- ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lli shca=%p "
- "e_mr=%p hca_hndl=%llx mr_hndl=%llx mr->lkey=%x",
- h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle, mr->lkey);
- ret = ehca2ib_return_code(h_ret);
- goto dereg_mr_exit0;
- }
-
- if (e_mr->umem)
- ib_umem_release(e_mr->umem);
-
- /* successful deregistration */
- ehca_mr_delete(e_mr);
-
-dereg_mr_exit0:
- if (ret)
- ehca_err(mr->device, "ret=%i mr=%p", ret, mr);
- return ret;
-} /* end ehca_dereg_mr() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
-{
- struct ib_mw *ib_mw;
- u64 h_ret;
- struct ehca_mw *e_mw;
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_mw_hipzout_parms hipzout;
-
- if (type != IB_MW_TYPE_1)
- return ERR_PTR(-EINVAL);
-
- e_mw = ehca_mw_new();
- if (!e_mw) {
- ib_mw = ERR_PTR(-ENOMEM);
- goto alloc_mw_exit0;
- }
-
- h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
- e_pd->fw_pd, &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lli "
- "shca=%p hca_hndl=%llx mw=%p",
- h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
- ib_mw = ERR_PTR(ehca2ib_return_code(h_ret));
- goto alloc_mw_exit1;
- }
- /* successful MW allocation */
- e_mw->ipz_mw_handle = hipzout.handle;
- e_mw->ib_mw.rkey = hipzout.rkey;
- return &e_mw->ib_mw;
-
-alloc_mw_exit1:
- ehca_mw_delete(e_mw);
-alloc_mw_exit0:
- if (IS_ERR(ib_mw))
- ehca_err(pd->device, "h_ret=%li pd=%p", PTR_ERR(ib_mw), pd);
- return ib_mw;
-} /* end ehca_alloc_mw() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_bind_mw(struct ib_qp *qp,
- struct ib_mw *mw,
- struct ib_mw_bind *mw_bind)
-{
- /* TODO: not supported up to now */
- ehca_gen_err("bind MW currently not supported by HCAD");
-
- return -EPERM;
-} /* end ehca_bind_mw() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dealloc_mw(struct ib_mw *mw)
-{
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(mw->device, struct ehca_shca, ib_device);
- struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
-
- h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
- if (h_ret != H_SUCCESS) {
- ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lli shca=%p "
- "mw=%p rkey=%x hca_hndl=%llx mw_hndl=%llx",
- h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
- e_mw->ipz_mw_handle.handle);
- return ehca2ib_return_code(h_ret);
- }
- /* successful deallocation */
- ehca_mw_delete(e_mw);
- return 0;
-} /* end ehca_dealloc_mw() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
- int mr_access_flags,
- struct ib_fmr_attr *fmr_attr)
-{
- struct ib_fmr *ib_fmr;
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_mr *e_fmr;
- int ret;
- u32 tmp_lkey, tmp_rkey;
- struct ehca_mr_pginfo pginfo;
- u64 hw_pgsize;
-
- /* check other parameters */
- if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
- if (mr_access_flags & IB_ACCESS_MW_BIND) {
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
- if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
- ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
- "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
- fmr_attr->max_pages, fmr_attr->max_maps,
- fmr_attr->page_shift);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
-
- hw_pgsize = 1 << fmr_attr->page_shift;
- if (!(hw_pgsize & shca->hca_cap_mr_pgsize)) {
- ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
- fmr_attr->page_shift);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
-
- e_fmr = ehca_mr_new();
- if (!e_fmr) {
- ib_fmr = ERR_PTR(-ENOMEM);
- goto alloc_fmr_exit0;
- }
- e_fmr->flags |= EHCA_MR_FLAG_FMR;
-
- /* register MR on HCA */
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.hwpage_size = hw_pgsize;
- /*
- * pginfo.num_hwpages==0, ie register_rpages() will not be called
- * but deferred to map_phys_fmr()
- */
- ret = ehca_reg_mr(shca, e_fmr, NULL,
- fmr_attr->max_pages * (1 << fmr_attr->page_shift),
- mr_access_flags, e_pd, &pginfo,
- &tmp_lkey, &tmp_rkey, EHCA_REG_MR);
- if (ret) {
- ib_fmr = ERR_PTR(ret);
- goto alloc_fmr_exit1;
- }
-
- /* successful */
- e_fmr->hwpage_size = hw_pgsize;
- e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
- e_fmr->fmr_max_pages = fmr_attr->max_pages;
- e_fmr->fmr_max_maps = fmr_attr->max_maps;
- e_fmr->fmr_map_cnt = 0;
- return &e_fmr->ib.ib_fmr;
-
-alloc_fmr_exit1:
- ehca_mr_delete(e_fmr);
-alloc_fmr_exit0:
- return ib_fmr;
-} /* end ehca_alloc_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_map_phys_fmr(struct ib_fmr *fmr,
- u64 *page_list,
- int list_len,
- u64 iova)
-{
- int ret;
- struct ehca_shca *shca =
- container_of(fmr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
- struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
- struct ehca_mr_pginfo pginfo;
- u32 tmp_lkey, tmp_rkey;
-
- if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
- e_fmr, e_fmr->flags);
- ret = -EINVAL;
- goto map_phys_fmr_exit0;
- }
- ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
- if (ret)
- goto map_phys_fmr_exit0;
- if (iova % e_fmr->fmr_page_size) {
- /* only whole-numbered pages */
- ehca_err(fmr->device, "bad iova, iova=%llx fmr_page_size=%x",
- iova, e_fmr->fmr_page_size);
- ret = -EINVAL;
- goto map_phys_fmr_exit0;
- }
- if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
- /* HCAD does not limit the maps, however trace this anyway */
- ehca_info(fmr->device, "map limit exceeded, fmr=%p "
- "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
- fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
- }
-
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_FMR;
- pginfo.num_kpages = list_len;
- pginfo.hwpage_size = e_fmr->hwpage_size;
- pginfo.num_hwpages =
- list_len * e_fmr->fmr_page_size / pginfo.hwpage_size;
- pginfo.u.fmr.page_list = page_list;
- pginfo.next_hwpage =
- (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size;
- pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
-
- ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
- list_len * e_fmr->fmr_page_size,
- e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
- if (ret)
- goto map_phys_fmr_exit0;
-
- /* successful reregistration */
- e_fmr->fmr_map_cnt++;
- e_fmr->ib.ib_fmr.lkey = tmp_lkey;
- e_fmr->ib.ib_fmr.rkey = tmp_rkey;
- return 0;
-
-map_phys_fmr_exit0:
- if (ret)
- ehca_err(fmr->device, "ret=%i fmr=%p page_list=%p list_len=%x "
- "iova=%llx", ret, fmr, page_list, list_len, iova);
- return ret;
-} /* end ehca_map_phys_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_unmap_fmr(struct list_head *fmr_list)
-{
- int ret = 0;
- struct ib_fmr *ib_fmr;
- struct ehca_shca *shca = NULL;
- struct ehca_shca *prev_shca;
- struct ehca_mr *e_fmr;
- u32 num_fmr = 0;
- u32 unmap_fmr_cnt = 0;
-
- /* check all FMR belong to same SHCA, and check internal flag */
- list_for_each_entry(ib_fmr, fmr_list, list) {
- prev_shca = shca;
- shca = container_of(ib_fmr->device, struct ehca_shca,
- ib_device);
- e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
- if ((shca != prev_shca) && prev_shca) {
- ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
- "prev_shca=%p e_fmr=%p",
- shca, prev_shca, e_fmr);
- ret = -EINVAL;
- goto unmap_fmr_exit0;
- }
- if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
- "e_fmr->flags=%x", e_fmr, e_fmr->flags);
- ret = -EINVAL;
- goto unmap_fmr_exit0;
- }
- num_fmr++;
- }
-
- /* loop over all FMRs to unmap */
- list_for_each_entry(ib_fmr, fmr_list, list) {
- unmap_fmr_cnt++;
- e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
- shca = container_of(ib_fmr->device, struct ehca_shca,
- ib_device);
- ret = ehca_unmap_one_fmr(shca, e_fmr);
- if (ret) {
- /* unmap failed, stop unmapping of rest of FMRs */
- ehca_err(&shca->ib_device, "unmap of one FMR failed, "
- "stop rest, e_fmr=%p num_fmr=%x "
- "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
- unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
- goto unmap_fmr_exit0;
- }
- }
-
-unmap_fmr_exit0:
- if (ret)
- ehca_gen_err("ret=%i fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
- ret, fmr_list, num_fmr, unmap_fmr_cnt);
- return ret;
-} /* end ehca_unmap_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dealloc_fmr(struct ib_fmr *fmr)
-{
- int ret;
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(fmr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
-
- if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
- e_fmr, e_fmr->flags);
- ret = -EINVAL;
- goto free_fmr_exit0;
- }
-
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
- if (h_ret != H_SUCCESS) {
- ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lli e_fmr=%p "
- "hca_hndl=%llx fmr_hndl=%llx fmr->lkey=%x",
- h_ret, e_fmr, shca->ipz_hca_handle.handle,
- e_fmr->ipz_mr_handle.handle, fmr->lkey);
- ret = ehca2ib_return_code(h_ret);
- goto free_fmr_exit0;
- }
- /* successful deregistration */
- ehca_mr_delete(e_fmr);
- return 0;
-
-free_fmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i fmr=%p", ret, fmr);
- return ret;
-} /* end ehca_dealloc_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo);
-
-int ehca_reg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey, /*OUT*/
- u32 *rkey, /*OUT*/
- enum ehca_reg_type reg_type)
-{
- int ret;
- u64 h_ret;
- u32 hipz_acl;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
- if (ehca_use_hp_mr == 1)
- hipz_acl |= 0x00000001;
-
- h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
- (u64)iova_start, size, hipz_acl,
- e_pd->fw_pd, &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lli "
- "hca_hndl=%llx", h_ret, shca->ipz_hca_handle.handle);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_reg_mr_exit0;
- }
-
- e_mr->ipz_mr_handle = hipzout.handle;
-
- if (reg_type == EHCA_REG_BUSMAP_MR)
- ret = ehca_reg_bmap_mr_rpages(shca, e_mr, pginfo);
- else if (reg_type == EHCA_REG_MR)
- ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
- else
- ret = -EINVAL;
-
- if (ret)
- goto ehca_reg_mr_exit1;
-
- /* successful registration */
- e_mr->num_kpages = pginfo->num_kpages;
- e_mr->num_hwpages = pginfo->num_hwpages;
- e_mr->hwpage_size = pginfo->hwpage_size;
- e_mr->start = iova_start;
- e_mr->size = size;
- e_mr->acl = acl;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- return 0;
-
-ehca_reg_mr_exit1:
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "h_ret=%lli shca=%p e_mr=%p "
- "iova_start=%p size=%llx acl=%x e_pd=%p lkey=%x "
- "pginfo=%p num_kpages=%llx num_hwpages=%llx ret=%i",
- h_ret, shca, e_mr, iova_start, size, acl, e_pd,
- hipzout.lkey, pginfo, pginfo->num_kpages,
- pginfo->num_hwpages, ret);
- ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
- "not recoverable");
- }
-ehca_reg_mr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
- "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
- "num_kpages=%llx num_hwpages=%llx",
- ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
- pginfo->num_kpages, pginfo->num_hwpages);
- return ret;
-} /* end ehca_reg_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_reg_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo)
-{
- int ret = 0;
- u64 h_ret;
- u32 rnum;
- u64 rpage;
- u32 i;
- u64 *kpage;
-
- if (!pginfo->num_hwpages) /* in case of fmr */
- return 0;
-
- kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!kpage) {
- ehca_err(&shca->ib_device, "kpage alloc failed");
- ret = -ENOMEM;
- goto ehca_reg_mr_rpages_exit0;
- }
-
- /* max MAX_RPAGES ehca mr pages per register call */
- for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
-
- if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
- rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */
- if (rnum == 0)
- rnum = MAX_RPAGES; /* last shot is full */
- } else
- rnum = MAX_RPAGES;
-
- ret = ehca_set_pagebuf(pginfo, rnum, kpage);
- if (ret) {
- ehca_err(&shca->ib_device, "ehca_set_pagebuf "
- "bad rc, ret=%i rnum=%x kpage=%p",
- ret, rnum, kpage);
- goto ehca_reg_mr_rpages_exit1;
- }
-
- if (rnum > 1) {
- rpage = __pa(kpage);
- if (!rpage) {
- ehca_err(&shca->ib_device, "kpage=%p i=%x",
- kpage, i);
- ret = -EFAULT;
- goto ehca_reg_mr_rpages_exit1;
- }
- } else
- rpage = *kpage;
-
- h_ret = hipz_h_register_rpage_mr(
- shca->ipz_hca_handle, e_mr,
- ehca_encode_hwpage_size(pginfo->hwpage_size),
- 0, rpage, rnum);
-
- if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
- /*
- * check for 'registration complete'==H_SUCCESS
- * and for 'page registered'==H_PAGE_REGISTERED
- */
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "last "
- "hipz_reg_rpage_mr failed, h_ret=%lli "
- "e_mr=%p i=%x hca_hndl=%llx mr_hndl=%llx"
- " lkey=%x", h_ret, e_mr, i,
- shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle,
- e_mr->ib.ib_mr.lkey);
- ret = ehca2ib_return_code(h_ret);
- break;
- } else
- ret = 0;
- } else if (h_ret != H_PAGE_REGISTERED) {
- ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
- "h_ret=%lli e_mr=%p i=%x lkey=%x hca_hndl=%llx "
- "mr_hndl=%llx", h_ret, e_mr, i,
- e_mr->ib.ib_mr.lkey,
- shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle);
- ret = ehca2ib_return_code(h_ret);
- break;
- } else
- ret = 0;
- } /* end for(i) */
-
-
-ehca_reg_mr_rpages_exit1:
- ehca_free_fw_ctrlblock(kpage);
-ehca_reg_mr_rpages_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p pginfo=%p "
- "num_kpages=%llx num_hwpages=%llx", ret, shca, e_mr,
- pginfo, pginfo->num_kpages, pginfo->num_hwpages);
- return ret;
-} /* end ehca_reg_mr_rpages() */
-
-/*----------------------------------------------------------------------*/
-
-inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- u32 acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey, /*OUT*/
- u32 *rkey) /*OUT*/
-{
- int ret;
- u64 h_ret;
- u32 hipz_acl;
- u64 *kpage;
- u64 rpage;
- struct ehca_mr_pginfo pginfo_save;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
-
- kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!kpage) {
- ehca_err(&shca->ib_device, "kpage alloc failed");
- ret = -ENOMEM;
- goto ehca_rereg_mr_rereg1_exit0;
- }
-
- pginfo_save = *pginfo;
- ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);
- if (ret) {
- ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
- "pginfo=%p type=%x num_kpages=%llx num_hwpages=%llx "
- "kpage=%p", e_mr, pginfo, pginfo->type,
- pginfo->num_kpages, pginfo->num_hwpages, kpage);
- goto ehca_rereg_mr_rereg1_exit1;
- }
- rpage = __pa(kpage);
- if (!rpage) {
- ehca_err(&shca->ib_device, "kpage=%p", kpage);
- ret = -EFAULT;
- goto ehca_rereg_mr_rereg1_exit1;
- }
- h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
- (u64)iova_start, size, hipz_acl,
- e_pd->fw_pd, rpage, &hipzout);
- if (h_ret != H_SUCCESS) {
- /*
- * reregistration unsuccessful, try it again with the 3 hCalls,
- * e.g. this is required in case H_MR_CONDITION
- * (MW bound or MR is shared)
- */
- ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
- "(Rereg1), h_ret=%lli e_mr=%p", h_ret, e_mr);
- *pginfo = pginfo_save;
- ret = -EAGAIN;
- } else if ((u64 *)hipzout.vaddr != iova_start) {
- ehca_err(&shca->ib_device, "PHYP changed iova_start in "
- "rereg_pmr, iova_start=%p iova_start_out=%llx e_mr=%p "
- "mr_handle=%llx lkey=%x lkey_out=%x", iova_start,
- hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
- e_mr->ib.ib_mr.lkey, hipzout.lkey);
- ret = -EFAULT;
- } else {
- /*
- * successful reregistration
- * note: start and start_out are identical for eServer HCAs
- */
- e_mr->num_kpages = pginfo->num_kpages;
- e_mr->num_hwpages = pginfo->num_hwpages;
- e_mr->hwpage_size = pginfo->hwpage_size;
- e_mr->start = iova_start;
- e_mr->size = size;
- e_mr->acl = acl;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- }
-
-ehca_rereg_mr_rereg1_exit1:
- ehca_free_fw_ctrlblock(kpage);
-ehca_rereg_mr_rereg1_exit0:
- if ( ret && (ret != -EAGAIN) )
- ehca_err(&shca->ib_device, "ret=%i lkey=%x rkey=%x "
- "pginfo=%p num_kpages=%llx num_hwpages=%llx",
- ret, *lkey, *rkey, pginfo, pginfo->num_kpages,
- pginfo->num_hwpages);
- return ret;
-} /* end ehca_rereg_mr_rereg1() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_rereg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey,
- u32 *rkey)
-{
- int ret = 0;
- u64 h_ret;
- int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
- int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
-
- /* first determine reregistration hCall(s) */
- if ((pginfo->num_hwpages > MAX_RPAGES) ||
- (e_mr->num_hwpages > MAX_RPAGES) ||
- (pginfo->num_hwpages > e_mr->num_hwpages)) {
- ehca_dbg(&shca->ib_device, "Rereg3 case, "
- "pginfo->num_hwpages=%llx e_mr->num_hwpages=%x",
- pginfo->num_hwpages, e_mr->num_hwpages);
- rereg_1_hcall = 0;
- rereg_3_hcall = 1;
- }
-
- if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
- rereg_1_hcall = 0;
- rereg_3_hcall = 1;
- e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
- ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
- e_mr);
- }
-
- if (rereg_1_hcall) {
- ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
- acl, e_pd, pginfo, lkey, rkey);
- if (ret) {
- if (ret == -EAGAIN)
- rereg_3_hcall = 1;
- else
- goto ehca_rereg_mr_exit0;
- }
- }
-
- if (rereg_3_hcall) {
- struct ehca_mr save_mr;
-
- /* first deregister old MR */
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_free_mr failed, "
- "h_ret=%lli e_mr=%p hca_hndl=%llx mr_hndl=%llx "
- "mr->lkey=%x",
- h_ret, e_mr, shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle,
- e_mr->ib.ib_mr.lkey);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_rereg_mr_exit0;
- }
- /* clean ehca_mr_t, without changing struct ib_mr and lock */
- save_mr = *e_mr;
- ehca_mr_deletenew(e_mr);
-
- /* set some MR values */
- e_mr->flags = save_mr.flags;
- e_mr->hwpage_size = save_mr.hwpage_size;
- e_mr->fmr_page_size = save_mr.fmr_page_size;
- e_mr->fmr_max_pages = save_mr.fmr_max_pages;
- e_mr->fmr_max_maps = save_mr.fmr_max_maps;
- e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
-
- ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
- e_pd, pginfo, lkey, rkey, EHCA_REG_MR);
- if (ret) {
- u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
- memcpy(&e_mr->flags, &(save_mr.flags),
- sizeof(struct ehca_mr) - offset);
- goto ehca_rereg_mr_exit0;
- }
- }
-
-ehca_rereg_mr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
- "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
- "num_kpages=%llx lkey=%x rkey=%x rereg_1_hcall=%x "
- "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
- acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,
- rereg_1_hcall, rereg_3_hcall);
- return ret;
-} /* end ehca_rereg_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_unmap_one_fmr(struct ehca_shca *shca,
- struct ehca_mr *e_fmr)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_pd *e_pd =
- container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
- struct ehca_mr save_fmr;
- u32 tmp_lkey, tmp_rkey;
- struct ehca_mr_pginfo pginfo;
- struct ehca_mr_hipzout_parms hipzout;
- struct ehca_mr save_mr;
-
- if (e_fmr->fmr_max_pages <= MAX_RPAGES) {
- /*
- * note: after using rereg hcall with len=0,
- * rereg hcall must be used again for registering pages
- */
- h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
- 0, 0, e_pd->fw_pd, 0, &hipzout);
- if (h_ret == H_SUCCESS) {
- /* successful reregistration */
- e_fmr->start = NULL;
- e_fmr->size = 0;
- tmp_lkey = hipzout.lkey;
- tmp_rkey = hipzout.rkey;
- return 0;
- }
- /*
- * should not happen, because length checked above,
- * FMRs are not shared and no MW bound to FMRs
- */
- ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
- "(Rereg1), h_ret=%lli e_fmr=%p hca_hndl=%llx "
- "mr_hndl=%llx lkey=%x lkey_out=%x",
- h_ret, e_fmr, shca->ipz_hca_handle.handle,
- e_fmr->ipz_mr_handle.handle,
- e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
- /* try free and rereg */
- }
-
- /* first free old FMR */
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_free_mr failed, "
- "h_ret=%lli e_fmr=%p hca_hndl=%llx mr_hndl=%llx "
- "lkey=%x",
- h_ret, e_fmr, shca->ipz_hca_handle.handle,
- e_fmr->ipz_mr_handle.handle,
- e_fmr->ib.ib_fmr.lkey);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_unmap_one_fmr_exit0;
- }
- /* clean ehca_mr_t, without changing lock */
- save_fmr = *e_fmr;
- ehca_mr_deletenew(e_fmr);
-
- /* set some MR values */
- e_fmr->flags = save_fmr.flags;
- e_fmr->hwpage_size = save_fmr.hwpage_size;
- e_fmr->fmr_page_size = save_fmr.fmr_page_size;
- e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
- e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
- e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
- e_fmr->acl = save_fmr.acl;
-
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_FMR;
- ret = ehca_reg_mr(shca, e_fmr, NULL,
- (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
- e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
- &tmp_rkey, EHCA_REG_MR);
- if (ret) {
- u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
- memcpy(&e_fmr->flags, &(save_mr.flags),
- sizeof(struct ehca_mr) - offset);
- }
-
-ehca_unmap_one_fmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i tmp_lkey=%x tmp_rkey=%x "
- "fmr_max_pages=%x",
- ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages);
- return ret;
-} /* end ehca_unmap_one_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_reg_smr(struct ehca_shca *shca,
- struct ehca_mr *e_origmr,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey, /*OUT*/
- u32 *rkey) /*OUT*/
-{
- int ret = 0;
- u64 h_ret;
- u32 hipz_acl;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
-
- h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
- (u64)iova_start, hipz_acl, e_pd->fw_pd,
- &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
- "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
- "e_pd=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
- h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
- shca->ipz_hca_handle.handle,
- e_origmr->ipz_mr_handle.handle,
- e_origmr->ib.ib_mr.lkey);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_reg_smr_exit0;
- }
- /* successful registration */
- e_newmr->num_kpages = e_origmr->num_kpages;
- e_newmr->num_hwpages = e_origmr->num_hwpages;
- e_newmr->hwpage_size = e_origmr->hwpage_size;
- e_newmr->start = iova_start;
- e_newmr->size = e_origmr->size;
- e_newmr->acl = acl;
- e_newmr->ipz_mr_handle = hipzout.handle;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- return 0;
-
-ehca_reg_smr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_origmr=%p "
- "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
- ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
- return ret;
-} /* end ehca_reg_smr() */
-
-/*----------------------------------------------------------------------*/
-static inline void *ehca_calc_sectbase(int top, int dir, int idx)
-{
- unsigned long ret = idx;
- ret |= dir << EHCA_DIR_INDEX_SHIFT;
- ret |= top << EHCA_TOP_INDEX_SHIFT;
- return __va(ret << SECTION_SIZE_BITS);
-}
-
-#define ehca_bmap_valid(entry) \
- ((u64)entry != (u64)EHCA_INVAL_ADDR)
-
-static u64 ehca_reg_mr_section(int top, int dir, int idx, u64 *kpage,
- struct ehca_shca *shca, struct ehca_mr *mr,
- struct ehca_mr_pginfo *pginfo)
-{
- u64 h_ret = 0;
- unsigned long page = 0;
- u64 rpage = __pa(kpage);
- int page_count;
-
- void *sectbase = ehca_calc_sectbase(top, dir, idx);
- if ((unsigned long)sectbase & (pginfo->hwpage_size - 1)) {
- ehca_err(&shca->ib_device, "reg_mr_section will probably fail:"
- "hwpage_size does not fit to "
- "section start address");
- }
- page_count = EHCA_SECTSIZE / pginfo->hwpage_size;
-
- while (page < page_count) {
- u64 rnum;
- for (rnum = 0; (rnum < MAX_RPAGES) && (page < page_count);
- rnum++) {
- void *pg = sectbase + ((page++) * pginfo->hwpage_size);
- kpage[rnum] = __pa(pg);
- }
-
- h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, mr,
- ehca_encode_hwpage_size(pginfo->hwpage_size),
- 0, rpage, rnum);
-
- if ((h_ret != H_SUCCESS) && (h_ret != H_PAGE_REGISTERED)) {
- ehca_err(&shca->ib_device, "register_rpage_mr failed");
- return h_ret;
- }
- }
- return h_ret;
-}
-
-static u64 ehca_reg_mr_sections(int top, int dir, u64 *kpage,
- struct ehca_shca *shca, struct ehca_mr *mr,
- struct ehca_mr_pginfo *pginfo)
-{
- u64 hret = H_SUCCESS;
- int idx;
-
- for (idx = 0; idx < EHCA_MAP_ENTRIES; idx++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]->ent[idx]))
- continue;
-
- hret = ehca_reg_mr_section(top, dir, idx, kpage, shca, mr,
- pginfo);
- if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
- return hret;
- }
- return hret;
-}
-
-static u64 ehca_reg_mr_dir_sections(int top, u64 *kpage, struct ehca_shca *shca,
- struct ehca_mr *mr,
- struct ehca_mr_pginfo *pginfo)
-{
- u64 hret = H_SUCCESS;
- int dir;
-
- for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
- continue;
-
- hret = ehca_reg_mr_sections(top, dir, kpage, shca, mr, pginfo);
- if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
- return hret;
- }
- return hret;
-}
-
-/* register internal max-MR to internal SHCA */
-int ehca_reg_internal_maxmr(
- struct ehca_shca *shca,
- struct ehca_pd *e_pd,
- struct ehca_mr **e_maxmr) /*OUT*/
-{
- int ret;
- struct ehca_mr *e_mr;
- u64 *iova_start;
- u64 size_maxmr;
- struct ehca_mr_pginfo pginfo;
- struct ib_phys_buf ib_pbuf;
- u32 num_kpages;
- u32 num_hwpages;
- u64 hw_pgsize;
-
- if (!ehca_bmap) {
- ret = -EFAULT;
- goto ehca_reg_internal_maxmr_exit0;
- }
-
- e_mr = ehca_mr_new();
- if (!e_mr) {
- ehca_err(&shca->ib_device, "out of memory");
- ret = -ENOMEM;
- goto ehca_reg_internal_maxmr_exit0;
- }
- e_mr->flags |= EHCA_MR_FLAG_MAXMR;
-
- /* register internal max-MR on HCA */
- size_maxmr = ehca_mr_len;
- iova_start = (u64 *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START));
- ib_pbuf.addr = 0;
- ib_pbuf.size = size_maxmr;
- num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
- PAGE_SIZE);
- hw_pgsize = ehca_get_max_hwpage_size(shca);
- num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr,
- hw_pgsize);
-
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_PHYS;
- pginfo.num_kpages = num_kpages;
- pginfo.num_hwpages = num_hwpages;
- pginfo.hwpage_size = hw_pgsize;
- pginfo.u.phy.num_phys_buf = 1;
- pginfo.u.phy.phys_buf_array = &ib_pbuf;
-
- ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
- &pginfo, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey, EHCA_REG_BUSMAP_MR);
- if (ret) {
- ehca_err(&shca->ib_device, "reg of internal max MR failed, "
- "e_mr=%p iova_start=%p size_maxmr=%llx num_kpages=%x "
- "num_hwpages=%x", e_mr, iova_start, size_maxmr,
- num_kpages, num_hwpages);
- goto ehca_reg_internal_maxmr_exit1;
- }
-
- /* successful registration of all pages */
- e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
- e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
- e_mr->ib.ib_mr.uobject = NULL;
- atomic_inc(&(e_pd->ib_pd.usecnt));
- atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
- *e_maxmr = e_mr;
- return 0;
-
-ehca_reg_internal_maxmr_exit1:
- ehca_mr_delete(e_mr);
-ehca_reg_internal_maxmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_pd=%p e_maxmr=%p",
- ret, shca, e_pd, e_maxmr);
- return ret;
-} /* end ehca_reg_internal_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_reg_maxmr(struct ehca_shca *shca,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey,
- u32 *rkey)
-{
- u64 h_ret;
- struct ehca_mr *e_origmr = shca->maxmr;
- u32 hipz_acl;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
-
- h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
- (u64)iova_start, hipz_acl, e_pd->fw_pd,
- &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
- "e_origmr=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
- h_ret, e_origmr, shca->ipz_hca_handle.handle,
- e_origmr->ipz_mr_handle.handle,
- e_origmr->ib.ib_mr.lkey);
- return ehca2ib_return_code(h_ret);
- }
- /* successful registration */
- e_newmr->num_kpages = e_origmr->num_kpages;
- e_newmr->num_hwpages = e_origmr->num_hwpages;
- e_newmr->hwpage_size = e_origmr->hwpage_size;
- e_newmr->start = iova_start;
- e_newmr->size = e_origmr->size;
- e_newmr->acl = acl;
- e_newmr->ipz_mr_handle = hipzout.handle;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- return 0;
-} /* end ehca_reg_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
-{
- int ret;
- struct ehca_mr *e_maxmr;
- struct ib_pd *ib_pd;
-
- if (!shca->maxmr) {
- ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
- ret = -EINVAL;
- goto ehca_dereg_internal_maxmr_exit0;
- }
-
- e_maxmr = shca->maxmr;
- ib_pd = e_maxmr->ib.ib_mr.pd;
- shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
-
- ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
- if (ret) {
- ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
- "ret=%i e_maxmr=%p shca=%p lkey=%x",
- ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
- shca->maxmr = e_maxmr;
- goto ehca_dereg_internal_maxmr_exit0;
- }
-
- atomic_dec(&ib_pd->usecnt);
-
-ehca_dereg_internal_maxmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p shca->maxmr=%p",
- ret, shca, shca->maxmr);
- return ret;
-} /* end ehca_dereg_internal_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-/*
- * check physical buffer array of MR verbs for validness and
- * calculates MR size
- */
-int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- u64 *iova_start,
- u64 *size)
-{
- struct ib_phys_buf *pbuf = phys_buf_array;
- u64 size_count = 0;
- u32 i;
-
- if (num_phys_buf == 0) {
- ehca_gen_err("bad phys buf array len, num_phys_buf=0");
- return -EINVAL;
- }
- /* check first buffer */
- if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
- ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
- "pbuf->addr=%llx pbuf->size=%llx",
- iova_start, pbuf->addr, pbuf->size);
- return -EINVAL;
- }
- if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
- (num_phys_buf > 1)) {
- ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%llx "
- "pbuf->size=%llx", pbuf->addr, pbuf->size);
- return -EINVAL;
- }
-
- for (i = 0; i < num_phys_buf; i++) {
- if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
- ehca_gen_err("bad address, i=%x pbuf->addr=%llx "
- "pbuf->size=%llx",
- i, pbuf->addr, pbuf->size);
- return -EINVAL;
- }
- if (((i > 0) && /* not 1st */
- (i < (num_phys_buf - 1)) && /* not last */
- (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
- ehca_gen_err("bad size, i=%x pbuf->size=%llx",
- i, pbuf->size);
- return -EINVAL;
- }
- size_count += pbuf->size;
- pbuf++;
- }
-
- *size = size_count;
- return 0;
-} /* end ehca_mr_chk_buf_and_calc_size() */
-
-/*----------------------------------------------------------------------*/
-
-/* check page list of map FMR verb for validness */
-int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
- u64 *page_list,
- int list_len)
-{
- u32 i;
- u64 *page;
-
- if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
- ehca_gen_err("bad list_len, list_len=%x "
- "e_fmr->fmr_max_pages=%x fmr=%p",
- list_len, e_fmr->fmr_max_pages, e_fmr);
- return -EINVAL;
- }
-
- /* each page must be aligned */
- page = page_list;
- for (i = 0; i < list_len; i++) {
- if (*page % e_fmr->fmr_page_size) {
- ehca_gen_err("bad page, i=%x *page=%llx page=%p fmr=%p "
- "fmr_page_size=%x", i, *page, page, e_fmr,
- e_fmr->fmr_page_size);
- return -EINVAL;
- }
- page++;
- }
-
- return 0;
-} /* end ehca_fmr_check_page_list() */
-
-/*----------------------------------------------------------------------*/
-
-/* PAGE_SIZE >= pginfo->hwpage_size */
-static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
-{
- int ret = 0;
- u64 pgaddr;
- u32 j = 0;
- int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
- struct scatterlist **sg = &pginfo->u.usr.next_sg;
-
- while (*sg != NULL) {
- pgaddr = page_to_pfn(sg_page(*sg))
- << PAGE_SHIFT;
- *kpage = pgaddr + (pginfo->next_hwpage *
- pginfo->hwpage_size);
- if (!(*kpage)) {
- ehca_gen_err("pgaddr=%llx "
- "sg_dma_address=%llx "
- "entry=%llx next_hwpage=%llx",
- pgaddr, (u64)sg_dma_address(*sg),
- pginfo->u.usr.next_nmap,
- pginfo->next_hwpage);
- return -EFAULT;
- }
- (pginfo->hwpage_cnt)++;
- (pginfo->next_hwpage)++;
- kpage++;
- if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
- (pginfo->kpage_cnt)++;
- (pginfo->u.usr.next_nmap)++;
- pginfo->next_hwpage = 0;
- *sg = sg_next(*sg);
- }
- j++;
- if (j >= number)
- break;
- }
-
- return ret;
-}
-
-/*
- * check given pages for contiguous layout
- * last page addr is returned in prev_pgaddr for further check
- */
-static int ehca_check_kpages_per_ate(struct scatterlist **sg,
- int num_pages,
- u64 *prev_pgaddr)
-{
- for (; *sg && num_pages > 0; *sg = sg_next(*sg), num_pages--) {
- u64 pgaddr = page_to_pfn(sg_page(*sg)) << PAGE_SHIFT;
- if (ehca_debug_level >= 3)
- ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr,
- *(u64 *)__va(pgaddr));
- if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
- ehca_gen_err("uncontiguous page found pgaddr=%llx "
- "prev_pgaddr=%llx entries_left_in_hwpage=%x",
- pgaddr, *prev_pgaddr, num_pages);
- return -EINVAL;
- }
- *prev_pgaddr = pgaddr;
- }
- return 0;
-}
-
-/* PAGE_SIZE < pginfo->hwpage_size */
-static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
-{
- int ret = 0;
- u64 pgaddr, prev_pgaddr;
- u32 j = 0;
- int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
- int nr_kpages = kpages_per_hwpage;
- struct scatterlist **sg = &pginfo->u.usr.next_sg;
-
- while (*sg != NULL) {
-
- if (nr_kpages == kpages_per_hwpage) {
- pgaddr = (page_to_pfn(sg_page(*sg))
- << PAGE_SHIFT);
- *kpage = pgaddr;
- if (!(*kpage)) {
- ehca_gen_err("pgaddr=%llx entry=%llx",
- pgaddr, pginfo->u.usr.next_nmap);
- ret = -EFAULT;
- return ret;
- }
- /*
- * The first page in a hwpage must be aligned;
- * the first MR page is exempt from this rule.
- */
- if (pgaddr & (pginfo->hwpage_size - 1)) {
- if (pginfo->hwpage_cnt) {
- ehca_gen_err(
- "invalid alignment "
- "pgaddr=%llx entry=%llx "
- "mr_pgsize=%llx",
- pgaddr, pginfo->u.usr.next_nmap,
- pginfo->hwpage_size);
- ret = -EFAULT;
- return ret;
- }
- /* first MR page */
- pginfo->kpage_cnt =
- (pgaddr &
- (pginfo->hwpage_size - 1)) >>
- PAGE_SHIFT;
- nr_kpages -= pginfo->kpage_cnt;
- *kpage = pgaddr &
- ~(pginfo->hwpage_size - 1);
- }
- if (ehca_debug_level >= 3) {
- u64 val = *(u64 *)__va(pgaddr);
- ehca_gen_dbg("kpage=%llx page=%llx "
- "value=%016llx",
- *kpage, pgaddr, val);
- }
- prev_pgaddr = pgaddr;
- *sg = sg_next(*sg);
- pginfo->kpage_cnt++;
- pginfo->u.usr.next_nmap++;
- nr_kpages--;
- if (!nr_kpages)
- goto next_kpage;
- continue;
- }
-
- ret = ehca_check_kpages_per_ate(sg, nr_kpages,
- &prev_pgaddr);
- if (ret)
- return ret;
- pginfo->kpage_cnt += nr_kpages;
- pginfo->u.usr.next_nmap += nr_kpages;
-
-next_kpage:
- nr_kpages = kpages_per_hwpage;
- (pginfo->hwpage_cnt)++;
- kpage++;
- j++;
- if (j >= number)
- break;
- }
-
- return ret;
-}
-
-static int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
- u32 number, u64 *kpage)
-{
- int ret = 0;
- struct ib_phys_buf *pbuf;
- u64 num_hw, offs_hw;
- u32 i = 0;
-
- /* loop over desired phys_buf_array entries */
- while (i < number) {
- pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
- num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +
- pbuf->size, pginfo->hwpage_size);
- offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /
- pginfo->hwpage_size;
- while (pginfo->next_hwpage < offs_hw + num_hw) {
- /* sanity check */
- if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
- (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
- ehca_gen_err("kpage_cnt >= num_kpages, "
- "kpage_cnt=%llx num_kpages=%llx "
- "hwpage_cnt=%llx "
- "num_hwpages=%llx i=%x",
- pginfo->kpage_cnt,
- pginfo->num_kpages,
- pginfo->hwpage_cnt,
- pginfo->num_hwpages, i);
- return -EFAULT;
- }
- *kpage = (pbuf->addr & ~(pginfo->hwpage_size - 1)) +
- (pginfo->next_hwpage * pginfo->hwpage_size);
- if ( !(*kpage) && pbuf->addr ) {
- ehca_gen_err("pbuf->addr=%llx pbuf->size=%llx "
- "next_hwpage=%llx", pbuf->addr,
- pbuf->size, pginfo->next_hwpage);
- return -EFAULT;
- }
- (pginfo->hwpage_cnt)++;
- (pginfo->next_hwpage)++;
- if (PAGE_SIZE >= pginfo->hwpage_size) {
- if (pginfo->next_hwpage %
- (PAGE_SIZE / pginfo->hwpage_size) == 0)
- (pginfo->kpage_cnt)++;
- } else
- pginfo->kpage_cnt += pginfo->hwpage_size /
- PAGE_SIZE;
- kpage++;
- i++;
- if (i >= number) break;
- }
- if (pginfo->next_hwpage >= offs_hw + num_hw) {
- (pginfo->u.phy.next_buf)++;
- pginfo->next_hwpage = 0;
- }
- }
- return ret;
-}
-
-static int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
- u32 number, u64 *kpage)
-{
- int ret = 0;
- u64 *fmrlist;
- u32 i;
-
- /* loop over desired page_list entries */
- fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
- for (i = 0; i < number; i++) {
- *kpage = (*fmrlist & ~(pginfo->hwpage_size - 1)) +
- pginfo->next_hwpage * pginfo->hwpage_size;
- if ( !(*kpage) ) {
- ehca_gen_err("*fmrlist=%llx fmrlist=%p "
- "next_listelem=%llx next_hwpage=%llx",
- *fmrlist, fmrlist,
- pginfo->u.fmr.next_listelem,
- pginfo->next_hwpage);
- return -EFAULT;
- }
- (pginfo->hwpage_cnt)++;
- if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) {
- if (pginfo->next_hwpage %
- (pginfo->u.fmr.fmr_pgsize /
- pginfo->hwpage_size) == 0) {
- (pginfo->kpage_cnt)++;
- (pginfo->u.fmr.next_listelem)++;
- fmrlist++;
- pginfo->next_hwpage = 0;
- } else
- (pginfo->next_hwpage)++;
- } else {
- unsigned int cnt_per_hwpage = pginfo->hwpage_size /
- pginfo->u.fmr.fmr_pgsize;
- unsigned int j;
- u64 prev = *kpage;
- /* check if adrs are contiguous */
- for (j = 1; j < cnt_per_hwpage; j++) {
- u64 p = fmrlist[j] & ~(pginfo->hwpage_size - 1);
- if (prev + pginfo->u.fmr.fmr_pgsize != p) {
- ehca_gen_err("uncontiguous fmr pages "
- "found prev=%llx p=%llx "
- "idx=%x", prev, p, i + j);
- return -EINVAL;
- }
- prev = p;
- }
- pginfo->kpage_cnt += cnt_per_hwpage;
- pginfo->u.fmr.next_listelem += cnt_per_hwpage;
- fmrlist += cnt_per_hwpage;
- }
- kpage++;
- }
- return ret;
-}
-
-/* setup page buffer from page info */
-int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
-{
- int ret;
-
- switch (pginfo->type) {
- case EHCA_MR_PGI_PHYS:
- ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
- break;
- case EHCA_MR_PGI_USER:
- ret = PAGE_SIZE >= pginfo->hwpage_size ?
- ehca_set_pagebuf_user1(pginfo, number, kpage) :
- ehca_set_pagebuf_user2(pginfo, number, kpage);
- break;
- case EHCA_MR_PGI_FMR:
- ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
- break;
- default:
- ehca_gen_err("bad pginfo->type=%x", pginfo->type);
- ret = -EFAULT;
- break;
- }
- return ret;
-} /* end ehca_set_pagebuf() */
-
-/*----------------------------------------------------------------------*/
-
-/*
- * check MR if it is a max-MR, i.e. uses whole memory
- * in case it's a max-MR 1 is returned, else 0
- */
-int ehca_mr_is_maxmr(u64 size,
- u64 *iova_start)
-{
- /* a MR is treated as max-MR only if it fits following: */
- if ((size == ehca_mr_len) &&
- (iova_start == (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)))) {
- ehca_gen_dbg("this is a max-MR");
- return 1;
- } else
- return 0;
-} /* end ehca_mr_is_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-/* map access control for MR/MW. This routine is used for MR and MW. */
-void ehca_mrmw_map_acl(int ib_acl,
- u32 *hipz_acl)
-{
- *hipz_acl = 0;
- if (ib_acl & IB_ACCESS_REMOTE_READ)
- *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
- if (ib_acl & IB_ACCESS_REMOTE_WRITE)
- *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
- if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
- *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
- if (ib_acl & IB_ACCESS_LOCAL_WRITE)
- *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
- if (ib_acl & IB_ACCESS_MW_BIND)
- *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
-} /* end ehca_mrmw_map_acl() */
-
-/*----------------------------------------------------------------------*/
-
-/* sets page size in hipz access control for MR/MW. */
-void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/
-{
- *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24);
-} /* end ehca_mrmw_set_pgsize_hipz_acl() */
-
-/*----------------------------------------------------------------------*/
-
-/*
- * reverse map access control for MR/MW.
- * This routine is used for MR and MW.
- */
-void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
- int *ib_acl) /*OUT*/
-{
- *ib_acl = 0;
- if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
- *ib_acl |= IB_ACCESS_REMOTE_READ;
- if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
- *ib_acl |= IB_ACCESS_REMOTE_WRITE;
- if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
- *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
- if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
- *ib_acl |= IB_ACCESS_LOCAL_WRITE;
- if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
- *ib_acl |= IB_ACCESS_MW_BIND;
-} /* end ehca_mrmw_reverse_map_acl() */
-
-
-/*----------------------------------------------------------------------*/
-
-/*
- * MR destructor and constructor
- * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
- * except struct ib_mr and spinlock
- */
-void ehca_mr_deletenew(struct ehca_mr *mr)
-{
- mr->flags = 0;
- mr->num_kpages = 0;
- mr->num_hwpages = 0;
- mr->acl = 0;
- mr->start = NULL;
- mr->fmr_page_size = 0;
- mr->fmr_max_pages = 0;
- mr->fmr_max_maps = 0;
- mr->fmr_map_cnt = 0;
- memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
- memset(&mr->galpas, 0, sizeof(mr->galpas));
-} /* end ehca_mr_deletenew() */
-
-int ehca_init_mrmw_cache(void)
-{
- mr_cache = kmem_cache_create("ehca_cache_mr",
- sizeof(struct ehca_mr), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!mr_cache)
- return -ENOMEM;
- mw_cache = kmem_cache_create("ehca_cache_mw",
- sizeof(struct ehca_mw), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!mw_cache) {
- kmem_cache_destroy(mr_cache);
- mr_cache = NULL;
- return -ENOMEM;
- }
- return 0;
-}
-
-void ehca_cleanup_mrmw_cache(void)
-{
- if (mr_cache)
- kmem_cache_destroy(mr_cache);
- if (mw_cache)
- kmem_cache_destroy(mw_cache);
-}
-
-static inline int ehca_init_top_bmap(struct ehca_top_bmap *ehca_top_bmap,
- int dir)
-{
- if (!ehca_bmap_valid(ehca_top_bmap->dir[dir])) {
- ehca_top_bmap->dir[dir] =
- kmalloc(sizeof(struct ehca_dir_bmap), GFP_KERNEL);
- if (!ehca_top_bmap->dir[dir])
- return -ENOMEM;
- /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
- memset(ehca_top_bmap->dir[dir], 0xFF, EHCA_ENT_MAP_SIZE);
- }
- return 0;
-}
-
-static inline int ehca_init_bmap(struct ehca_bmap *ehca_bmap, int top, int dir)
-{
- if (!ehca_bmap_valid(ehca_bmap->top[top])) {
- ehca_bmap->top[top] =
- kmalloc(sizeof(struct ehca_top_bmap), GFP_KERNEL);
- if (!ehca_bmap->top[top])
- return -ENOMEM;
- /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
- memset(ehca_bmap->top[top], 0xFF, EHCA_DIR_MAP_SIZE);
- }
- return ehca_init_top_bmap(ehca_bmap->top[top], dir);
-}
-
-static inline int ehca_calc_index(unsigned long i, unsigned long s)
-{
- return (i >> s) & EHCA_INDEX_MASK;
-}
-
-void ehca_destroy_busmap(void)
-{
- int top, dir;
-
- if (!ehca_bmap)
- return;
-
- for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]))
- continue;
- for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
- continue;
-
- kfree(ehca_bmap->top[top]->dir[dir]);
- }
-
- kfree(ehca_bmap->top[top]);
- }
-
- kfree(ehca_bmap);
- ehca_bmap = NULL;
-}
-
-static int ehca_update_busmap(unsigned long pfn, unsigned long nr_pages)
-{
- unsigned long i, start_section, end_section;
- int top, dir, idx;
-
- if (!nr_pages)
- return 0;
-
- if (!ehca_bmap) {
- ehca_bmap = kmalloc(sizeof(struct ehca_bmap), GFP_KERNEL);
- if (!ehca_bmap)
- return -ENOMEM;
- /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
- memset(ehca_bmap, 0xFF, EHCA_TOP_MAP_SIZE);
- }
-
- start_section = (pfn * PAGE_SIZE) / EHCA_SECTSIZE;
- end_section = ((pfn + nr_pages) * PAGE_SIZE) / EHCA_SECTSIZE;
- for (i = start_section; i < end_section; i++) {
- int ret;
- top = ehca_calc_index(i, EHCA_TOP_INDEX_SHIFT);
- dir = ehca_calc_index(i, EHCA_DIR_INDEX_SHIFT);
- idx = i & EHCA_INDEX_MASK;
-
- ret = ehca_init_bmap(ehca_bmap, top, dir);
- if (ret) {
- ehca_destroy_busmap();
- return ret;
- }
- ehca_bmap->top[top]->dir[dir]->ent[idx] = ehca_mr_len;
- ehca_mr_len += EHCA_SECTSIZE;
- }
- return 0;
-}
-
-static int ehca_is_hugepage(unsigned long pfn)
-{
- int page_order;
-
- if (pfn & EHCA_HUGEPAGE_PFN_MASK)
- return 0;
-
- page_order = compound_order(pfn_to_page(pfn));
- if (page_order + PAGE_SHIFT != EHCA_HUGEPAGESHIFT)
- return 0;
-
- return 1;
-}
-
-static int ehca_create_busmap_callback(unsigned long initial_pfn,
- unsigned long total_nr_pages, void *arg)
-{
- int ret;
- unsigned long pfn, start_pfn, end_pfn, nr_pages;
-
- if ((total_nr_pages * PAGE_SIZE) < EHCA_HUGEPAGE_SIZE)
- return ehca_update_busmap(initial_pfn, total_nr_pages);
-
- /* Given chunk is >= 16GB -> check for hugepages */
- start_pfn = initial_pfn;
- end_pfn = initial_pfn + total_nr_pages;
- pfn = start_pfn;
-
- while (pfn < end_pfn) {
- if (ehca_is_hugepage(pfn)) {
- /* Add mem found in front of the hugepage */
- nr_pages = pfn - start_pfn;
- ret = ehca_update_busmap(start_pfn, nr_pages);
- if (ret)
- return ret;
- /* Skip the hugepage */
- pfn += (EHCA_HUGEPAGE_SIZE / PAGE_SIZE);
- start_pfn = pfn;
- } else
- pfn += (EHCA_SECTSIZE / PAGE_SIZE);
- }
-
- /* Add mem found behind the hugepage(s) */
- nr_pages = pfn - start_pfn;
- return ehca_update_busmap(start_pfn, nr_pages);
-}
-
-int ehca_create_busmap(void)
-{
- int ret;
-
- ehca_mr_len = 0;
- ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL,
- ehca_create_busmap_callback);
- return ret;
-}
-
-static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo)
-{
- int top;
- u64 hret, *kpage;
-
- kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!kpage) {
- ehca_err(&shca->ib_device, "kpage alloc failed");
- return -ENOMEM;
- }
- for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]))
- continue;
- hret = ehca_reg_mr_dir_sections(top, kpage, shca, e_mr, pginfo);
- if ((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS))
- break;
- }
-
- ehca_free_fw_ctrlblock(kpage);
-
- if (hret == H_SUCCESS)
- return 0; /* Everything is fine */
- else {
- ehca_err(&shca->ib_device, "ehca_reg_bmap_mr_rpages failed, "
- "h_ret=%lli e_mr=%p top=%x lkey=%x "
- "hca_hndl=%llx mr_hndl=%llx", hret, e_mr, top,
- e_mr->ib.ib_mr.lkey,
- shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle);
- return ehca2ib_return_code(hret);
- }
-}
-
-static u64 ehca_map_vaddr(void *caddr)
-{
- int top, dir, idx;
- unsigned long abs_addr, offset;
- u64 entry;
-
- if (!ehca_bmap)
- return EHCA_INVAL_ADDR;
-
- abs_addr = __pa(caddr);
- top = ehca_calc_index(abs_addr, EHCA_TOP_INDEX_SHIFT + EHCA_SECTSHIFT);
- if (!ehca_bmap_valid(ehca_bmap->top[top]))
- return EHCA_INVAL_ADDR;
-
- dir = ehca_calc_index(abs_addr, EHCA_DIR_INDEX_SHIFT + EHCA_SECTSHIFT);
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
- return EHCA_INVAL_ADDR;
-
- idx = ehca_calc_index(abs_addr, EHCA_SECTSHIFT);
-
- entry = ehca_bmap->top[top]->dir[dir]->ent[idx];
- if (ehca_bmap_valid(entry)) {
- offset = (unsigned long)caddr & (EHCA_SECTSIZE - 1);
- return entry | offset;
- } else
- return EHCA_INVAL_ADDR;
-}
-
-static int ehca_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
-{
- return dma_addr == EHCA_INVAL_ADDR;
-}
-
-static u64 ehca_dma_map_single(struct ib_device *dev, void *cpu_addr,
- size_t size, enum dma_data_direction direction)
-{
- if (cpu_addr)
- return ehca_map_vaddr(cpu_addr);
- else
- return EHCA_INVAL_ADDR;
-}
-
-static void ehca_dma_unmap_single(struct ib_device *dev, u64 addr, size_t size,
- enum dma_data_direction direction)
-{
- /* This is only a stub; nothing to be done here */
-}
-
-static u64 ehca_dma_map_page(struct ib_device *dev, struct page *page,
- unsigned long offset, size_t size,
- enum dma_data_direction direction)
-{
- u64 addr;
-
- if (offset + size > PAGE_SIZE)
- return EHCA_INVAL_ADDR;
-
- addr = ehca_map_vaddr(page_address(page));
- if (!ehca_dma_mapping_error(dev, addr))
- addr += offset;
-
- return addr;
-}
-
-static void ehca_dma_unmap_page(struct ib_device *dev, u64 addr, size_t size,
- enum dma_data_direction direction)
-{
- /* This is only a stub; nothing to be done here */
-}
-
-static int ehca_dma_map_sg(struct ib_device *dev, struct scatterlist *sgl,
- int nents, enum dma_data_direction direction)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sgl, sg, nents, i) {
- u64 addr;
- addr = ehca_map_vaddr(sg_virt(sg));
- if (ehca_dma_mapping_error(dev, addr))
- return 0;
-
- sg->dma_address = addr;
- sg->dma_length = sg->length;
- }
- return nents;
-}
-
-static void ehca_dma_unmap_sg(struct ib_device *dev, struct scatterlist *sg,
- int nents, enum dma_data_direction direction)
-{
- /* This is only a stub; nothing to be done here */
-}
-
-static void ehca_dma_sync_single_for_cpu(struct ib_device *dev, u64 addr,
- size_t size,
- enum dma_data_direction dir)
-{
- dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
-}
-
-static void ehca_dma_sync_single_for_device(struct ib_device *dev, u64 addr,
- size_t size,
- enum dma_data_direction dir)
-{
- dma_sync_single_for_device(dev->dma_device, addr, size, dir);
-}
-
-static void *ehca_dma_alloc_coherent(struct ib_device *dev, size_t size,
- u64 *dma_handle, gfp_t flag)
-{
- struct page *p;
- void *addr = NULL;
- u64 dma_addr;
-
- p = alloc_pages(flag, get_order(size));
- if (p) {
- addr = page_address(p);
- dma_addr = ehca_map_vaddr(addr);
- if (ehca_dma_mapping_error(dev, dma_addr)) {
- free_pages((unsigned long)addr, get_order(size));
- return NULL;
- }
- if (dma_handle)
- *dma_handle = dma_addr;
- return addr;
- }
- return NULL;
-}
-
-static void ehca_dma_free_coherent(struct ib_device *dev, size_t size,
- void *cpu_addr, u64 dma_handle)
-{
- if (cpu_addr && size)
- free_pages((unsigned long)cpu_addr, get_order(size));
-}
-
-
-struct ib_dma_mapping_ops ehca_dma_mapping_ops = {
- .mapping_error = ehca_dma_mapping_error,
- .map_single = ehca_dma_map_single,
- .unmap_single = ehca_dma_unmap_single,
- .map_page = ehca_dma_map_page,
- .unmap_page = ehca_dma_unmap_page,
- .map_sg = ehca_dma_map_sg,
- .unmap_sg = ehca_dma_unmap_sg,
- .sync_single_for_cpu = ehca_dma_sync_single_for_cpu,
- .sync_single_for_device = ehca_dma_sync_single_for_device,
- .alloc_coherent = ehca_dma_alloc_coherent,
- .free_coherent = ehca_dma_free_coherent,
-};
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * MR/MW declarations and inline functions
- *
- * Authors: Dietmar Decker <ddecker@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _EHCA_MRMW_H_
-#define _EHCA_MRMW_H_
-
-enum ehca_reg_type {
- EHCA_REG_MR,
- EHCA_REG_BUSMAP_MR
-};
-
-int ehca_reg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey,
- u32 *rkey,
- enum ehca_reg_type reg_type);
-
-int ehca_reg_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo);
-
-int ehca_rereg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int mr_access_flags,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey,
- u32 *rkey);
-
-int ehca_unmap_one_fmr(struct ehca_shca *shca,
- struct ehca_mr *e_fmr);
-
-int ehca_reg_smr(struct ehca_shca *shca,
- struct ehca_mr *e_origmr,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey,
- u32 *rkey);
-
-int ehca_reg_internal_maxmr(struct ehca_shca *shca,
- struct ehca_pd *e_pd,
- struct ehca_mr **maxmr);
-
-int ehca_reg_maxmr(struct ehca_shca *shca,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey,
- u32 *rkey);
-
-int ehca_dereg_internal_maxmr(struct ehca_shca *shca);
-
-int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- u64 *iova_start,
- u64 *size);
-
-int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
- u64 *page_list,
- int list_len);
-
-int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage);
-
-int ehca_mr_is_maxmr(u64 size,
- u64 *iova_start);
-
-void ehca_mrmw_map_acl(int ib_acl,
- u32 *hipz_acl);
-
-void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl);
-
-void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
- int *ib_acl);
-
-void ehca_mr_deletenew(struct ehca_mr *mr);
-
-int ehca_create_busmap(void);
-
-void ehca_destroy_busmap(void);
-
-extern struct ib_dma_mapping_ops ehca_dma_mapping_ops;
-#endif /*_EHCA_MRMW_H_*/
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * PD functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-
-static struct kmem_cache *pd_cache;
-
-struct ib_pd *ehca_alloc_pd(struct ib_device *device,
- struct ib_ucontext *context, struct ib_udata *udata)
-{
- struct ehca_pd *pd;
- int i;
-
- pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL);
- if (!pd) {
- ehca_err(device, "device=%p context=%p out of memory",
- device, context);
- return ERR_PTR(-ENOMEM);
- }
-
- for (i = 0; i < 2; i++) {
- INIT_LIST_HEAD(&pd->free[i]);
- INIT_LIST_HEAD(&pd->full[i]);
- }
- mutex_init(&pd->lock);
-
- /*
- * Kernel PD: when device = -1, 0
- * User PD: when context != -1
- */
- if (!context) {
- /*
- * Kernel PDs after init reuses always
- * the one created in ehca_shca_reopen()
- */
- struct ehca_shca *shca = container_of(device, struct ehca_shca,
- ib_device);
- pd->fw_pd.value = shca->pd->fw_pd.value;
- } else
- pd->fw_pd.value = (u64)pd;
-
- return &pd->ib_pd;
-}
-
-int ehca_dealloc_pd(struct ib_pd *pd)
-{
- struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
- int i, leftovers = 0;
- struct ipz_small_queue_page *page, *tmp;
-
- for (i = 0; i < 2; i++) {
- list_splice(&my_pd->full[i], &my_pd->free[i]);
- list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) {
- leftovers = 1;
- free_page(page->page);
- kmem_cache_free(small_qp_cache, page);
- }
- }
-
- if (leftovers)
- ehca_warn(pd->device,
- "Some small queue pages were not freed");
-
- kmem_cache_free(pd_cache, my_pd);
-
- return 0;
-}
-
-int ehca_init_pd_cache(void)
-{
- pd_cache = kmem_cache_create("ehca_cache_pd",
- sizeof(struct ehca_pd), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!pd_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_pd_cache(void)
-{
- if (pd_cache)
- kmem_cache_destroy(pd_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Hardware request structures
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef _EHCA_QES_H_
-#define _EHCA_QES_H_
-
-#include "ehca_tools.h"
-
-/* virtual scatter gather entry to specify remote addresses with length */
-struct ehca_vsgentry {
- u64 vaddr;
- u32 lkey;
- u32 length;
-};
-
-#define GRH_FLAG_MASK EHCA_BMASK_IBM( 7, 7)
-#define GRH_IPVERSION_MASK EHCA_BMASK_IBM( 0, 3)
-#define GRH_TCLASS_MASK EHCA_BMASK_IBM( 4, 12)
-#define GRH_FLOWLABEL_MASK EHCA_BMASK_IBM(13, 31)
-#define GRH_PAYLEN_MASK EHCA_BMASK_IBM(32, 47)
-#define GRH_NEXTHEADER_MASK EHCA_BMASK_IBM(48, 55)
-#define GRH_HOPLIMIT_MASK EHCA_BMASK_IBM(56, 63)
-
-/*
- * Unreliable Datagram Address Vector Format
- * see IBTA Vol1 chapter 8.3 Global Routing Header
- */
-struct ehca_ud_av {
- u8 sl;
- u8 lnh;
- u16 dlid;
- u8 reserved1;
- u8 reserved2;
- u8 reserved3;
- u8 slid_path_bits;
- u8 reserved4;
- u8 ipd;
- u8 reserved5;
- u8 pmtu;
- u32 reserved6;
- u64 reserved7;
- union {
- struct {
- u64 word_0; /* always set to 6 */
- /*should be 0x1B for IB transport */
- u64 word_1;
- u64 word_2;
- u64 word_3;
- u64 word_4;
- } grh;
- struct {
- u32 wd_0;
- u32 wd_1;
- /* DWord_1 --> SGID */
-
- u32 sgid_wd3;
- u32 sgid_wd2;
-
- u32 sgid_wd1;
- u32 sgid_wd0;
- /* DWord_3 --> DGID */
-
- u32 dgid_wd3;
- u32 dgid_wd2;
-
- u32 dgid_wd1;
- u32 dgid_wd0;
- } grh_l;
- };
-};
-
-/* maximum number of sg entries allowed in a WQE */
-#define MAX_WQE_SG_ENTRIES 252
-
-#define WQE_OPTYPE_SEND 0x80
-#define WQE_OPTYPE_RDMAREAD 0x40
-#define WQE_OPTYPE_RDMAWRITE 0x20
-#define WQE_OPTYPE_CMPSWAP 0x10
-#define WQE_OPTYPE_FETCHADD 0x08
-#define WQE_OPTYPE_BIND 0x04
-
-#define WQE_WRFLAG_REQ_SIGNAL_COM 0x80
-#define WQE_WRFLAG_FENCE 0x40
-#define WQE_WRFLAG_IMM_DATA_PRESENT 0x20
-#define WQE_WRFLAG_SOLIC_EVENT 0x10
-
-#define WQEF_CACHE_HINT 0x80
-#define WQEF_CACHE_HINT_RD_WR 0x40
-#define WQEF_TIMED_WQE 0x20
-#define WQEF_PURGE 0x08
-#define WQEF_HIGH_NIBBLE 0xF0
-
-#define MW_BIND_ACCESSCTRL_R_WRITE 0x40
-#define MW_BIND_ACCESSCTRL_R_READ 0x20
-#define MW_BIND_ACCESSCTRL_R_ATOMIC 0x10
-
-struct ehca_wqe {
- u64 work_request_id;
- u8 optype;
- u8 wr_flag;
- u16 pkeyi;
- u8 wqef;
- u8 nr_of_data_seg;
- u16 wqe_provided_slid;
- u32 destination_qp_number;
- u32 resync_psn_sqp;
- u32 local_ee_context_qkey;
- u32 immediate_data;
- union {
- struct {
- u64 remote_virtual_address;
- u32 rkey;
- u32 reserved;
- u64 atomic_1st_op_dma_len;
- u64 atomic_2nd_op;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
-
- } nud;
- struct {
- u64 ehca_ud_av_ptr;
- u64 reserved1;
- u64 reserved2;
- u64 reserved3;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
- } ud_avp;
- struct {
- struct ehca_ud_av ud_av;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES -
- 2];
- } ud_av;
- struct {
- u64 reserved0;
- u64 reserved1;
- u64 reserved2;
- u64 reserved3;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
- } all_rcv;
-
- struct {
- u64 reserved;
- u32 rkey;
- u32 old_rkey;
- u64 reserved1;
- u64 reserved2;
- u64 virtual_address;
- u32 reserved3;
- u32 length;
- u32 reserved4;
- u16 reserved5;
- u8 reserved6;
- u8 lr_ctl;
- u32 lkey;
- u32 reserved7;
- u64 reserved8;
- u64 reserved9;
- u64 reserved10;
- u64 reserved11;
- } bind;
- struct {
- u64 reserved12;
- u64 reserved13;
- u32 size;
- u32 start;
- } inline_data;
- } u;
-
-};
-
-#define WC_SEND_RECEIVE EHCA_BMASK_IBM(0, 0)
-#define WC_IMM_DATA EHCA_BMASK_IBM(1, 1)
-#define WC_GRH_PRESENT EHCA_BMASK_IBM(2, 2)
-#define WC_SE_BIT EHCA_BMASK_IBM(3, 3)
-#define WC_STATUS_ERROR_BIT 0x80000000
-#define WC_STATUS_REMOTE_ERROR_FLAGS 0x0000F800
-#define WC_STATUS_PURGE_BIT 0x10
-#define WC_SEND_RECEIVE_BIT 0x80
-
-struct ehca_cqe {
- u64 work_request_id;
- u8 optype;
- u8 w_completion_flags;
- u16 reserved1;
- u32 nr_bytes_transferred;
- u32 immediate_data;
- u32 local_qp_number;
- u8 freed_resource_count;
- u8 service_level;
- u16 wqe_count;
- u32 qp_token;
- u32 qkey_ee_token;
- u32 remote_qp_number;
- u16 dlid;
- u16 rlid;
- u16 reserved2;
- u16 pkey_index;
- u32 cqe_timestamp;
- u32 wqe_timestamp;
- u8 wqe_timestamp_valid;
- u8 reserved3;
- u8 reserved4;
- u8 cqe_flags;
- u32 status;
-};
-
-struct ehca_eqe {
- u64 entry;
-};
-
-struct ehca_mrte {
- u64 starting_va;
- u64 length; /* length of memory region in bytes*/
- u32 pd;
- u8 key_instance;
- u8 pagesize;
- u8 mr_control;
- u8 local_remote_access_ctrl;
- u8 reserved[0x20 - 0x18];
- u64 at_pointer[4];
-};
-#endif /*_EHCA_QES_H_*/
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * QP functions
- *
- * Authors: Joachim Fenkes <fenkes@de.ibm.com>
- * Stefan Roscher <stefan.roscher@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-#include "hipz_fns.h"
-
-static struct kmem_cache *qp_cache;
-
-/*
- * attributes not supported by query qp
- */
-#define QP_ATTR_QUERY_NOT_SUPPORTED (IB_QP_ACCESS_FLAGS | \
- IB_QP_EN_SQD_ASYNC_NOTIFY)
-
-/*
- * ehca (internal) qp state values
- */
-enum ehca_qp_state {
- EHCA_QPS_RESET = 1,
- EHCA_QPS_INIT = 2,
- EHCA_QPS_RTR = 3,
- EHCA_QPS_RTS = 5,
- EHCA_QPS_SQD = 6,
- EHCA_QPS_SQE = 8,
- EHCA_QPS_ERR = 128
-};
-
-/*
- * qp state transitions as defined by IB Arch Rel 1.1 page 431
- */
-enum ib_qp_statetrans {
- IB_QPST_ANY2RESET,
- IB_QPST_ANY2ERR,
- IB_QPST_RESET2INIT,
- IB_QPST_INIT2RTR,
- IB_QPST_INIT2INIT,
- IB_QPST_RTR2RTS,
- IB_QPST_RTS2SQD,
- IB_QPST_RTS2RTS,
- IB_QPST_SQD2RTS,
- IB_QPST_SQE2RTS,
- IB_QPST_SQD2SQD,
- IB_QPST_MAX /* nr of transitions, this must be last!!! */
-};
-
-/*
- * ib2ehca_qp_state maps IB to ehca qp_state
- * returns ehca qp state corresponding to given ib qp state
- */
-static inline enum ehca_qp_state ib2ehca_qp_state(enum ib_qp_state ib_qp_state)
-{
- switch (ib_qp_state) {
- case IB_QPS_RESET:
- return EHCA_QPS_RESET;
- case IB_QPS_INIT:
- return EHCA_QPS_INIT;
- case IB_QPS_RTR:
- return EHCA_QPS_RTR;
- case IB_QPS_RTS:
- return EHCA_QPS_RTS;
- case IB_QPS_SQD:
- return EHCA_QPS_SQD;
- case IB_QPS_SQE:
- return EHCA_QPS_SQE;
- case IB_QPS_ERR:
- return EHCA_QPS_ERR;
- default:
- ehca_gen_err("invalid ib_qp_state=%x", ib_qp_state);
- return -EINVAL;
- }
-}
-
-/*
- * ehca2ib_qp_state maps ehca to IB qp_state
- * returns ib qp state corresponding to given ehca qp state
- */
-static inline enum ib_qp_state ehca2ib_qp_state(enum ehca_qp_state
- ehca_qp_state)
-{
- switch (ehca_qp_state) {
- case EHCA_QPS_RESET:
- return IB_QPS_RESET;
- case EHCA_QPS_INIT:
- return IB_QPS_INIT;
- case EHCA_QPS_RTR:
- return IB_QPS_RTR;
- case EHCA_QPS_RTS:
- return IB_QPS_RTS;
- case EHCA_QPS_SQD:
- return IB_QPS_SQD;
- case EHCA_QPS_SQE:
- return IB_QPS_SQE;
- case EHCA_QPS_ERR:
- return IB_QPS_ERR;
- default:
- ehca_gen_err("invalid ehca_qp_state=%x", ehca_qp_state);
- return -EINVAL;
- }
-}
-
-/*
- * ehca_qp_type used as index for req_attr and opt_attr of
- * struct ehca_modqp_statetrans
- */
-enum ehca_qp_type {
- QPT_RC = 0,
- QPT_UC = 1,
- QPT_UD = 2,
- QPT_SQP = 3,
- QPT_MAX
-};
-
-/*
- * ib2ehcaqptype maps Ib to ehca qp_type
- * returns ehca qp type corresponding to ib qp type
- */
-static inline enum ehca_qp_type ib2ehcaqptype(enum ib_qp_type ibqptype)
-{
- switch (ibqptype) {
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- return QPT_SQP;
- case IB_QPT_RC:
- return QPT_RC;
- case IB_QPT_UC:
- return QPT_UC;
- case IB_QPT_UD:
- return QPT_UD;
- default:
- ehca_gen_err("Invalid ibqptype=%x", ibqptype);
- return -EINVAL;
- }
-}
-
-static inline enum ib_qp_statetrans get_modqp_statetrans(int ib_fromstate,
- int ib_tostate)
-{
- int index = -EINVAL;
- switch (ib_tostate) {
- case IB_QPS_RESET:
- index = IB_QPST_ANY2RESET;
- break;
- case IB_QPS_INIT:
- switch (ib_fromstate) {
- case IB_QPS_RESET:
- index = IB_QPST_RESET2INIT;
- break;
- case IB_QPS_INIT:
- index = IB_QPST_INIT2INIT;
- break;
- }
- break;
- case IB_QPS_RTR:
- if (ib_fromstate == IB_QPS_INIT)
- index = IB_QPST_INIT2RTR;
- break;
- case IB_QPS_RTS:
- switch (ib_fromstate) {
- case IB_QPS_RTR:
- index = IB_QPST_RTR2RTS;
- break;
- case IB_QPS_RTS:
- index = IB_QPST_RTS2RTS;
- break;
- case IB_QPS_SQD:
- index = IB_QPST_SQD2RTS;
- break;
- case IB_QPS_SQE:
- index = IB_QPST_SQE2RTS;
- break;
- }
- break;
- case IB_QPS_SQD:
- if (ib_fromstate == IB_QPS_RTS)
- index = IB_QPST_RTS2SQD;
- break;
- case IB_QPS_SQE:
- break;
- case IB_QPS_ERR:
- index = IB_QPST_ANY2ERR;
- break;
- default:
- break;
- }
- return index;
-}
-
-/*
- * ibqptype2servicetype returns hcp service type corresponding to given
- * ib qp type used by create_qp()
- */
-static inline int ibqptype2servicetype(enum ib_qp_type ibqptype)
-{
- switch (ibqptype) {
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- return ST_UD;
- case IB_QPT_RC:
- return ST_RC;
- case IB_QPT_UC:
- return ST_UC;
- case IB_QPT_UD:
- return ST_UD;
- case IB_QPT_RAW_IPV6:
- return -EINVAL;
- case IB_QPT_RAW_ETHERTYPE:
- return -EINVAL;
- default:
- ehca_gen_err("Invalid ibqptype=%x", ibqptype);
- return -EINVAL;
- }
-}
-
-/*
- * init userspace queue info from ipz_queue data
- */
-static inline void queue2resp(struct ipzu_queue_resp *resp,
- struct ipz_queue *queue)
-{
- resp->qe_size = queue->qe_size;
- resp->act_nr_of_sg = queue->act_nr_of_sg;
- resp->queue_length = queue->queue_length;
- resp->pagesize = queue->pagesize;
- resp->toggle_state = queue->toggle_state;
- resp->offset = queue->offset;
-}
-
-/*
- * init_qp_queue initializes/constructs r/squeue and registers queue pages.
- */
-static inline int init_qp_queue(struct ehca_shca *shca,
- struct ehca_pd *pd,
- struct ehca_qp *my_qp,
- struct ipz_queue *queue,
- int q_type,
- u64 expected_hret,
- struct ehca_alloc_queue_parms *parms,
- int wqe_size)
-{
- int ret, cnt, ipz_rc, nr_q_pages;
- void *vpage;
- u64 rpage, h_ret;
- struct ib_device *ib_dev = &shca->ib_device;
- struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle;
-
- if (!parms->queue_size)
- return 0;
-
- if (parms->is_small) {
- nr_q_pages = 1;
- ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
- 128 << parms->page_size,
- wqe_size, parms->act_nr_sges, 1);
- } else {
- nr_q_pages = parms->queue_size;
- ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
- EHCA_PAGESIZE, wqe_size,
- parms->act_nr_sges, 0);
- }
-
- if (!ipz_rc) {
- ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%i",
- ipz_rc);
- return -EBUSY;
- }
-
- /* register queue pages */
- for (cnt = 0; cnt < nr_q_pages; cnt++) {
- vpage = ipz_qpageit_get_inc(queue);
- if (!vpage) {
- ehca_err(ib_dev, "ipz_qpageit_get_inc() "
- "failed p_vpage= %p", vpage);
- ret = -EINVAL;
- goto init_qp_queue1;
- }
- rpage = __pa(vpage);
-
- h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
- my_qp->ipz_qp_handle,
- NULL, 0, q_type,
- rpage, parms->is_small ? 0 : 1,
- my_qp->galpas.kernel);
- if (cnt == (nr_q_pages - 1)) { /* last page! */
- if (h_ret != expected_hret) {
- ehca_err(ib_dev, "hipz_qp_register_rpage() "
- "h_ret=%lli", h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto init_qp_queue1;
- }
- vpage = ipz_qpageit_get_inc(&my_qp->ipz_rqueue);
- if (vpage) {
- ehca_err(ib_dev, "ipz_qpageit_get_inc() "
- "should not succeed vpage=%p", vpage);
- ret = -EINVAL;
- goto init_qp_queue1;
- }
- } else {
- if (h_ret != H_PAGE_REGISTERED) {
- ehca_err(ib_dev, "hipz_qp_register_rpage() "
- "h_ret=%lli", h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto init_qp_queue1;
- }
- }
- }
-
- ipz_qeit_reset(queue);
-
- return 0;
-
-init_qp_queue1:
- ipz_queue_dtor(pd, queue);
- return ret;
-}
-
-static inline int ehca_calc_wqe_size(int act_nr_sge, int is_llqp)
-{
- if (is_llqp)
- return 128 << act_nr_sge;
- else
- return offsetof(struct ehca_wqe,
- u.nud.sg_list[act_nr_sge]);
-}
-
-static void ehca_determine_small_queue(struct ehca_alloc_queue_parms *queue,
- int req_nr_sge, int is_llqp)
-{
- u32 wqe_size, q_size;
- int act_nr_sge = req_nr_sge;
-
- if (!is_llqp)
- /* round up #SGEs so WQE size is a power of 2 */
- for (act_nr_sge = 4; act_nr_sge <= 252;
- act_nr_sge = 4 + 2 * act_nr_sge)
- if (act_nr_sge >= req_nr_sge)
- break;
-
- wqe_size = ehca_calc_wqe_size(act_nr_sge, is_llqp);
- q_size = wqe_size * (queue->max_wr + 1);
-
- if (q_size <= 512)
- queue->page_size = 2;
- else if (q_size <= 1024)
- queue->page_size = 3;
- else
- queue->page_size = 0;
-
- queue->is_small = (queue->page_size != 0);
-}
-
-/* needs to be called with cq->spinlock held */
-void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq)
-{
- struct list_head *list, *node;
-
- /* TODO: support low latency QPs */
- if (qp->ext_type == EQPT_LLQP)
- return;
-
- if (on_sq) {
- list = &qp->send_cq->sqp_err_list;
- node = &qp->sq_err_node;
- } else {
- list = &qp->recv_cq->rqp_err_list;
- node = &qp->rq_err_node;
- }
-
- if (list_empty(node))
- list_add_tail(node, list);
-
- return;
-}
-
-static void del_from_err_list(struct ehca_cq *cq, struct list_head *node)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&cq->spinlock, flags);
-
- if (!list_empty(node))
- list_del_init(node);
-
- spin_unlock_irqrestore(&cq->spinlock, flags);
-}
-
-static void reset_queue_map(struct ehca_queue_map *qmap)
-{
- int i;
-
- qmap->tail = qmap->entries - 1;
- qmap->left_to_poll = 0;
- qmap->next_wqe_idx = 0;
- for (i = 0; i < qmap->entries; i++) {
- qmap->map[i].reported = 1;
- qmap->map[i].cqe_req = 0;
- }
-}
-
-/*
- * Create an ib_qp struct that is either a QP or an SRQ, depending on
- * the value of the is_srq parameter. If init_attr and srq_init_attr share
- * fields, the field out of init_attr is used.
- */
-static struct ehca_qp *internal_create_qp(
- struct ib_pd *pd,
- struct ib_qp_init_attr *init_attr,
- struct ib_srq_init_attr *srq_init_attr,
- struct ib_udata *udata, int is_srq)
-{
- struct ehca_qp *my_qp, *my_srq = NULL;
- struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
- ib_device);
- struct ib_ucontext *context = NULL;
- u64 h_ret;
- int is_llqp = 0, has_srq = 0, is_user = 0;
- int qp_type, max_send_sge, max_recv_sge, ret;
-
- /* h_call's out parameters */
- struct ehca_alloc_qp_parms parms;
- u32 swqe_size = 0, rwqe_size = 0, ib_qp_num;
- unsigned long flags;
-
- if (!atomic_add_unless(&shca->num_qps, 1, shca->max_num_qps)) {
- ehca_err(pd->device, "Unable to create QP, max number of %i "
- "QPs reached.", shca->max_num_qps);
- ehca_err(pd->device, "To increase the maximum number of QPs "
- "use the number_of_qps module parameter.\n");
- return ERR_PTR(-ENOSPC);
- }
-
- if (init_attr->create_flags) {
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- memset(&parms, 0, sizeof(parms));
- qp_type = init_attr->qp_type;
-
- if (init_attr->sq_sig_type != IB_SIGNAL_REQ_WR &&
- init_attr->sq_sig_type != IB_SIGNAL_ALL_WR) {
- ehca_err(pd->device, "init_attr->sg_sig_type=%x not allowed",
- init_attr->sq_sig_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- /* save LLQP info */
- if (qp_type & 0x80) {
- is_llqp = 1;
- parms.ext_type = EQPT_LLQP;
- parms.ll_comp_flags = qp_type & LLQP_COMP_MASK;
- }
- qp_type &= 0x1F;
- init_attr->qp_type &= 0x1F;
-
- /* handle SRQ base QPs */
- if (init_attr->srq) {
- my_srq = container_of(init_attr->srq, struct ehca_qp, ib_srq);
-
- if (qp_type == IB_QPT_UC) {
- ehca_err(pd->device, "UC with SRQ not supported");
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- has_srq = 1;
- parms.ext_type = EQPT_SRQBASE;
- parms.srq_qpn = my_srq->real_qp_num;
- }
-
- if (is_llqp && has_srq) {
- ehca_err(pd->device, "LLQPs can't have an SRQ");
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- /* handle SRQs */
- if (is_srq) {
- parms.ext_type = EQPT_SRQ;
- parms.srq_limit = srq_init_attr->attr.srq_limit;
- if (init_attr->cap.max_recv_sge > 3) {
- ehca_err(pd->device, "no more than three SGEs "
- "supported for SRQ pd=%p max_sge=%x",
- pd, init_attr->cap.max_recv_sge);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- }
-
- /* check QP type */
- if (qp_type != IB_QPT_UD &&
- qp_type != IB_QPT_UC &&
- qp_type != IB_QPT_RC &&
- qp_type != IB_QPT_SMI &&
- qp_type != IB_QPT_GSI) {
- ehca_err(pd->device, "wrong QP Type=%x", qp_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- if (is_llqp) {
- switch (qp_type) {
- case IB_QPT_RC:
- if ((init_attr->cap.max_send_wr > 255) ||
- (init_attr->cap.max_recv_wr > 255)) {
- ehca_err(pd->device,
- "Invalid Number of max_sq_wr=%x "
- "or max_rq_wr=%x for RC LLQP",
- init_attr->cap.max_send_wr,
- init_attr->cap.max_recv_wr);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- break;
- case IB_QPT_UD:
- if (!EHCA_BMASK_GET(HCA_CAP_UD_LL_QP, shca->hca_cap)) {
- ehca_err(pd->device, "UD LLQP not supported "
- "by this adapter");
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-ENOSYS);
- }
- if (!(init_attr->cap.max_send_sge <= 5
- && init_attr->cap.max_send_sge >= 1
- && init_attr->cap.max_recv_sge <= 5
- && init_attr->cap.max_recv_sge >= 1)) {
- ehca_err(pd->device,
- "Invalid Number of max_send_sge=%x "
- "or max_recv_sge=%x for UD LLQP",
- init_attr->cap.max_send_sge,
- init_attr->cap.max_recv_sge);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- } else if (init_attr->cap.max_send_wr > 255) {
- ehca_err(pd->device,
- "Invalid Number of "
- "max_send_wr=%x for UD QP_TYPE=%x",
- init_attr->cap.max_send_wr, qp_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- break;
- default:
- ehca_err(pd->device, "unsupported LL QP Type=%x",
- qp_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- } else {
- int max_sge = (qp_type == IB_QPT_UD || qp_type == IB_QPT_SMI
- || qp_type == IB_QPT_GSI) ? 250 : 252;
-
- if (init_attr->cap.max_send_sge > max_sge
- || init_attr->cap.max_recv_sge > max_sge) {
- ehca_err(pd->device, "Invalid number of SGEs requested "
- "send_sge=%x recv_sge=%x max_sge=%x",
- init_attr->cap.max_send_sge,
- init_attr->cap.max_recv_sge, max_sge);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- }
-
- my_qp = kmem_cache_zalloc(qp_cache, GFP_KERNEL);
- if (!my_qp) {
- ehca_err(pd->device, "pd=%p not enough memory to alloc qp", pd);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-ENOMEM);
- }
-
- if (pd->uobject && udata) {
- is_user = 1;
- context = pd->uobject->context;
- }
-
- atomic_set(&my_qp->nr_events, 0);
- init_waitqueue_head(&my_qp->wait_completion);
- spin_lock_init(&my_qp->spinlock_s);
- spin_lock_init(&my_qp->spinlock_r);
- my_qp->qp_type = qp_type;
- my_qp->ext_type = parms.ext_type;
- my_qp->state = IB_QPS_RESET;
-
- if (init_attr->recv_cq)
- my_qp->recv_cq =
- container_of(init_attr->recv_cq, struct ehca_cq, ib_cq);
- if (init_attr->send_cq)
- my_qp->send_cq =
- container_of(init_attr->send_cq, struct ehca_cq, ib_cq);
-
- idr_preload(GFP_KERNEL);
- write_lock_irqsave(&ehca_qp_idr_lock, flags);
-
- ret = idr_alloc(&ehca_qp_idr, my_qp, 0, 0x2000000, GFP_NOWAIT);
- if (ret >= 0)
- my_qp->token = ret;
-
- write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
- idr_preload_end();
- if (ret < 0) {
- if (ret == -ENOSPC) {
- ret = -EINVAL;
- ehca_err(pd->device, "Invalid number of qp");
- } else {
- ret = -ENOMEM;
- ehca_err(pd->device, "Can't allocate new idr entry.");
- }
- goto create_qp_exit0;
- }
-
- if (has_srq)
- parms.srq_token = my_qp->token;
-
- parms.servicetype = ibqptype2servicetype(qp_type);
- if (parms.servicetype < 0) {
- ret = -EINVAL;
- ehca_err(pd->device, "Invalid qp_type=%x", qp_type);
- goto create_qp_exit1;
- }
-
- /* Always signal by WQE so we can hide circ. WQEs */
- parms.sigtype = HCALL_SIGT_BY_WQE;
-
- /* UD_AV CIRCUMVENTION */
- max_send_sge = init_attr->cap.max_send_sge;
- max_recv_sge = init_attr->cap.max_recv_sge;
- if (parms.servicetype == ST_UD && !is_llqp) {
- max_send_sge += 2;
- max_recv_sge += 2;
- }
-
- parms.token = my_qp->token;
- parms.eq_handle = shca->eq.ipz_eq_handle;
- parms.pd = my_pd->fw_pd;
- if (my_qp->send_cq)
- parms.send_cq_handle = my_qp->send_cq->ipz_cq_handle;
- if (my_qp->recv_cq)
- parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle;
-
- parms.squeue.max_wr = init_attr->cap.max_send_wr;
- parms.rqueue.max_wr = init_attr->cap.max_recv_wr;
- parms.squeue.max_sge = max_send_sge;
- parms.rqueue.max_sge = max_recv_sge;
-
- /* RC QPs need one more SWQE for unsolicited ack circumvention */
- if (qp_type == IB_QPT_RC)
- parms.squeue.max_wr++;
-
- if (EHCA_BMASK_GET(HCA_CAP_MINI_QP, shca->hca_cap)) {
- if (HAS_SQ(my_qp))
- ehca_determine_small_queue(
- &parms.squeue, max_send_sge, is_llqp);
- if (HAS_RQ(my_qp))
- ehca_determine_small_queue(
- &parms.rqueue, max_recv_sge, is_llqp);
- parms.qp_storage =
- (parms.squeue.is_small || parms.rqueue.is_small);
- }
-
- h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms, is_user);
- if (h_ret != H_SUCCESS) {
- ehca_err(pd->device, "h_alloc_resource_qp() failed h_ret=%lli",
- h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto create_qp_exit1;
- }
-
- ib_qp_num = my_qp->real_qp_num = parms.real_qp_num;
- my_qp->ipz_qp_handle = parms.qp_handle;
- my_qp->galpas = parms.galpas;
-
- swqe_size = ehca_calc_wqe_size(parms.squeue.act_nr_sges, is_llqp);
- rwqe_size = ehca_calc_wqe_size(parms.rqueue.act_nr_sges, is_llqp);
-
- switch (qp_type) {
- case IB_QPT_RC:
- if (is_llqp) {
- parms.squeue.act_nr_sges = 1;
- parms.rqueue.act_nr_sges = 1;
- }
- /* hide the extra WQE */
- parms.squeue.act_nr_wqes--;
- break;
- case IB_QPT_UD:
- case IB_QPT_GSI:
- case IB_QPT_SMI:
- /* UD circumvention */
- if (is_llqp) {
- parms.squeue.act_nr_sges = 1;
- parms.rqueue.act_nr_sges = 1;
- } else {
- parms.squeue.act_nr_sges -= 2;
- parms.rqueue.act_nr_sges -= 2;
- }
-
- if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) {
- parms.squeue.act_nr_wqes = init_attr->cap.max_send_wr;
- parms.rqueue.act_nr_wqes = init_attr->cap.max_recv_wr;
- parms.squeue.act_nr_sges = init_attr->cap.max_send_sge;
- parms.rqueue.act_nr_sges = init_attr->cap.max_recv_sge;
- ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1;
- }
-
- break;
-
- default:
- break;
- }
-
- /* initialize r/squeue and register queue pages */
- if (HAS_SQ(my_qp)) {
- ret = init_qp_queue(
- shca, my_pd, my_qp, &my_qp->ipz_squeue, 0,
- HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS,
- &parms.squeue, swqe_size);
- if (ret) {
- ehca_err(pd->device, "Couldn't initialize squeue "
- "and pages ret=%i", ret);
- goto create_qp_exit2;
- }
-
- if (!is_user) {
- my_qp->sq_map.entries = my_qp->ipz_squeue.queue_length /
- my_qp->ipz_squeue.qe_size;
- my_qp->sq_map.map = vmalloc(my_qp->sq_map.entries *
- sizeof(struct ehca_qmap_entry));
- if (!my_qp->sq_map.map) {
- ehca_err(pd->device, "Couldn't allocate squeue "
- "map ret=%i", ret);
- goto create_qp_exit3;
- }
- INIT_LIST_HEAD(&my_qp->sq_err_node);
- /* to avoid the generation of bogus flush CQEs */
- reset_queue_map(&my_qp->sq_map);
- }
- }
-
- if (HAS_RQ(my_qp)) {
- ret = init_qp_queue(
- shca, my_pd, my_qp, &my_qp->ipz_rqueue, 1,
- H_SUCCESS, &parms.rqueue, rwqe_size);
- if (ret) {
- ehca_err(pd->device, "Couldn't initialize rqueue "
- "and pages ret=%i", ret);
- goto create_qp_exit4;
- }
- if (!is_user) {
- my_qp->rq_map.entries = my_qp->ipz_rqueue.queue_length /
- my_qp->ipz_rqueue.qe_size;
- my_qp->rq_map.map = vmalloc(my_qp->rq_map.entries *
- sizeof(struct ehca_qmap_entry));
- if (!my_qp->rq_map.map) {
- ehca_err(pd->device, "Couldn't allocate squeue "
- "map ret=%i", ret);
- goto create_qp_exit5;
- }
- INIT_LIST_HEAD(&my_qp->rq_err_node);
- /* to avoid the generation of bogus flush CQEs */
- reset_queue_map(&my_qp->rq_map);
- }
- } else if (init_attr->srq && !is_user) {
- /* this is a base QP, use the queue map of the SRQ */
- my_qp->rq_map = my_srq->rq_map;
- INIT_LIST_HEAD(&my_qp->rq_err_node);
-
- my_qp->ipz_rqueue = my_srq->ipz_rqueue;
- }
-
- if (is_srq) {
- my_qp->ib_srq.pd = &my_pd->ib_pd;
- my_qp->ib_srq.device = my_pd->ib_pd.device;
-
- my_qp->ib_srq.srq_context = init_attr->qp_context;
- my_qp->ib_srq.event_handler = init_attr->event_handler;
- } else {
- my_qp->ib_qp.qp_num = ib_qp_num;
- my_qp->ib_qp.pd = &my_pd->ib_pd;
- my_qp->ib_qp.device = my_pd->ib_pd.device;
-
- my_qp->ib_qp.recv_cq = init_attr->recv_cq;
- my_qp->ib_qp.send_cq = init_attr->send_cq;
-
- my_qp->ib_qp.qp_type = qp_type;
- my_qp->ib_qp.srq = init_attr->srq;
-
- my_qp->ib_qp.qp_context = init_attr->qp_context;
- my_qp->ib_qp.event_handler = init_attr->event_handler;
- }
-
- init_attr->cap.max_inline_data = 0; /* not supported yet */
- init_attr->cap.max_recv_sge = parms.rqueue.act_nr_sges;
- init_attr->cap.max_recv_wr = parms.rqueue.act_nr_wqes;
- init_attr->cap.max_send_sge = parms.squeue.act_nr_sges;
- init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes;
- my_qp->init_attr = *init_attr;
-
- if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
- shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
- &my_qp->ib_qp;
- if (ehca_nr_ports < 0) {
- /* alloc array to cache subsequent modify qp parms
- * for autodetect mode
- */
- my_qp->mod_qp_parm =
- kzalloc(EHCA_MOD_QP_PARM_MAX *
- sizeof(*my_qp->mod_qp_parm),
- GFP_KERNEL);
- if (!my_qp->mod_qp_parm) {
- ehca_err(pd->device,
- "Could not alloc mod_qp_parm");
- goto create_qp_exit5;
- }
- }
- }
-
- /* NOTE: define_apq0() not supported yet */
- if (qp_type == IB_QPT_GSI) {
- h_ret = ehca_define_sqp(shca, my_qp, init_attr);
- if (h_ret != H_SUCCESS) {
- kfree(my_qp->mod_qp_parm);
- my_qp->mod_qp_parm = NULL;
- /* the QP pointer is no longer valid */
- shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
- NULL;
- ret = ehca2ib_return_code(h_ret);
- goto create_qp_exit6;
- }
- }
-
- if (my_qp->send_cq) {
- ret = ehca_cq_assign_qp(my_qp->send_cq, my_qp);
- if (ret) {
- ehca_err(pd->device,
- "Couldn't assign qp to send_cq ret=%i", ret);
- goto create_qp_exit7;
- }
- }
-
- /* copy queues, galpa data to user space */
- if (context && udata) {
- struct ehca_create_qp_resp resp;
- memset(&resp, 0, sizeof(resp));
-
- resp.qp_num = my_qp->real_qp_num;
- resp.token = my_qp->token;
- resp.qp_type = my_qp->qp_type;
- resp.ext_type = my_qp->ext_type;
- resp.qkey = my_qp->qkey;
- resp.real_qp_num = my_qp->real_qp_num;
-
- if (HAS_SQ(my_qp))
- queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue);
- if (HAS_RQ(my_qp))
- queue2resp(&resp.ipz_rqueue, &my_qp->ipz_rqueue);
- resp.fw_handle_ofs = (u32)
- (my_qp->galpas.user.fw_handle & (PAGE_SIZE - 1));
-
- if (ib_copy_to_udata(udata, &resp, sizeof resp)) {
- ehca_err(pd->device, "Copy to udata failed");
- ret = -EINVAL;
- goto create_qp_exit8;
- }
- }
-
- return my_qp;
-
-create_qp_exit8:
- ehca_cq_unassign_qp(my_qp->send_cq, my_qp->real_qp_num);
-
-create_qp_exit7:
- kfree(my_qp->mod_qp_parm);
-
-create_qp_exit6:
- if (HAS_RQ(my_qp) && !is_user)
- vfree(my_qp->rq_map.map);
-
-create_qp_exit5:
- if (HAS_RQ(my_qp))
- ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
-
-create_qp_exit4:
- if (HAS_SQ(my_qp) && !is_user)
- vfree(my_qp->sq_map.map);
-
-create_qp_exit3:
- if (HAS_SQ(my_qp))
- ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
-
-create_qp_exit2:
- hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
-
-create_qp_exit1:
- write_lock_irqsave(&ehca_qp_idr_lock, flags);
- idr_remove(&ehca_qp_idr, my_qp->token);
- write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
-
-create_qp_exit0:
- kmem_cache_free(qp_cache, my_qp);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(ret);
-}
-
-struct ib_qp *ehca_create_qp(struct ib_pd *pd,
- struct ib_qp_init_attr *qp_init_attr,
- struct ib_udata *udata)
-{
- struct ehca_qp *ret;
-
- ret = internal_create_qp(pd, qp_init_attr, NULL, udata, 0);
- return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp;
-}
-
-static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
- struct ib_uobject *uobject);
-
-struct ib_srq *ehca_create_srq(struct ib_pd *pd,
- struct ib_srq_init_attr *srq_init_attr,
- struct ib_udata *udata)
-{
- struct ib_qp_init_attr qp_init_attr;
- struct ehca_qp *my_qp;
- struct ib_srq *ret;
- struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
- ib_device);
- struct hcp_modify_qp_control_block *mqpcb;
- u64 hret, update_mask;
-
- if (srq_init_attr->srq_type != IB_SRQT_BASIC)
- return ERR_PTR(-ENOSYS);
-
- /* For common attributes, internal_create_qp() takes its info
- * out of qp_init_attr, so copy all common attrs there.
- */
- memset(&qp_init_attr, 0, sizeof(qp_init_attr));
- qp_init_attr.event_handler = srq_init_attr->event_handler;
- qp_init_attr.qp_context = srq_init_attr->srq_context;
- qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
- qp_init_attr.qp_type = IB_QPT_RC;
- qp_init_attr.cap.max_recv_wr = srq_init_attr->attr.max_wr;
- qp_init_attr.cap.max_recv_sge = srq_init_attr->attr.max_sge;
-
- my_qp = internal_create_qp(pd, &qp_init_attr, srq_init_attr, udata, 1);
- if (IS_ERR(my_qp))
- return (struct ib_srq *)my_qp;
-
- /* copy back return values */
- srq_init_attr->attr.max_wr = qp_init_attr.cap.max_recv_wr;
- srq_init_attr->attr.max_sge = 3;
-
- /* drive SRQ into RTR state */
- mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!mqpcb) {
- ehca_err(pd->device, "Could not get zeroed page for mqpcb "
- "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
- ret = ERR_PTR(-ENOMEM);
- goto create_srq1;
- }
-
- mqpcb->qp_state = EHCA_QPS_INIT;
- mqpcb->prim_phys_port = 1;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
- hret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
- if (hret != H_SUCCESS) {
- ehca_err(pd->device, "Could not modify SRQ to INIT "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, hret);
- goto create_srq2;
- }
-
- mqpcb->qp_enable = 1;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
- hret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
- if (hret != H_SUCCESS) {
- ehca_err(pd->device, "Could not enable SRQ "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, hret);
- goto create_srq2;
- }
-
- mqpcb->qp_state = EHCA_QPS_RTR;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
- hret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
- if (hret != H_SUCCESS) {
- ehca_err(pd->device, "Could not modify SRQ to RTR "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, hret);
- goto create_srq2;
- }
-
- ehca_free_fw_ctrlblock(mqpcb);
-
- return &my_qp->ib_srq;
-
-create_srq2:
- ret = ERR_PTR(ehca2ib_return_code(hret));
- ehca_free_fw_ctrlblock(mqpcb);
-
-create_srq1:
- internal_destroy_qp(pd->device, my_qp, my_qp->ib_srq.uobject);
-
- return ret;
-}
-
-/*
- * prepare_sqe_rts called by internal_modify_qp() at trans sqe -> rts
- * set purge bit of bad wqe and subsequent wqes to avoid reentering sqe
- * returns total number of bad wqes in bad_wqe_cnt
- */
-static int prepare_sqe_rts(struct ehca_qp *my_qp, struct ehca_shca *shca,
- int *bad_wqe_cnt)
-{
- u64 h_ret;
- struct ipz_queue *squeue;
- void *bad_send_wqe_p, *bad_send_wqe_v;
- u64 q_ofs;
- struct ehca_wqe *wqe;
- int qp_num = my_qp->ib_qp.qp_num;
-
- /* get send wqe pointer */
- h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle, &my_qp->pf,
- &bad_send_wqe_p, NULL, 2);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_h_disable_and_get_wqe() failed"
- " ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, qp_num, h_ret);
- return ehca2ib_return_code(h_ret);
- }
- bad_send_wqe_p = (void *)((u64)bad_send_wqe_p & (~(1L << 63)));
- ehca_dbg(&shca->ib_device, "qp_num=%x bad_send_wqe_p=%p",
- qp_num, bad_send_wqe_p);
- /* convert wqe pointer to vadr */
- bad_send_wqe_v = __va((u64)bad_send_wqe_p);
- if (ehca_debug_level >= 2)
- ehca_dmp(bad_send_wqe_v, 32, "qp_num=%x bad_wqe", qp_num);
- squeue = &my_qp->ipz_squeue;
- if (ipz_queue_abs_to_offset(squeue, (u64)bad_send_wqe_p, &q_ofs)) {
- ehca_err(&shca->ib_device, "failed to get wqe offset qp_num=%x"
- " bad_send_wqe_p=%p", qp_num, bad_send_wqe_p);
- return -EFAULT;
- }
-
- /* loop sets wqe's purge bit */
- wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
- *bad_wqe_cnt = 0;
- while (wqe->optype != 0xff && wqe->wqef != 0xff) {
- if (ehca_debug_level >= 2)
- ehca_dmp(wqe, 32, "qp_num=%x wqe", qp_num);
- wqe->nr_of_data_seg = 0; /* suppress data access */
- wqe->wqef = WQEF_PURGE; /* WQE to be purged */
- q_ofs = ipz_queue_advance_offset(squeue, q_ofs);
- wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
- *bad_wqe_cnt = (*bad_wqe_cnt)+1;
- }
- /*
- * bad wqe will be reprocessed and ignored when pol_cq() is called,
- * i.e. nr of wqes with flush error status is one less
- */
- ehca_dbg(&shca->ib_device, "qp_num=%x flusherr_wqe_cnt=%x",
- qp_num, (*bad_wqe_cnt)-1);
- wqe->wqef = 0;
-
- return 0;
-}
-
-static int calc_left_cqes(u64 wqe_p, struct ipz_queue *ipz_queue,
- struct ehca_queue_map *qmap)
-{
- void *wqe_v;
- u64 q_ofs;
- u32 wqe_idx;
- unsigned int tail_idx;
-
- /* convert real to abs address */
- wqe_p = wqe_p & (~(1UL << 63));
-
- wqe_v = __va(wqe_p);
-
- if (ipz_queue_abs_to_offset(ipz_queue, wqe_p, &q_ofs)) {
- ehca_gen_err("Invalid offset for calculating left cqes "
- "wqe_p=%#llx wqe_v=%p\n", wqe_p, wqe_v);
- return -EFAULT;
- }
-
- tail_idx = next_index(qmap->tail, qmap->entries);
- wqe_idx = q_ofs / ipz_queue->qe_size;
-
- /* check all processed wqes, whether a cqe is requested or not */
- while (tail_idx != wqe_idx) {
- if (qmap->map[tail_idx].cqe_req)
- qmap->left_to_poll++;
- tail_idx = next_index(tail_idx, qmap->entries);
- }
- /* save index in queue, where we have to start flushing */
- qmap->next_wqe_idx = wqe_idx;
- return 0;
-}
-
-static int check_for_left_cqes(struct ehca_qp *my_qp, struct ehca_shca *shca)
-{
- u64 h_ret;
- void *send_wqe_p, *recv_wqe_p;
- int ret;
- unsigned long flags;
- int qp_num = my_qp->ib_qp.qp_num;
-
- /* this hcall is not supported on base QPs */
- if (my_qp->ext_type != EQPT_SRQBASE) {
- /* get send and receive wqe pointer */
- h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle, &my_qp->pf,
- &send_wqe_p, &recv_wqe_p, 4);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "disable_and_get_wqe() "
- "failed ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, qp_num, h_ret);
- return ehca2ib_return_code(h_ret);
- }
-
- /*
- * acquire lock to ensure that nobody is polling the cq which
- * could mean that the qmap->tail pointer is in an
- * inconsistent state.
- */
- spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
- ret = calc_left_cqes((u64)send_wqe_p, &my_qp->ipz_squeue,
- &my_qp->sq_map);
- spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
- if (ret)
- return ret;
-
-
- spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
- ret = calc_left_cqes((u64)recv_wqe_p, &my_qp->ipz_rqueue,
- &my_qp->rq_map);
- spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
- if (ret)
- return ret;
- } else {
- spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
- my_qp->sq_map.left_to_poll = 0;
- my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
- my_qp->sq_map.entries);
- spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
-
- spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
- my_qp->rq_map.left_to_poll = 0;
- my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
- my_qp->rq_map.entries);
- spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
- }
-
- /* this assures flush cqes being generated only for pending wqes */
- if ((my_qp->sq_map.left_to_poll == 0) &&
- (my_qp->rq_map.left_to_poll == 0)) {
- spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
- ehca_add_to_err_list(my_qp, 1);
- spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
-
- if (HAS_RQ(my_qp)) {
- spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
- ehca_add_to_err_list(my_qp, 0);
- spin_unlock_irqrestore(&my_qp->recv_cq->spinlock,
- flags);
- }
- }
-
- return 0;
-}
-
-/*
- * internal_modify_qp with circumvention to handle aqp0 properly
- * smi_reset2init indicates if this is an internal reset-to-init-call for
- * smi. This flag must always be zero if called from ehca_modify_qp()!
- * This internal func was intorduced to avoid recursion of ehca_modify_qp()!
- */
-static int internal_modify_qp(struct ib_qp *ibqp,
- struct ib_qp_attr *attr,
- int attr_mask, int smi_reset2init)
-{
- enum ib_qp_state qp_cur_state, qp_new_state;
- int cnt, qp_attr_idx, ret = 0;
- enum ib_qp_statetrans statetrans;
- struct hcp_modify_qp_control_block *mqpcb;
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca =
- container_of(ibqp->pd->device, struct ehca_shca, ib_device);
- u64 update_mask;
- u64 h_ret;
- int bad_wqe_cnt = 0;
- int is_user = 0;
- int squeue_locked = 0;
- unsigned long flags = 0;
-
- /* do query_qp to obtain current attr values */
- mqpcb = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
- if (!mqpcb) {
- ehca_err(ibqp->device, "Could not get zeroed page for mqpcb "
- "ehca_qp=%p qp_num=%x ", my_qp, ibqp->qp_num);
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- mqpcb, my_qp->galpas.kernel);
- if (h_ret != H_SUCCESS) {
- ehca_err(ibqp->device, "hipz_h_query_qp() failed "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, ibqp->qp_num, h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto modify_qp_exit1;
- }
- if (ibqp->uobject)
- is_user = 1;
-
- qp_cur_state = ehca2ib_qp_state(mqpcb->qp_state);
-
- if (qp_cur_state == -EINVAL) { /* invalid qp state */
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid current ehca_qp_state=%x "
- "ehca_qp=%p qp_num=%x",
- mqpcb->qp_state, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
- /*
- * circumvention to set aqp0 initial state to init
- * as expected by IB spec
- */
- if (smi_reset2init == 0 &&
- ibqp->qp_type == IB_QPT_SMI &&
- qp_cur_state == IB_QPS_RESET &&
- (attr_mask & IB_QP_STATE) &&
- attr->qp_state == IB_QPS_INIT) { /* RESET -> INIT */
- struct ib_qp_attr smiqp_attr = {
- .qp_state = IB_QPS_INIT,
- .port_num = my_qp->init_attr.port_num,
- .pkey_index = 0,
- .qkey = 0
- };
- int smiqp_attr_mask = IB_QP_STATE | IB_QP_PORT |
- IB_QP_PKEY_INDEX | IB_QP_QKEY;
- int smirc = internal_modify_qp(
- ibqp, &smiqp_attr, smiqp_attr_mask, 1);
- if (smirc) {
- ehca_err(ibqp->device, "SMI RESET -> INIT failed. "
- "ehca_modify_qp() rc=%i", smirc);
- ret = H_PARAMETER;
- goto modify_qp_exit1;
- }
- qp_cur_state = IB_QPS_INIT;
- ehca_dbg(ibqp->device, "SMI RESET -> INIT succeeded");
- }
- /* is transmitted current state equal to "real" current state */
- if ((attr_mask & IB_QP_CUR_STATE) &&
- qp_cur_state != attr->cur_qp_state) {
- ret = -EINVAL;
- ehca_err(ibqp->device,
- "Invalid IB_QP_CUR_STATE attr->curr_qp_state=%x <>"
- " actual cur_qp_state=%x. ehca_qp=%p qp_num=%x",
- attr->cur_qp_state, qp_cur_state, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- ehca_dbg(ibqp->device, "ehca_qp=%p qp_num=%x current qp_state=%x "
- "new qp_state=%x attribute_mask=%x",
- my_qp, ibqp->qp_num, qp_cur_state, attr->qp_state, attr_mask);
-
- qp_new_state = attr_mask & IB_QP_STATE ? attr->qp_state : qp_cur_state;
- if (!smi_reset2init &&
- !ib_modify_qp_is_ok(qp_cur_state, qp_new_state, ibqp->qp_type,
- attr_mask, IB_LINK_LAYER_UNSPECIFIED)) {
- ret = -EINVAL;
- ehca_err(ibqp->device,
- "Invalid qp transition new_state=%x cur_state=%x "
- "ehca_qp=%p qp_num=%x attr_mask=%x", qp_new_state,
- qp_cur_state, my_qp, ibqp->qp_num, attr_mask);
- goto modify_qp_exit1;
- }
-
- mqpcb->qp_state = ib2ehca_qp_state(qp_new_state);
- if (mqpcb->qp_state)
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
- else {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid new qp state=%x "
- "ehca_qp=%p qp_num=%x",
- qp_new_state, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- /* retrieve state transition struct to get req and opt attrs */
- statetrans = get_modqp_statetrans(qp_cur_state, qp_new_state);
- if (statetrans < 0) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "<INVALID STATE CHANGE> qp_cur_state=%x "
- "new_qp_state=%x State_xsition=%x ehca_qp=%p "
- "qp_num=%x", qp_cur_state, qp_new_state,
- statetrans, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- qp_attr_idx = ib2ehcaqptype(ibqp->qp_type);
-
- if (qp_attr_idx < 0) {
- ret = qp_attr_idx;
- ehca_err(ibqp->device,
- "Invalid QP type=%x ehca_qp=%p qp_num=%x",
- ibqp->qp_type, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- ehca_dbg(ibqp->device,
- "ehca_qp=%p qp_num=%x <VALID STATE CHANGE> qp_state_xsit=%x",
- my_qp, ibqp->qp_num, statetrans);
-
- /* eHCA2 rev2 and higher require the SEND_GRH_FLAG to be set
- * in non-LL UD QPs.
- */
- if ((my_qp->qp_type == IB_QPT_UD) &&
- (my_qp->ext_type != EQPT_LLQP) &&
- (statetrans == IB_QPST_INIT2RTR) &&
- (shca->hw_level >= 0x22)) {
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
- mqpcb->send_grh_flag = 1;
- }
-
- /* sqe -> rts: set purge bit of bad wqe before actual trans */
- if ((my_qp->qp_type == IB_QPT_UD ||
- my_qp->qp_type == IB_QPT_GSI ||
- my_qp->qp_type == IB_QPT_SMI) &&
- statetrans == IB_QPST_SQE2RTS) {
- /* mark next free wqe if kernel */
- if (!ibqp->uobject) {
- struct ehca_wqe *wqe;
- /* lock send queue */
- spin_lock_irqsave(&my_qp->spinlock_s, flags);
- squeue_locked = 1;
- /* mark next free wqe */
- wqe = (struct ehca_wqe *)
- ipz_qeit_get(&my_qp->ipz_squeue);
- wqe->optype = wqe->wqef = 0xff;
- ehca_dbg(ibqp->device, "qp_num=%x next_free_wqe=%p",
- ibqp->qp_num, wqe);
- }
- ret = prepare_sqe_rts(my_qp, shca, &bad_wqe_cnt);
- if (ret) {
- ehca_err(ibqp->device, "prepare_sqe_rts() failed "
- "ehca_qp=%p qp_num=%x ret=%i",
- my_qp, ibqp->qp_num, ret);
- goto modify_qp_exit2;
- }
- }
-
- /*
- * enable RDMA_Atomic_Control if reset->init und reliable con
- * this is necessary since gen2 does not provide that flag,
- * but pHyp requires it
- */
- if (statetrans == IB_QPST_RESET2INIT &&
- (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_UC)) {
- mqpcb->rdma_atomic_ctrl = 3;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RDMA_ATOMIC_CTRL, 1);
- }
- /* circ. pHyp requires #RDMA/Atomic Resp Res for UC INIT -> RTR */
- if (statetrans == IB_QPST_INIT2RTR &&
- (ibqp->qp_type == IB_QPT_UC) &&
- !(attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)) {
- mqpcb->rdma_nr_atomic_resp_res = 1; /* default to 1 */
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
- }
-
- if (attr_mask & IB_QP_PKEY_INDEX) {
- if (attr->pkey_index >= 16) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid pkey_index=%x. "
- "ehca_qp=%p qp_num=%x max_pkey_index=f",
- attr->pkey_index, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
- mqpcb->prim_p_key_idx = attr->pkey_index;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_P_KEY_IDX, 1);
- }
- if (attr_mask & IB_QP_PORT) {
- struct ehca_sport *sport;
- struct ehca_qp *aqp1;
- if (attr->port_num < 1 || attr->port_num > shca->num_ports) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid port=%x. "
- "ehca_qp=%p qp_num=%x num_ports=%x",
- attr->port_num, my_qp, ibqp->qp_num,
- shca->num_ports);
- goto modify_qp_exit2;
- }
- sport = &shca->sport[attr->port_num - 1];
- if (!sport->ibqp_sqp[IB_QPT_GSI]) {
- /* should not occur */
- ret = -EFAULT;
- ehca_err(ibqp->device, "AQP1 was not created for "
- "port=%x", attr->port_num);
- goto modify_qp_exit2;
- }
- aqp1 = container_of(sport->ibqp_sqp[IB_QPT_GSI],
- struct ehca_qp, ib_qp);
- if (ibqp->qp_type != IB_QPT_GSI &&
- ibqp->qp_type != IB_QPT_SMI &&
- aqp1->mod_qp_parm) {
- /*
- * firmware will reject this modify_qp() because
- * port is not activated/initialized fully
- */
- ret = -EFAULT;
- ehca_warn(ibqp->device, "Couldn't modify qp port=%x: "
- "either port is being activated (try again) "
- "or cabling issue", attr->port_num);
- goto modify_qp_exit2;
- }
- mqpcb->prim_phys_port = attr->port_num;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_PHYS_PORT, 1);
- }
- if (attr_mask & IB_QP_QKEY) {
- mqpcb->qkey = attr->qkey;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_QKEY, 1);
- }
- if (attr_mask & IB_QP_AV) {
- mqpcb->dlid = attr->ah_attr.dlid;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DLID, 1);
- mqpcb->source_path_bits = attr->ah_attr.src_path_bits;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS, 1);
- mqpcb->service_level = attr->ah_attr.sl;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL, 1);
-
- if (ehca_calc_ipd(shca, mqpcb->prim_phys_port,
- attr->ah_attr.static_rate,
- &mqpcb->max_static_rate)) {
- ret = -EINVAL;
- goto modify_qp_exit2;
- }
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE, 1);
-
- /*
- * Always supply the GRH flag, even if it's zero, to give the
- * hypervisor a clear "yes" or "no" instead of a "perhaps"
- */
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
-
- /*
- * only if GRH is TRUE we might consider SOURCE_GID_IDX
- * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
- */
- if (attr->ah_attr.ah_flags == IB_AH_GRH) {
- mqpcb->send_grh_flag = 1;
-
- mqpcb->source_gid_idx = attr->ah_attr.grh.sgid_index;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX, 1);
-
- for (cnt = 0; cnt < 16; cnt++)
- mqpcb->dest_gid.byte[cnt] =
- attr->ah_attr.grh.dgid.raw[cnt];
-
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_GID, 1);
- mqpcb->flow_label = attr->ah_attr.grh.flow_label;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL, 1);
- mqpcb->hop_limit = attr->ah_attr.grh.hop_limit;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT, 1);
- mqpcb->traffic_class = attr->ah_attr.grh.traffic_class;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS, 1);
- }
- }
-
- if (attr_mask & IB_QP_PATH_MTU) {
- /* store ld(MTU) */
- my_qp->mtu_shift = attr->path_mtu + 7;
- mqpcb->path_mtu = attr->path_mtu;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PATH_MTU, 1);
- }
- if (attr_mask & IB_QP_TIMEOUT) {
- mqpcb->timeout = attr->timeout;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT, 1);
- }
- if (attr_mask & IB_QP_RETRY_CNT) {
- mqpcb->retry_count = attr->retry_cnt;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT, 1);
- }
- if (attr_mask & IB_QP_RNR_RETRY) {
- mqpcb->rnr_retry_count = attr->rnr_retry;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT, 1);
- }
- if (attr_mask & IB_QP_RQ_PSN) {
- mqpcb->receive_psn = attr->rq_psn;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RECEIVE_PSN, 1);
- }
- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
- mqpcb->rdma_nr_atomic_resp_res = attr->max_dest_rd_atomic < 3 ?
- attr->max_dest_rd_atomic : 2;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
- }
- if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
- mqpcb->rdma_atomic_outst_dest_qp = attr->max_rd_atomic < 3 ?
- attr->max_rd_atomic : 2;
- update_mask |=
- EHCA_BMASK_SET
- (MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP, 1);
- }
- if (attr_mask & IB_QP_ALT_PATH) {
- if (attr->alt_port_num < 1
- || attr->alt_port_num > shca->num_ports) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid alt_port=%x. "
- "ehca_qp=%p qp_num=%x num_ports=%x",
- attr->alt_port_num, my_qp, ibqp->qp_num,
- shca->num_ports);
- goto modify_qp_exit2;
- }
- mqpcb->alt_phys_port = attr->alt_port_num;
-
- if (attr->alt_pkey_index >= 16) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid alt_pkey_index=%x. "
- "ehca_qp=%p qp_num=%x max_pkey_index=f",
- attr->pkey_index, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
- mqpcb->alt_p_key_idx = attr->alt_pkey_index;
-
- mqpcb->timeout_al = attr->alt_timeout;
- mqpcb->dlid_al = attr->alt_ah_attr.dlid;
- mqpcb->source_path_bits_al = attr->alt_ah_attr.src_path_bits;
- mqpcb->service_level_al = attr->alt_ah_attr.sl;
-
- if (ehca_calc_ipd(shca, mqpcb->alt_phys_port,
- attr->alt_ah_attr.static_rate,
- &mqpcb->max_static_rate_al)) {
- ret = -EINVAL;
- goto modify_qp_exit2;
- }
-
- /* OpenIB doesn't support alternate retry counts - copy them */
- mqpcb->retry_count_al = mqpcb->retry_count;
- mqpcb->rnr_retry_count_al = mqpcb->rnr_retry_count;
-
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_ALT_PHYS_PORT, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_ALT_P_KEY_IDX, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_DLID_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT_AL, 1);
-
- /*
- * Always supply the GRH flag, even if it's zero, to give the
- * hypervisor a clear "yes" or "no" instead of a "perhaps"
- */
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG_AL, 1);
-
- /*
- * only if GRH is TRUE we might consider SOURCE_GID_IDX
- * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
- */
- if (attr->alt_ah_attr.ah_flags == IB_AH_GRH) {
- mqpcb->send_grh_flag_al = 1;
-
- for (cnt = 0; cnt < 16; cnt++)
- mqpcb->dest_gid_al.byte[cnt] =
- attr->alt_ah_attr.grh.dgid.raw[cnt];
- mqpcb->source_gid_idx_al =
- attr->alt_ah_attr.grh.sgid_index;
- mqpcb->flow_label_al = attr->alt_ah_attr.grh.flow_label;
- mqpcb->hop_limit_al = attr->alt_ah_attr.grh.hop_limit;
- mqpcb->traffic_class_al =
- attr->alt_ah_attr.grh.traffic_class;
-
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_DEST_GID_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT_AL, 1) |
- EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS_AL, 1);
- }
- }
-
- if (attr_mask & IB_QP_MIN_RNR_TIMER) {
- mqpcb->min_rnr_nak_timer_field = attr->min_rnr_timer;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD, 1);
- }
-
- if (attr_mask & IB_QP_SQ_PSN) {
- mqpcb->send_psn = attr->sq_psn;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_PSN, 1);
- }
-
- if (attr_mask & IB_QP_DEST_QPN) {
- mqpcb->dest_qp_nr = attr->dest_qp_num;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_QP_NR, 1);
- }
-
- if (attr_mask & IB_QP_PATH_MIG_STATE) {
- if (attr->path_mig_state != IB_MIG_REARM
- && attr->path_mig_state != IB_MIG_MIGRATED) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid mig_state=%x",
- attr->path_mig_state);
- goto modify_qp_exit2;
- }
- mqpcb->path_migration_state = attr->path_mig_state + 1;
- if (attr->path_mig_state == IB_MIG_REARM)
- my_qp->mig_armed = 1;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_PATH_MIGRATION_STATE, 1);
- }
-
- if (attr_mask & IB_QP_CAP) {
- mqpcb->max_nr_outst_send_wr = attr->cap.max_send_wr+1;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_SEND_WR, 1);
- mqpcb->max_nr_outst_recv_wr = attr->cap.max_recv_wr+1;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_RECV_WR, 1);
- /* no support for max_send/recv_sge yet */
- }
-
- if (ehca_debug_level >= 2)
- ehca_dmp(mqpcb, 4*70, "qp_num=%x", ibqp->qp_num);
-
- h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(ibqp->device, "hipz_h_modify_qp() failed h_ret=%lli "
- "ehca_qp=%p qp_num=%x", h_ret, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
-
- if ((my_qp->qp_type == IB_QPT_UD ||
- my_qp->qp_type == IB_QPT_GSI ||
- my_qp->qp_type == IB_QPT_SMI) &&
- statetrans == IB_QPST_SQE2RTS) {
- /* doorbell to reprocessing wqes */
- iosync(); /* serialize GAL register access */
- hipz_update_sqa(my_qp, bad_wqe_cnt-1);
- ehca_gen_dbg("doorbell for %x wqes", bad_wqe_cnt);
- }
-
- if (statetrans == IB_QPST_RESET2INIT ||
- statetrans == IB_QPST_INIT2INIT) {
- mqpcb->qp_enable = 1;
- mqpcb->qp_state = EHCA_QPS_INIT;
- update_mask = 0;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
-
- h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb,
- my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(ibqp->device, "ENABLE in context of "
- "RESET_2_INIT failed! Maybe you didn't get "
- "a LID h_ret=%lli ehca_qp=%p qp_num=%x",
- h_ret, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
- }
- if ((qp_new_state == IB_QPS_ERR) && (qp_cur_state != IB_QPS_ERR)
- && !is_user) {
- ret = check_for_left_cqes(my_qp, shca);
- if (ret)
- goto modify_qp_exit2;
- }
-
- if (statetrans == IB_QPST_ANY2RESET) {
- ipz_qeit_reset(&my_qp->ipz_rqueue);
- ipz_qeit_reset(&my_qp->ipz_squeue);
-
- if (qp_cur_state == IB_QPS_ERR && !is_user) {
- del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
-
- if (HAS_RQ(my_qp))
- del_from_err_list(my_qp->recv_cq,
- &my_qp->rq_err_node);
- }
- if (!is_user)
- reset_queue_map(&my_qp->sq_map);
-
- if (HAS_RQ(my_qp) && !is_user)
- reset_queue_map(&my_qp->rq_map);
- }
-
- if (attr_mask & IB_QP_QKEY)
- my_qp->qkey = attr->qkey;
-
-modify_qp_exit2:
- if (squeue_locked) { /* this means: sqe -> rts */
- spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
- my_qp->sqerr_purgeflag = 1;
- }
-
-modify_qp_exit1:
- ehca_free_fw_ctrlblock(mqpcb);
-
- return ret;
-}
-
-int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
- struct ib_udata *udata)
-{
- int ret = 0;
-
- struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
- ib_device);
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
-
- /* The if-block below caches qp_attr to be modified for GSI and SMI
- * qps during the initialization by ib_mad. When the respective port
- * is activated, ie we got an event PORT_ACTIVE, we'll replay the
- * cached modify calls sequence, see ehca_recover_sqs() below.
- * Why that is required:
- * 1) If one port is connected, older code requires that port one
- * to be connected and module option nr_ports=1 to be given by
- * user, which is very inconvenient for end user.
- * 2) Firmware accepts modify_qp() only if respective port has become
- * active. Older code had a wait loop of 30sec create_qp()/
- * define_aqp1(), which is not appropriate in practice. This
- * code now removes that wait loop, see define_aqp1(), and always
- * reports all ports to ib_mad resp. users. Only activated ports
- * will then usable for the users.
- */
- if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) {
- int port = my_qp->init_attr.port_num;
- struct ehca_sport *sport = &shca->sport[port - 1];
- unsigned long flags;
- spin_lock_irqsave(&sport->mod_sqp_lock, flags);
- /* cache qp_attr only during init */
- if (my_qp->mod_qp_parm) {
- struct ehca_mod_qp_parm *p;
- if (my_qp->mod_qp_parm_idx >= EHCA_MOD_QP_PARM_MAX) {
- ehca_err(&shca->ib_device,
- "mod_qp_parm overflow state=%x port=%x"
- " type=%x", attr->qp_state,
- my_qp->init_attr.port_num,
- ibqp->qp_type);
- spin_unlock_irqrestore(&sport->mod_sqp_lock,
- flags);
- return -EINVAL;
- }
- p = &my_qp->mod_qp_parm[my_qp->mod_qp_parm_idx];
- p->mask = attr_mask;
- p->attr = *attr;
- my_qp->mod_qp_parm_idx++;
- ehca_dbg(&shca->ib_device,
- "Saved qp_attr for state=%x port=%x type=%x",
- attr->qp_state, my_qp->init_attr.port_num,
- ibqp->qp_type);
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
- goto out;
- }
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
- }
-
- ret = internal_modify_qp(ibqp, attr, attr_mask, 0);
-
-out:
- if ((ret == 0) && (attr_mask & IB_QP_STATE))
- my_qp->state = attr->qp_state;
-
- return ret;
-}
-
-void ehca_recover_sqp(struct ib_qp *sqp)
-{
- struct ehca_qp *my_sqp = container_of(sqp, struct ehca_qp, ib_qp);
- int port = my_sqp->init_attr.port_num;
- struct ib_qp_attr attr;
- struct ehca_mod_qp_parm *qp_parm;
- int i, qp_parm_idx, ret;
- unsigned long flags, wr_cnt;
-
- if (!my_sqp->mod_qp_parm)
- return;
- ehca_dbg(sqp->device, "SQP port=%x qp_num=%x", port, sqp->qp_num);
-
- qp_parm = my_sqp->mod_qp_parm;
- qp_parm_idx = my_sqp->mod_qp_parm_idx;
- for (i = 0; i < qp_parm_idx; i++) {
- attr = qp_parm[i].attr;
- ret = internal_modify_qp(sqp, &attr, qp_parm[i].mask, 0);
- if (ret) {
- ehca_err(sqp->device, "Could not modify SQP port=%x "
- "qp_num=%x ret=%x", port, sqp->qp_num, ret);
- goto free_qp_parm;
- }
- ehca_dbg(sqp->device, "SQP port=%x qp_num=%x in state=%x",
- port, sqp->qp_num, attr.qp_state);
- }
-
- /* re-trigger posted recv wrs */
- wr_cnt = my_sqp->ipz_rqueue.current_q_offset /
- my_sqp->ipz_rqueue.qe_size;
- if (wr_cnt) {
- spin_lock_irqsave(&my_sqp->spinlock_r, flags);
- hipz_update_rqa(my_sqp, wr_cnt);
- spin_unlock_irqrestore(&my_sqp->spinlock_r, flags);
- ehca_dbg(sqp->device, "doorbell port=%x qp_num=%x wr_cnt=%lx",
- port, sqp->qp_num, wr_cnt);
- }
-
-free_qp_parm:
- kfree(qp_parm);
- /* this prevents subsequent calls to modify_qp() to cache qp_attr */
- my_sqp->mod_qp_parm = NULL;
-}
-
-int ehca_query_qp(struct ib_qp *qp,
- struct ib_qp_attr *qp_attr,
- int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
-{
- struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca = container_of(qp->device, struct ehca_shca,
- ib_device);
- struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
- struct hcp_modify_qp_control_block *qpcb;
- int cnt, ret = 0;
- u64 h_ret;
-
- if (qp_attr_mask & QP_ATTR_QUERY_NOT_SUPPORTED) {
- ehca_err(qp->device, "Invalid attribute mask "
- "ehca_qp=%p qp_num=%x qp_attr_mask=%x ",
- my_qp, qp->qp_num, qp_attr_mask);
- return -EINVAL;
- }
-
- qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!qpcb) {
- ehca_err(qp->device, "Out of memory for qpcb "
- "ehca_qp=%p qp_num=%x", my_qp, qp->qp_num);
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_qp(adapter_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- qpcb, my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(qp->device, "hipz_h_query_qp() failed "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, qp->qp_num, h_ret);
- goto query_qp_exit1;
- }
-
- qp_attr->cur_qp_state = ehca2ib_qp_state(qpcb->qp_state);
- qp_attr->qp_state = qp_attr->cur_qp_state;
-
- if (qp_attr->cur_qp_state == -EINVAL) {
- ret = -EINVAL;
- ehca_err(qp->device, "Got invalid ehca_qp_state=%x "
- "ehca_qp=%p qp_num=%x",
- qpcb->qp_state, my_qp, qp->qp_num);
- goto query_qp_exit1;
- }
-
- if (qp_attr->qp_state == IB_QPS_SQD)
- qp_attr->sq_draining = 1;
-
- qp_attr->qkey = qpcb->qkey;
- qp_attr->path_mtu = qpcb->path_mtu;
- qp_attr->path_mig_state = qpcb->path_migration_state - 1;
- qp_attr->rq_psn = qpcb->receive_psn;
- qp_attr->sq_psn = qpcb->send_psn;
- qp_attr->min_rnr_timer = qpcb->min_rnr_nak_timer_field;
- qp_attr->cap.max_send_wr = qpcb->max_nr_outst_send_wr-1;
- qp_attr->cap.max_recv_wr = qpcb->max_nr_outst_recv_wr-1;
- /* UD_AV CIRCUMVENTION */
- if (my_qp->qp_type == IB_QPT_UD) {
- qp_attr->cap.max_send_sge =
- qpcb->actual_nr_sges_in_sq_wqe - 2;
- qp_attr->cap.max_recv_sge =
- qpcb->actual_nr_sges_in_rq_wqe - 2;
- } else {
- qp_attr->cap.max_send_sge =
- qpcb->actual_nr_sges_in_sq_wqe;
- qp_attr->cap.max_recv_sge =
- qpcb->actual_nr_sges_in_rq_wqe;
- }
-
- qp_attr->cap.max_inline_data = my_qp->sq_max_inline_data_size;
- qp_attr->dest_qp_num = qpcb->dest_qp_nr;
-
- qp_attr->pkey_index = qpcb->prim_p_key_idx;
- qp_attr->port_num = qpcb->prim_phys_port;
- qp_attr->timeout = qpcb->timeout;
- qp_attr->retry_cnt = qpcb->retry_count;
- qp_attr->rnr_retry = qpcb->rnr_retry_count;
-
- qp_attr->alt_pkey_index = qpcb->alt_p_key_idx;
- qp_attr->alt_port_num = qpcb->alt_phys_port;
- qp_attr->alt_timeout = qpcb->timeout_al;
-
- qp_attr->max_dest_rd_atomic = qpcb->rdma_nr_atomic_resp_res;
- qp_attr->max_rd_atomic = qpcb->rdma_atomic_outst_dest_qp;
-
- /* primary av */
- qp_attr->ah_attr.sl = qpcb->service_level;
-
- if (qpcb->send_grh_flag) {
- qp_attr->ah_attr.ah_flags = IB_AH_GRH;
- }
-
- qp_attr->ah_attr.static_rate = qpcb->max_static_rate;
- qp_attr->ah_attr.dlid = qpcb->dlid;
- qp_attr->ah_attr.src_path_bits = qpcb->source_path_bits;
- qp_attr->ah_attr.port_num = qp_attr->port_num;
-
- /* primary GRH */
- qp_attr->ah_attr.grh.traffic_class = qpcb->traffic_class;
- qp_attr->ah_attr.grh.hop_limit = qpcb->hop_limit;
- qp_attr->ah_attr.grh.sgid_index = qpcb->source_gid_idx;
- qp_attr->ah_attr.grh.flow_label = qpcb->flow_label;
-
- for (cnt = 0; cnt < 16; cnt++)
- qp_attr->ah_attr.grh.dgid.raw[cnt] =
- qpcb->dest_gid.byte[cnt];
-
- /* alternate AV */
- qp_attr->alt_ah_attr.sl = qpcb->service_level_al;
- if (qpcb->send_grh_flag_al) {
- qp_attr->alt_ah_attr.ah_flags = IB_AH_GRH;
- }
-
- qp_attr->alt_ah_attr.static_rate = qpcb->max_static_rate_al;
- qp_attr->alt_ah_attr.dlid = qpcb->dlid_al;
- qp_attr->alt_ah_attr.src_path_bits = qpcb->source_path_bits_al;
-
- /* alternate GRH */
- qp_attr->alt_ah_attr.grh.traffic_class = qpcb->traffic_class_al;
- qp_attr->alt_ah_attr.grh.hop_limit = qpcb->hop_limit_al;
- qp_attr->alt_ah_attr.grh.sgid_index = qpcb->source_gid_idx_al;
- qp_attr->alt_ah_attr.grh.flow_label = qpcb->flow_label_al;
-
- for (cnt = 0; cnt < 16; cnt++)
- qp_attr->alt_ah_attr.grh.dgid.raw[cnt] =
- qpcb->dest_gid_al.byte[cnt];
-
- /* return init attributes given in ehca_create_qp */
- if (qp_init_attr)
- *qp_init_attr = my_qp->init_attr;
-
- if (ehca_debug_level >= 2)
- ehca_dmp(qpcb, 4*70, "qp_num=%x", qp->qp_num);
-
-query_qp_exit1:
- ehca_free_fw_ctrlblock(qpcb);
-
- return ret;
-}
-
-int ehca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
- enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
-{
- struct ehca_qp *my_qp =
- container_of(ibsrq, struct ehca_qp, ib_srq);
- struct ehca_shca *shca =
- container_of(ibsrq->pd->device, struct ehca_shca, ib_device);
- struct hcp_modify_qp_control_block *mqpcb;
- u64 update_mask;
- u64 h_ret;
- int ret = 0;
-
- mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!mqpcb) {
- ehca_err(ibsrq->device, "Could not get zeroed page for mqpcb "
- "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
- return -ENOMEM;
- }
-
- update_mask = 0;
- if (attr_mask & IB_SRQ_LIMIT) {
- attr_mask &= ~IB_SRQ_LIMIT;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_CURR_SRQ_LIMIT, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG, 1);
- mqpcb->curr_srq_limit = attr->srq_limit;
- mqpcb->qp_aff_asyn_ev_log_reg =
- EHCA_BMASK_SET(QPX_AAELOG_RESET_SRQ_LIMIT, 1);
- }
-
- /* by now, all bits in attr_mask should have been cleared */
- if (attr_mask) {
- ehca_err(ibsrq->device, "invalid attribute mask bits set "
- "attr_mask=%x", attr_mask);
- ret = -EINVAL;
- goto modify_srq_exit0;
- }
-
- if (ehca_debug_level >= 2)
- ehca_dmp(mqpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
-
- h_ret = hipz_h_modify_qp(shca->ipz_hca_handle, my_qp->ipz_qp_handle,
- NULL, update_mask, mqpcb,
- my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(ibsrq->device, "hipz_h_modify_qp() failed h_ret=%lli "
- "ehca_qp=%p qp_num=%x",
- h_ret, my_qp, my_qp->real_qp_num);
- }
-
-modify_srq_exit0:
- ehca_free_fw_ctrlblock(mqpcb);
-
- return ret;
-}
-
-int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr)
-{
- struct ehca_qp *my_qp = container_of(srq, struct ehca_qp, ib_srq);
- struct ehca_shca *shca = container_of(srq->device, struct ehca_shca,
- ib_device);
- struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
- struct hcp_modify_qp_control_block *qpcb;
- int ret = 0;
- u64 h_ret;
-
- qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!qpcb) {
- ehca_err(srq->device, "Out of memory for qpcb "
- "ehca_qp=%p qp_num=%x", my_qp, my_qp->real_qp_num);
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_qp(adapter_handle, my_qp->ipz_qp_handle,
- NULL, qpcb, my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(srq->device, "hipz_h_query_qp() failed "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, h_ret);
- goto query_srq_exit1;
- }
-
- srq_attr->max_wr = qpcb->max_nr_outst_recv_wr - 1;
- srq_attr->max_sge = 3;
- srq_attr->srq_limit = qpcb->curr_srq_limit;
-
- if (ehca_debug_level >= 2)
- ehca_dmp(qpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
-
-query_srq_exit1:
- ehca_free_fw_ctrlblock(qpcb);
-
- return ret;
-}
-
-static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
- struct ib_uobject *uobject)
-{
- struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device);
- struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
- ib_pd);
- struct ehca_sport *sport = &shca->sport[my_qp->init_attr.port_num - 1];
- u32 qp_num = my_qp->real_qp_num;
- int ret;
- u64 h_ret;
- u8 port_num;
- int is_user = 0;
- enum ib_qp_type qp_type;
- unsigned long flags;
-
- if (uobject) {
- is_user = 1;
- if (my_qp->mm_count_galpa ||
- my_qp->mm_count_rqueue || my_qp->mm_count_squeue) {
- ehca_err(dev, "Resources still referenced in "
- "user space qp_num=%x", qp_num);
- return -EINVAL;
- }
- }
-
- if (my_qp->send_cq) {
- ret = ehca_cq_unassign_qp(my_qp->send_cq, qp_num);
- if (ret) {
- ehca_err(dev, "Couldn't unassign qp from "
- "send_cq ret=%i qp_num=%x cq_num=%x", ret,
- qp_num, my_qp->send_cq->cq_number);
- return ret;
- }
- }
-
- write_lock_irqsave(&ehca_qp_idr_lock, flags);
- idr_remove(&ehca_qp_idr, my_qp->token);
- write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
-
- /*
- * SRQs will never get into an error list and do not have a recv_cq,
- * so we need to skip them here.
- */
- if (HAS_RQ(my_qp) && !IS_SRQ(my_qp) && !is_user)
- del_from_err_list(my_qp->recv_cq, &my_qp->rq_err_node);
-
- if (HAS_SQ(my_qp) && !is_user)
- del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
-
- /* now wait until all pending events have completed */
- wait_event(my_qp->wait_completion, !atomic_read(&my_qp->nr_events));
-
- h_ret = hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
- if (h_ret != H_SUCCESS) {
- ehca_err(dev, "hipz_h_destroy_qp() failed h_ret=%lli "
- "ehca_qp=%p qp_num=%x", h_ret, my_qp, qp_num);
- return ehca2ib_return_code(h_ret);
- }
-
- port_num = my_qp->init_attr.port_num;
- qp_type = my_qp->init_attr.qp_type;
-
- if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
- spin_lock_irqsave(&sport->mod_sqp_lock, flags);
- kfree(my_qp->mod_qp_parm);
- my_qp->mod_qp_parm = NULL;
- shca->sport[port_num - 1].ibqp_sqp[qp_type] = NULL;
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
- }
-
- /* no support for IB_QPT_SMI yet */
- if (qp_type == IB_QPT_GSI) {
- struct ib_event event;
- ehca_info(dev, "device %s: port %x is inactive.",
- shca->ib_device.name, port_num);
- event.device = &shca->ib_device;
- event.event = IB_EVENT_PORT_ERR;
- event.element.port_num = port_num;
- shca->sport[port_num - 1].port_state = IB_PORT_DOWN;
- ib_dispatch_event(&event);
- }
-
- if (HAS_RQ(my_qp)) {
- ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
- if (!is_user)
- vfree(my_qp->rq_map.map);
- }
- if (HAS_SQ(my_qp)) {
- ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
- if (!is_user)
- vfree(my_qp->sq_map.map);
- }
- kmem_cache_free(qp_cache, my_qp);
- atomic_dec(&shca->num_qps);
- return 0;
-}
-
-int ehca_destroy_qp(struct ib_qp *qp)
-{
- return internal_destroy_qp(qp->device,
- container_of(qp, struct ehca_qp, ib_qp),
- qp->uobject);
-}
-
-int ehca_destroy_srq(struct ib_srq *srq)
-{
- return internal_destroy_qp(srq->device,
- container_of(srq, struct ehca_qp, ib_srq),
- srq->uobject);
-}
-
-int ehca_init_qp_cache(void)
-{
- qp_cache = kmem_cache_create("ehca_cache_qp",
- sizeof(struct ehca_qp), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!qp_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_qp_cache(void)
-{
- if (qp_cache)
- kmem_cache_destroy(qp_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * post_send/recv, poll_cq, req_notify
- *
- * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-#include "hipz_fns.h"
-
-/* in RC traffic, insert an empty RDMA READ every this many packets */
-#define ACK_CIRC_THRESHOLD 2000000
-
-static u64 replace_wr_id(u64 wr_id, u16 idx)
-{
- u64 ret;
-
- ret = wr_id & ~QMAP_IDX_MASK;
- ret |= idx & QMAP_IDX_MASK;
-
- return ret;
-}
-
-static u16 get_app_wr_id(u64 wr_id)
-{
- return wr_id & QMAP_IDX_MASK;
-}
-
-static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
- struct ehca_wqe *wqe_p,
- struct ib_recv_wr *recv_wr,
- u32 rq_map_idx)
-{
- u8 cnt_ds;
- if (unlikely((recv_wr->num_sge < 0) ||
- (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
- ehca_gen_err("Invalid number of WQE SGE. "
- "num_sqe=%x max_nr_of_sg=%x",
- recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
- return -EINVAL; /* invalid SG list length */
- }
-
- /* clear wqe header until sglist */
- memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
-
- wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
- wqe_p->nr_of_data_seg = recv_wr->num_sge;
-
- for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
- wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
- recv_wr->sg_list[cnt_ds].addr;
- wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
- recv_wr->sg_list[cnt_ds].lkey;
- wqe_p->u.all_rcv.sg_list[cnt_ds].length =
- recv_wr->sg_list[cnt_ds].length;
- }
-
- if (ehca_debug_level >= 3) {
- ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
- ipz_rqueue);
- ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
- }
-
- return 0;
-}
-
-#if defined(DEBUG_GSI_SEND_WR)
-
-/* need ib_mad struct */
-#include <rdma/ib_mad.h>
-
-static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
-{
- int idx;
- int j;
- while (send_wr) {
- struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
- struct ib_sge *sge = send_wr->sg_list;
- ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
- "send_flags=%x opcode=%x", idx, send_wr->wr_id,
- send_wr->num_sge, send_wr->send_flags,
- send_wr->opcode);
- if (mad_hdr) {
- ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
- "mgmt_class=%x class_version=%x method=%x "
- "status=%x class_specific=%x tid=%lx "
- "attr_id=%x resv=%x attr_mod=%x",
- idx, mad_hdr->base_version,
- mad_hdr->mgmt_class,
- mad_hdr->class_version, mad_hdr->method,
- mad_hdr->status, mad_hdr->class_specific,
- mad_hdr->tid, mad_hdr->attr_id,
- mad_hdr->resv,
- mad_hdr->attr_mod);
- }
- for (j = 0; j < send_wr->num_sge; j++) {
- u8 *data = __va(sge->addr);
- ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
- "lkey=%x",
- idx, j, data, sge->length, sge->lkey);
- /* assume length is n*16 */
- ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
- idx, j);
- sge++;
- } /* eof for j */
- idx++;
- send_wr = send_wr->next;
- } /* eof while send_wr */
-}
-
-#endif /* DEBUG_GSI_SEND_WR */
-
-static inline int ehca_write_swqe(struct ehca_qp *qp,
- struct ehca_wqe *wqe_p,
- const struct ib_send_wr *send_wr,
- u32 sq_map_idx,
- int hidden)
-{
- u32 idx;
- u64 dma_length;
- struct ehca_av *my_av;
- u32 remote_qkey = send_wr->wr.ud.remote_qkey;
- struct ehca_qmap_entry *qmap_entry = &qp->sq_map.map[sq_map_idx];
-
- if (unlikely((send_wr->num_sge < 0) ||
- (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
- ehca_gen_err("Invalid number of WQE SGE. "
- "num_sqe=%x max_nr_of_sg=%x",
- send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
- return -EINVAL; /* invalid SG list length */
- }
-
- /* clear wqe header until sglist */
- memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
-
- wqe_p->work_request_id = replace_wr_id(send_wr->wr_id, sq_map_idx);
-
- qmap_entry->app_wr_id = get_app_wr_id(send_wr->wr_id);
- qmap_entry->reported = 0;
- qmap_entry->cqe_req = 0;
-
- switch (send_wr->opcode) {
- case IB_WR_SEND:
- case IB_WR_SEND_WITH_IMM:
- wqe_p->optype = WQE_OPTYPE_SEND;
- break;
- case IB_WR_RDMA_WRITE:
- case IB_WR_RDMA_WRITE_WITH_IMM:
- wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
- break;
- case IB_WR_RDMA_READ:
- wqe_p->optype = WQE_OPTYPE_RDMAREAD;
- break;
- default:
- ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
- return -EINVAL; /* invalid opcode */
- }
-
- wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
-
- wqe_p->wr_flag = 0;
-
- if ((send_wr->send_flags & IB_SEND_SIGNALED ||
- qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
- && !hidden) {
- wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
- qmap_entry->cqe_req = 1;
- }
-
- if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
- send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
- /* this might not work as long as HW does not support it */
- wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
- wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
- }
-
- wqe_p->nr_of_data_seg = send_wr->num_sge;
-
- switch (qp->qp_type) {
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- /* no break is intential here */
- case IB_QPT_UD:
- /* IB 1.2 spec C10-15 compliance */
- if (send_wr->wr.ud.remote_qkey & 0x80000000)
- remote_qkey = qp->qkey;
-
- wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
- wqe_p->local_ee_context_qkey = remote_qkey;
- if (unlikely(!send_wr->wr.ud.ah)) {
- ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
- return -EINVAL;
- }
- if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
- ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
- return -EINVAL;
- }
- my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
- wqe_p->u.ud_av.ud_av = my_av->av;
-
- /*
- * omitted check of IB_SEND_INLINE
- * since HW does not support it
- */
- for (idx = 0; idx < send_wr->num_sge; idx++) {
- wqe_p->u.ud_av.sg_list[idx].vaddr =
- send_wr->sg_list[idx].addr;
- wqe_p->u.ud_av.sg_list[idx].lkey =
- send_wr->sg_list[idx].lkey;
- wqe_p->u.ud_av.sg_list[idx].length =
- send_wr->sg_list[idx].length;
- } /* eof for idx */
- if (qp->qp_type == IB_QPT_SMI ||
- qp->qp_type == IB_QPT_GSI)
- wqe_p->u.ud_av.ud_av.pmtu = 1;
- if (qp->qp_type == IB_QPT_GSI) {
- wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
-#ifdef DEBUG_GSI_SEND_WR
- trace_send_wr_ud(send_wr);
-#endif /* DEBUG_GSI_SEND_WR */
- }
- break;
-
- case IB_QPT_UC:
- if (send_wr->send_flags & IB_SEND_FENCE)
- wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
- /* no break is intentional here */
- case IB_QPT_RC:
- /* TODO: atomic not implemented */
- wqe_p->u.nud.remote_virtual_address =
- send_wr->wr.rdma.remote_addr;
- wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
-
- /*
- * omitted checking of IB_SEND_INLINE
- * since HW does not support it
- */
- dma_length = 0;
- for (idx = 0; idx < send_wr->num_sge; idx++) {
- wqe_p->u.nud.sg_list[idx].vaddr =
- send_wr->sg_list[idx].addr;
- wqe_p->u.nud.sg_list[idx].lkey =
- send_wr->sg_list[idx].lkey;
- wqe_p->u.nud.sg_list[idx].length =
- send_wr->sg_list[idx].length;
- dma_length += send_wr->sg_list[idx].length;
- } /* eof idx */
- wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
-
- /* unsolicited ack circumvention */
- if (send_wr->opcode == IB_WR_RDMA_READ) {
- /* on RDMA read, switch on and reset counters */
- qp->message_count = qp->packet_count = 0;
- qp->unsol_ack_circ = 1;
- } else
- /* else estimate #packets */
- qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
-
- break;
-
- default:
- ehca_gen_err("Invalid qptype=%x", qp->qp_type);
- return -EINVAL;
- }
-
- if (ehca_debug_level >= 3) {
- ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
- ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
- }
- return 0;
-}
-
-/* map_ib_wc_status converts raw cqe_status to ib_wc_status */
-static inline void map_ib_wc_status(u32 cqe_status,
- enum ib_wc_status *wc_status)
-{
- if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
- switch (cqe_status & 0x3F) {
- case 0x01:
- case 0x21:
- *wc_status = IB_WC_LOC_LEN_ERR;
- break;
- case 0x02:
- case 0x22:
- *wc_status = IB_WC_LOC_QP_OP_ERR;
- break;
- case 0x03:
- case 0x23:
- *wc_status = IB_WC_LOC_EEC_OP_ERR;
- break;
- case 0x04:
- case 0x24:
- *wc_status = IB_WC_LOC_PROT_ERR;
- break;
- case 0x05:
- case 0x25:
- *wc_status = IB_WC_WR_FLUSH_ERR;
- break;
- case 0x06:
- *wc_status = IB_WC_MW_BIND_ERR;
- break;
- case 0x07: /* remote error - look into bits 20:24 */
- switch ((cqe_status
- & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
- case 0x0:
- /*
- * PSN Sequence Error!
- * couldn't find a matching status!
- */
- *wc_status = IB_WC_GENERAL_ERR;
- break;
- case 0x1:
- *wc_status = IB_WC_REM_INV_REQ_ERR;
- break;
- case 0x2:
- *wc_status = IB_WC_REM_ACCESS_ERR;
- break;
- case 0x3:
- *wc_status = IB_WC_REM_OP_ERR;
- break;
- case 0x4:
- *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
- break;
- }
- break;
- case 0x08:
- *wc_status = IB_WC_RETRY_EXC_ERR;
- break;
- case 0x09:
- *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
- break;
- case 0x0A:
- case 0x2D:
- *wc_status = IB_WC_REM_ABORT_ERR;
- break;
- case 0x0B:
- case 0x2E:
- *wc_status = IB_WC_INV_EECN_ERR;
- break;
- case 0x0C:
- case 0x2F:
- *wc_status = IB_WC_INV_EEC_STATE_ERR;
- break;
- case 0x0D:
- *wc_status = IB_WC_BAD_RESP_ERR;
- break;
- case 0x10:
- /* WQE purged */
- *wc_status = IB_WC_WR_FLUSH_ERR;
- break;
- default:
- *wc_status = IB_WC_FATAL_ERR;
-
- }
- } else
- *wc_status = IB_WC_SUCCESS;
-}
-
-static inline int post_one_send(struct ehca_qp *my_qp,
- struct ib_send_wr *cur_send_wr,
- int hidden)
-{
- struct ehca_wqe *wqe_p;
- int ret;
- u32 sq_map_idx;
- u64 start_offset = my_qp->ipz_squeue.current_q_offset;
-
- /* get pointer next to free WQE */
- wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
- if (unlikely(!wqe_p)) {
- /* too many posted work requests: queue overflow */
- ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
- "qp_num=%x", my_qp->ib_qp.qp_num);
- return -ENOMEM;
- }
-
- /*
- * Get the index of the WQE in the send queue. The same index is used
- * for writing into the sq_map.
- */
- sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
-
- /* write a SEND WQE into the QUEUE */
- ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
- /*
- * if something failed,
- * reset the free entry pointer to the start value
- */
- if (unlikely(ret)) {
- my_qp->ipz_squeue.current_q_offset = start_offset;
- ehca_err(my_qp->ib_qp.device, "Could not write WQE "
- "qp_num=%x", my_qp->ib_qp.qp_num);
- return -EINVAL;
- }
-
- return 0;
-}
-
-int ehca_post_send(struct ib_qp *qp,
- struct ib_send_wr *send_wr,
- struct ib_send_wr **bad_send_wr)
-{
- struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
- int wqe_cnt = 0;
- int ret = 0;
- unsigned long flags;
-
- /* Reject WR if QP is in RESET, INIT or RTR state */
- if (unlikely(my_qp->state < IB_QPS_RTS)) {
- ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
- my_qp->state, qp->qp_num);
- ret = -EINVAL;
- goto out;
- }
-
- /* LOCK the QUEUE */
- spin_lock_irqsave(&my_qp->spinlock_s, flags);
-
- /* Send an empty extra RDMA read if:
- * 1) there has been an RDMA read on this connection before
- * 2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
- * 3) we can be sure that any previous extra RDMA read has been
- * processed so we don't overflow the SQ
- */
- if (unlikely(my_qp->unsol_ack_circ &&
- my_qp->packet_count > ACK_CIRC_THRESHOLD &&
- my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
- /* insert an empty RDMA READ to fix up the remote QP state */
- struct ib_send_wr circ_wr;
- memset(&circ_wr, 0, sizeof(circ_wr));
- circ_wr.opcode = IB_WR_RDMA_READ;
- post_one_send(my_qp, &circ_wr, 1); /* ignore retcode */
- wqe_cnt++;
- ehca_dbg(qp->device, "posted circ wr qp_num=%x", qp->qp_num);
- my_qp->message_count = my_qp->packet_count = 0;
- }
-
- /* loop processes list of send reqs */
- while (send_wr) {
- ret = post_one_send(my_qp, send_wr, 0);
- if (unlikely(ret)) {
- goto post_send_exit0;
- }
- wqe_cnt++;
- send_wr = send_wr->next;
- }
-
-post_send_exit0:
- iosync(); /* serialize GAL register access */
- hipz_update_sqa(my_qp, wqe_cnt);
- if (unlikely(ret || ehca_debug_level >= 2))
- ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
- my_qp, qp->qp_num, wqe_cnt, ret);
- my_qp->message_count += wqe_cnt;
- spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
-
-out:
- if (ret)
- *bad_send_wr = send_wr;
- return ret;
-}
-
-static int internal_post_recv(struct ehca_qp *my_qp,
- struct ib_device *dev,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr)
-{
- struct ehca_wqe *wqe_p;
- int wqe_cnt = 0;
- int ret = 0;
- u32 rq_map_idx;
- unsigned long flags;
- struct ehca_qmap_entry *qmap_entry;
-
- if (unlikely(!HAS_RQ(my_qp))) {
- ehca_err(dev, "QP has no RQ ehca_qp=%p qp_num=%x ext_type=%d",
- my_qp, my_qp->real_qp_num, my_qp->ext_type);
- ret = -ENODEV;
- goto out;
- }
-
- /* LOCK the QUEUE */
- spin_lock_irqsave(&my_qp->spinlock_r, flags);
-
- /* loop processes list of recv reqs */
- while (recv_wr) {
- u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
- /* get pointer next to free WQE */
- wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
- if (unlikely(!wqe_p)) {
- /* too many posted work requests: queue overflow */
- ret = -ENOMEM;
- ehca_err(dev, "Too many posted WQEs "
- "qp_num=%x", my_qp->real_qp_num);
- goto post_recv_exit0;
- }
- /*
- * Get the index of the WQE in the recv queue. The same index
- * is used for writing into the rq_map.
- */
- rq_map_idx = start_offset / my_qp->ipz_rqueue.qe_size;
-
- /* write a RECV WQE into the QUEUE */
- ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, recv_wr,
- rq_map_idx);
- /*
- * if something failed,
- * reset the free entry pointer to the start value
- */
- if (unlikely(ret)) {
- my_qp->ipz_rqueue.current_q_offset = start_offset;
- ret = -EINVAL;
- ehca_err(dev, "Could not write WQE "
- "qp_num=%x", my_qp->real_qp_num);
- goto post_recv_exit0;
- }
-
- qmap_entry = &my_qp->rq_map.map[rq_map_idx];
- qmap_entry->app_wr_id = get_app_wr_id(recv_wr->wr_id);
- qmap_entry->reported = 0;
- qmap_entry->cqe_req = 1;
-
- wqe_cnt++;
- recv_wr = recv_wr->next;
- } /* eof for recv_wr */
-
-post_recv_exit0:
- iosync(); /* serialize GAL register access */
- hipz_update_rqa(my_qp, wqe_cnt);
- if (unlikely(ret || ehca_debug_level >= 2))
- ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
- my_qp, my_qp->real_qp_num, wqe_cnt, ret);
- spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
-
-out:
- if (ret)
- *bad_recv_wr = recv_wr;
-
- return ret;
-}
-
-int ehca_post_recv(struct ib_qp *qp,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr)
-{
- struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
-
- /* Reject WR if QP is in RESET state */
- if (unlikely(my_qp->state == IB_QPS_RESET)) {
- ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
- my_qp->state, qp->qp_num);
- *bad_recv_wr = recv_wr;
- return -EINVAL;
- }
-
- return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
-}
-
-int ehca_post_srq_recv(struct ib_srq *srq,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr)
-{
- return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
- srq->device, recv_wr, bad_recv_wr);
-}
-
-/*
- * ib_wc_opcode table converts ehca wc opcode to ib
- * Since we use zero to indicate invalid opcode, the actual ib opcode must
- * be decremented!!!
- */
-static const u8 ib_wc_opcode[255] = {
- [0x01] = IB_WC_RECV+1,
- [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
- [0x04] = IB_WC_BIND_MW+1,
- [0x08] = IB_WC_FETCH_ADD+1,
- [0x10] = IB_WC_COMP_SWAP+1,
- [0x20] = IB_WC_RDMA_WRITE+1,
- [0x40] = IB_WC_RDMA_READ+1,
- [0x80] = IB_WC_SEND+1
-};
-
-/* internal function to poll one entry of cq */
-static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
-{
- int ret = 0, qmap_tail_idx;
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- struct ehca_cqe *cqe;
- struct ehca_qp *my_qp;
- struct ehca_qmap_entry *qmap_entry;
- struct ehca_queue_map *qmap;
- int cqe_count = 0, is_error;
-
-repoll:
- cqe = (struct ehca_cqe *)
- ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
- if (!cqe) {
- ret = -EAGAIN;
- if (ehca_debug_level >= 3)
- ehca_dbg(cq->device, "Completion queue is empty "
- "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
- goto poll_cq_one_exit0;
- }
-
- /* prevents loads being reordered across this point */
- rmb();
-
- cqe_count++;
- if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
- struct ehca_qp *qp;
- int purgeflag;
- unsigned long flags;
-
- qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
- if (!qp) {
- ehca_err(cq->device, "cq_num=%x qp_num=%x "
- "could not find qp -> ignore cqe",
- my_cq->cq_number, cqe->local_qp_number);
- ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
- my_cq->cq_number, cqe->local_qp_number);
- /* ignore this purged cqe */
- goto repoll;
- }
- spin_lock_irqsave(&qp->spinlock_s, flags);
- purgeflag = qp->sqerr_purgeflag;
- spin_unlock_irqrestore(&qp->spinlock_s, flags);
-
- if (purgeflag) {
- ehca_dbg(cq->device,
- "Got CQE with purged bit qp_num=%x src_qp=%x",
- cqe->local_qp_number, cqe->remote_qp_number);
- if (ehca_debug_level >= 2)
- ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
- cqe->local_qp_number,
- cqe->remote_qp_number);
- /*
- * ignore this to avoid double cqes of bad wqe
- * that caused sqe and turn off purge flag
- */
- qp->sqerr_purgeflag = 0;
- goto repoll;
- }
- }
-
- is_error = cqe->status & WC_STATUS_ERROR_BIT;
-
- /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
- if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
- ehca_dbg(cq->device,
- "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
- is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
- ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
- my_cq, my_cq->cq_number);
- ehca_dbg(cq->device,
- "ehca_cq=%p cq_num=%x -------------------------",
- my_cq, my_cq->cq_number);
- }
-
- read_lock(&ehca_qp_idr_lock);
- my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
- read_unlock(&ehca_qp_idr_lock);
- if (!my_qp)
- goto repoll;
- wc->qp = &my_qp->ib_qp;
-
- qmap_tail_idx = get_app_wr_id(cqe->work_request_id);
- if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT))
- /* We got a send completion. */
- qmap = &my_qp->sq_map;
- else
- /* We got a receive completion. */
- qmap = &my_qp->rq_map;
-
- /* advance the tail pointer */
- qmap->tail = qmap_tail_idx;
-
- if (is_error) {
- /*
- * set left_to_poll to 0 because in error state, we will not
- * get any additional CQEs
- */
- my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
- my_qp->sq_map.entries);
- my_qp->sq_map.left_to_poll = 0;
- ehca_add_to_err_list(my_qp, 1);
-
- my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
- my_qp->rq_map.entries);
- my_qp->rq_map.left_to_poll = 0;
- if (HAS_RQ(my_qp))
- ehca_add_to_err_list(my_qp, 0);
- }
-
- qmap_entry = &qmap->map[qmap_tail_idx];
- if (qmap_entry->reported) {
- ehca_warn(cq->device, "Double cqe on qp_num=%#x",
- my_qp->real_qp_num);
- /* found a double cqe, discard it and read next one */
- goto repoll;
- }
-
- wc->wr_id = replace_wr_id(cqe->work_request_id, qmap_entry->app_wr_id);
- qmap_entry->reported = 1;
-
- /* if left_to_poll is decremented to 0, add the QP to the error list */
- if (qmap->left_to_poll > 0) {
- qmap->left_to_poll--;
- if ((my_qp->sq_map.left_to_poll == 0) &&
- (my_qp->rq_map.left_to_poll == 0)) {
- ehca_add_to_err_list(my_qp, 1);
- if (HAS_RQ(my_qp))
- ehca_add_to_err_list(my_qp, 0);
- }
- }
-
- /* eval ib_wc_opcode */
- wc->opcode = ib_wc_opcode[cqe->optype]-1;
- if (unlikely(wc->opcode == -1)) {
- ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
- "ehca_cq=%p cq_num=%x",
- cqe->optype, cqe->status, my_cq, my_cq->cq_number);
- /* dump cqe for other infos */
- ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
- my_cq, my_cq->cq_number);
- /* update also queue adder to throw away this entry!!! */
- goto repoll;
- }
-
- /* eval ib_wc_status */
- if (unlikely(is_error)) {
- /* complete with errors */
- map_ib_wc_status(cqe->status, &wc->status);
- wc->vendor_err = wc->status;
- } else
- wc->status = IB_WC_SUCCESS;
-
- wc->byte_len = cqe->nr_bytes_transferred;
- wc->pkey_index = cqe->pkey_index;
- wc->slid = cqe->rlid;
- wc->dlid_path_bits = cqe->dlid;
- wc->src_qp = cqe->remote_qp_number;
- /*
- * HW has "Immed data present" and "GRH present" in bits 6 and 5.
- * SW defines those in bits 1 and 0, so we can just shift and mask.
- */
- wc->wc_flags = (cqe->w_completion_flags >> 5) & 3;
- wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
- wc->sl = cqe->service_level;
-
-poll_cq_one_exit0:
- if (cqe_count > 0)
- hipz_update_feca(my_cq, cqe_count);
-
- return ret;
-}
-
-static int generate_flush_cqes(struct ehca_qp *my_qp, struct ib_cq *cq,
- struct ib_wc *wc, int num_entries,
- struct ipz_queue *ipz_queue, int on_sq)
-{
- int nr = 0;
- struct ehca_wqe *wqe;
- u64 offset;
- struct ehca_queue_map *qmap;
- struct ehca_qmap_entry *qmap_entry;
-
- if (on_sq)
- qmap = &my_qp->sq_map;
- else
- qmap = &my_qp->rq_map;
-
- qmap_entry = &qmap->map[qmap->next_wqe_idx];
-
- while ((nr < num_entries) && (qmap_entry->reported == 0)) {
- /* generate flush CQE */
-
- memset(wc, 0, sizeof(*wc));
-
- offset = qmap->next_wqe_idx * ipz_queue->qe_size;
- wqe = (struct ehca_wqe *)ipz_qeit_calc(ipz_queue, offset);
- if (!wqe) {
- ehca_err(cq->device, "Invalid wqe offset=%#llx on "
- "qp_num=%#x", offset, my_qp->real_qp_num);
- return nr;
- }
-
- wc->wr_id = replace_wr_id(wqe->work_request_id,
- qmap_entry->app_wr_id);
-
- if (on_sq) {
- switch (wqe->optype) {
- case WQE_OPTYPE_SEND:
- wc->opcode = IB_WC_SEND;
- break;
- case WQE_OPTYPE_RDMAWRITE:
- wc->opcode = IB_WC_RDMA_WRITE;
- break;
- case WQE_OPTYPE_RDMAREAD:
- wc->opcode = IB_WC_RDMA_READ;
- break;
- default:
- ehca_err(cq->device, "Invalid optype=%x",
- wqe->optype);
- return nr;
- }
- } else
- wc->opcode = IB_WC_RECV;
-
- if (wqe->wr_flag & WQE_WRFLAG_IMM_DATA_PRESENT) {
- wc->ex.imm_data = wqe->immediate_data;
- wc->wc_flags |= IB_WC_WITH_IMM;
- }
-
- wc->status = IB_WC_WR_FLUSH_ERR;
-
- wc->qp = &my_qp->ib_qp;
-
- /* mark as reported and advance next_wqe pointer */
- qmap_entry->reported = 1;
- qmap->next_wqe_idx = next_index(qmap->next_wqe_idx,
- qmap->entries);
- qmap_entry = &qmap->map[qmap->next_wqe_idx];
-
- wc++; nr++;
- }
-
- return nr;
-
-}
-
-int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
-{
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- int nr;
- struct ehca_qp *err_qp;
- struct ib_wc *current_wc = wc;
- int ret = 0;
- unsigned long flags;
- int entries_left = num_entries;
-
- if (num_entries < 1) {
- ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
- "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
- ret = -EINVAL;
- goto poll_cq_exit0;
- }
-
- spin_lock_irqsave(&my_cq->spinlock, flags);
-
- /* generate flush cqes for send queues */
- list_for_each_entry(err_qp, &my_cq->sqp_err_list, sq_err_node) {
- nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
- &err_qp->ipz_squeue, 1);
- entries_left -= nr;
- current_wc += nr;
-
- if (entries_left == 0)
- break;
- }
-
- /* generate flush cqes for receive queues */
- list_for_each_entry(err_qp, &my_cq->rqp_err_list, rq_err_node) {
- nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
- &err_qp->ipz_rqueue, 0);
- entries_left -= nr;
- current_wc += nr;
-
- if (entries_left == 0)
- break;
- }
-
- for (nr = 0; nr < entries_left; nr++) {
- ret = ehca_poll_cq_one(cq, current_wc);
- if (ret)
- break;
- current_wc++;
- } /* eof for nr */
- entries_left -= nr;
-
- spin_unlock_irqrestore(&my_cq->spinlock, flags);
- if (ret == -EAGAIN || !ret)
- ret = num_entries - entries_left;
-
-poll_cq_exit0:
- return ret;
-}
-
-int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
-{
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- int ret = 0;
-
- switch (notify_flags & IB_CQ_SOLICITED_MASK) {
- case IB_CQ_SOLICITED:
- hipz_set_cqx_n0(my_cq, 1);
- break;
- case IB_CQ_NEXT_COMP:
- hipz_set_cqx_n1(my_cq, 1);
- break;
- default:
- return -EINVAL;
- }
-
- if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
- unsigned long spl_flags;
- spin_lock_irqsave(&my_cq->spinlock, spl_flags);
- ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
- spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
- }
-
- return ret;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * SQP functions
- *
- * Authors: Khadija Souissi <souissi@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <rdma/ib_mad.h>
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-#define IB_MAD_STATUS_REDIRECT cpu_to_be16(0x0002)
-#define IB_MAD_STATUS_UNSUP_VERSION cpu_to_be16(0x0004)
-#define IB_MAD_STATUS_UNSUP_METHOD cpu_to_be16(0x0008)
-
-#define IB_PMA_CLASS_PORT_INFO cpu_to_be16(0x0001)
-
-/**
- * ehca_define_sqp - Defines special queue pair 1 (GSI QP). When special queue
- * pair is created successfully, the corresponding port gets active.
- *
- * Define Special Queue pair 0 (SMI QP) is still not supported.
- *
- * @qp_init_attr: Queue pair init attributes with port and queue pair type
- */
-
-u64 ehca_define_sqp(struct ehca_shca *shca,
- struct ehca_qp *ehca_qp,
- struct ib_qp_init_attr *qp_init_attr)
-{
- u32 pma_qp_nr, bma_qp_nr;
- u64 ret;
- u8 port = qp_init_attr->port_num;
- int counter;
-
- shca->sport[port - 1].port_state = IB_PORT_DOWN;
-
- switch (qp_init_attr->qp_type) {
- case IB_QPT_SMI:
- /* function not supported yet */
- break;
- case IB_QPT_GSI:
- ret = hipz_h_define_aqp1(shca->ipz_hca_handle,
- ehca_qp->ipz_qp_handle,
- ehca_qp->galpas.kernel,
- (u32) qp_init_attr->port_num,
- &pma_qp_nr, &bma_qp_nr);
-
- if (ret != H_SUCCESS) {
- ehca_err(&shca->ib_device,
- "Can't define AQP1 for port %x. h_ret=%lli",
- port, ret);
- return ret;
- }
- shca->sport[port - 1].pma_qp_nr = pma_qp_nr;
- ehca_dbg(&shca->ib_device, "port=%x pma_qp_nr=%x",
- port, pma_qp_nr);
- break;
- default:
- ehca_err(&shca->ib_device, "invalid qp_type=%x",
- qp_init_attr->qp_type);
- return H_PARAMETER;
- }
-
- if (ehca_nr_ports < 0) /* autodetect mode */
- return H_SUCCESS;
-
- for (counter = 0;
- shca->sport[port - 1].port_state != IB_PORT_ACTIVE &&
- counter < ehca_port_act_time;
- counter++) {
- ehca_dbg(&shca->ib_device, "... wait until port %x is active",
- port);
- msleep_interruptible(1000);
- }
-
- if (counter == ehca_port_act_time) {
- ehca_err(&shca->ib_device, "Port %x is not active.", port);
- return H_HARDWARE;
- }
-
- return H_SUCCESS;
-}
-
-struct ib_perf {
- struct ib_mad_hdr mad_hdr;
- u8 reserved[40];
- u8 data[192];
-} __attribute__ ((packed));
-
-/* TC/SL/FL packed into 32 bits, as in ClassPortInfo */
-struct tcslfl {
- u32 tc:8;
- u32 sl:4;
- u32 fl:20;
-} __attribute__ ((packed));
-
-/* IP Version/TC/FL packed into 32 bits, as in GRH */
-struct vertcfl {
- u32 ver:4;
- u32 tc:8;
- u32 fl:20;
-} __attribute__ ((packed));
-
-static int ehca_process_perf(struct ib_device *ibdev, u8 port_num,
- const struct ib_wc *in_wc, const struct ib_grh *in_grh,
- const struct ib_mad *in_mad, struct ib_mad *out_mad)
-{
- const struct ib_perf *in_perf = (const struct ib_perf *)in_mad;
- struct ib_perf *out_perf = (struct ib_perf *)out_mad;
- struct ib_class_port_info *poi =
- (struct ib_class_port_info *)out_perf->data;
- struct tcslfl *tcslfl =
- (struct tcslfl *)&poi->redirect_tcslfl;
- struct ehca_shca *shca =
- container_of(ibdev, struct ehca_shca, ib_device);
- struct ehca_sport *sport = &shca->sport[port_num - 1];
-
- ehca_dbg(ibdev, "method=%x", in_perf->mad_hdr.method);
-
- *out_mad = *in_mad;
-
- if (in_perf->mad_hdr.class_version != 1) {
- ehca_warn(ibdev, "Unsupported class_version=%x",
- in_perf->mad_hdr.class_version);
- out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_VERSION;
- goto perf_reply;
- }
-
- switch (in_perf->mad_hdr.method) {
- case IB_MGMT_METHOD_GET:
- case IB_MGMT_METHOD_SET:
- /* set class port info for redirection */
- out_perf->mad_hdr.attr_id = IB_PMA_CLASS_PORT_INFO;
- out_perf->mad_hdr.status = IB_MAD_STATUS_REDIRECT;
- memset(poi, 0, sizeof(*poi));
- poi->base_version = 1;
- poi->class_version = 1;
- poi->resp_time_value = 18;
-
- /* copy local routing information from WC where applicable */
- tcslfl->sl = in_wc->sl;
- poi->redirect_lid =
- sport->saved_attr.lid | in_wc->dlid_path_bits;
- poi->redirect_qp = sport->pma_qp_nr;
- poi->redirect_qkey = IB_QP1_QKEY;
-
- ehca_query_pkey(ibdev, port_num, in_wc->pkey_index,
- &poi->redirect_pkey);
-
- /* if request was globally routed, copy route info */
- if (in_grh) {
- const struct vertcfl *vertcfl =
- (const struct vertcfl *)&in_grh->version_tclass_flow;
- memcpy(poi->redirect_gid, in_grh->dgid.raw,
- sizeof(poi->redirect_gid));
- tcslfl->tc = vertcfl->tc;
- tcslfl->fl = vertcfl->fl;
- } else
- /* else only fill in default GID */
- ehca_query_gid(ibdev, port_num, 0,
- (union ib_gid *)&poi->redirect_gid);
-
- ehca_dbg(ibdev, "ehca_pma_lid=%x ehca_pma_qp=%x",
- sport->saved_attr.lid, sport->pma_qp_nr);
- break;
-
- case IB_MGMT_METHOD_GET_RESP:
- return IB_MAD_RESULT_FAILURE;
-
- default:
- out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_METHOD;
- break;
- }
-
-perf_reply:
- out_perf->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
-
- return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
-}
-
-int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
- const struct ib_wc *in_wc, const struct ib_grh *in_grh,
- const struct ib_mad_hdr *in, size_t in_mad_size,
- struct ib_mad_hdr *out, size_t *out_mad_size,
- u16 *out_mad_pkey_index)
-{
- int ret;
- const struct ib_mad *in_mad = (const struct ib_mad *)in;
- struct ib_mad *out_mad = (struct ib_mad *)out;
-
- if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||
- *out_mad_size != sizeof(*out_mad)))
- return IB_MAD_RESULT_FAILURE;
-
- if (!port_num || port_num > ibdev->phys_port_cnt || !in_wc)
- return IB_MAD_RESULT_FAILURE;
-
- /* accept only pma request */
- if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
- return IB_MAD_RESULT_SUCCESS;
-
- ehca_dbg(ibdev, "port_num=%x src_qp=%x", port_num, in_wc->src_qp);
- ret = ehca_process_perf(ibdev, port_num, in_wc, in_grh,
- in_mad, out_mad);
-
- return ret;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * auxiliary functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Khadija Souissi <souissik@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef EHCA_TOOLS_H
-#define EHCA_TOOLS_H
-
-#include <linux/kernel.h>
-#include <linux/spinlock.h>
-#include <linux/delay.h>
-#include <linux/idr.h>
-#include <linux/kthread.h>
-#include <linux/mm.h>
-#include <linux/mman.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/vmalloc.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/device.h>
-
-#include <linux/atomic.h>
-#include <asm/ibmebus.h>
-#include <asm/io.h>
-#include <asm/pgtable.h>
-#include <asm/hvcall.h>
-
-extern int ehca_debug_level;
-
-#define ehca_dbg(ib_dev, format, arg...) \
- do { \
- if (unlikely(ehca_debug_level)) \
- dev_printk(KERN_DEBUG, (ib_dev)->dma_device, \
- "PU%04x EHCA_DBG:%s " format "\n", \
- raw_smp_processor_id(), __func__, \
- ## arg); \
- } while (0)
-
-#define ehca_info(ib_dev, format, arg...) \
- dev_info((ib_dev)->dma_device, "PU%04x EHCA_INFO:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-#define ehca_warn(ib_dev, format, arg...) \
- dev_warn((ib_dev)->dma_device, "PU%04x EHCA_WARN:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-#define ehca_err(ib_dev, format, arg...) \
- dev_err((ib_dev)->dma_device, "PU%04x EHCA_ERR:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-/* use this one only if no ib_dev available */
-#define ehca_gen_dbg(format, arg...) \
- do { \
- if (unlikely(ehca_debug_level)) \
- printk(KERN_DEBUG "PU%04x EHCA_DBG:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg); \
- } while (0)
-
-#define ehca_gen_warn(format, arg...) \
- printk(KERN_INFO "PU%04x EHCA_WARN:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-#define ehca_gen_err(format, arg...) \
- printk(KERN_ERR "PU%04x EHCA_ERR:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-/**
- * ehca_dmp - printk a memory block, whose length is n*8 bytes.
- * Each line has the following layout:
- * <format string> adr=X ofs=Y <8 bytes hex> <8 bytes hex>
- */
-#define ehca_dmp(adr, len, format, args...) \
- do { \
- unsigned int x; \
- unsigned int l = (unsigned int)(len); \
- unsigned char *deb = (unsigned char *)(adr); \
- for (x = 0; x < l; x += 16) { \
- printk(KERN_INFO "EHCA_DMP:%s " format \
- " adr=%p ofs=%04x %016llx %016llx\n", \
- __func__, ##args, deb, x, \
- *((u64 *)&deb[0]), *((u64 *)&deb[8])); \
- deb += 16; \
- } \
- } while (0)
-
-/* define a bitmask, little endian version */
-#define EHCA_BMASK(pos, length) (((pos) << 16) + (length))
-
-/* define a bitmask, the ibm way... */
-#define EHCA_BMASK_IBM(from, to) (((63 - to) << 16) + ((to) - (from) + 1))
-
-/* internal function, don't use */
-#define EHCA_BMASK_SHIFTPOS(mask) (((mask) >> 16) & 0xffff)
-
-/* internal function, don't use */
-#define EHCA_BMASK_MASK(mask) (~0ULL >> ((64 - (mask)) & 0xffff))
-
-/**
- * EHCA_BMASK_SET - return value shifted and masked by mask
- * variable|=EHCA_BMASK_SET(MY_MASK,0x4711) ORs the bits in variable
- * variable&=~EHCA_BMASK_SET(MY_MASK,-1) clears the bits from the mask
- * in variable
- */
-#define EHCA_BMASK_SET(mask, value) \
- ((EHCA_BMASK_MASK(mask) & ((u64)(value))) << EHCA_BMASK_SHIFTPOS(mask))
-
-/**
- * EHCA_BMASK_GET - extract a parameter from value by mask
- */
-#define EHCA_BMASK_GET(mask, value) \
- (EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask)))
-
-/* Converts ehca to ib return code */
-int ehca2ib_return_code(u64 ehca_rc);
-
-#endif /* EHCA_TOOLS_H */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * userspace support verbs
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_classes.h"
-#include "ehca_iverbs.h"
-#include "ehca_mrmw.h"
-#include "ehca_tools.h"
-#include "hcp_if.h"
-
-struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
- struct ib_udata *udata)
-{
- struct ehca_ucontext *my_context;
-
- my_context = kzalloc(sizeof *my_context, GFP_KERNEL);
- if (!my_context) {
- ehca_err(device, "Out of memory device=%p", device);
- return ERR_PTR(-ENOMEM);
- }
-
- return &my_context->ib_ucontext;
-}
-
-int ehca_dealloc_ucontext(struct ib_ucontext *context)
-{
- kfree(container_of(context, struct ehca_ucontext, ib_ucontext));
- return 0;
-}
-
-static void ehca_mm_open(struct vm_area_struct *vma)
-{
- u32 *count = (u32 *)vma->vm_private_data;
- if (!count) {
- ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
- vma->vm_start, vma->vm_end);
- return;
- }
- (*count)++;
- if (!(*count))
- ehca_gen_err("Use count overflow vm_start=%lx vm_end=%lx",
- vma->vm_start, vma->vm_end);
- ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
- vma->vm_start, vma->vm_end, *count);
-}
-
-static void ehca_mm_close(struct vm_area_struct *vma)
-{
- u32 *count = (u32 *)vma->vm_private_data;
- if (!count) {
- ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
- vma->vm_start, vma->vm_end);
- return;
- }
- (*count)--;
- ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
- vma->vm_start, vma->vm_end, *count);
-}
-
-static const struct vm_operations_struct vm_ops = {
- .open = ehca_mm_open,
- .close = ehca_mm_close,
-};
-
-static int ehca_mmap_fw(struct vm_area_struct *vma, struct h_galpas *galpas,
- u32 *mm_count)
-{
- int ret;
- u64 vsize, physical;
-
- vsize = vma->vm_end - vma->vm_start;
- if (vsize < EHCA_PAGESIZE) {
- ehca_gen_err("invalid vsize=%lx", vma->vm_end - vma->vm_start);
- return -EINVAL;
- }
-
- physical = galpas->user.fw_handle;
- vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
- ehca_gen_dbg("vsize=%llx physical=%llx", vsize, physical);
- /* VM_IO | VM_DONTEXPAND | VM_DONTDUMP are set by remap_pfn_range() */
- ret = remap_4k_pfn(vma, vma->vm_start, physical >> EHCA_PAGESHIFT,
- vma->vm_page_prot);
- if (unlikely(ret)) {
- ehca_gen_err("remap_pfn_range() failed ret=%i", ret);
- return -ENOMEM;
- }
-
- vma->vm_private_data = mm_count;
- (*mm_count)++;
- vma->vm_ops = &vm_ops;
-
- return 0;
-}
-
-static int ehca_mmap_queue(struct vm_area_struct *vma, struct ipz_queue *queue,
- u32 *mm_count)
-{
- int ret;
- u64 start, ofs;
- struct page *page;
-
- vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
- start = vma->vm_start;
- for (ofs = 0; ofs < queue->queue_length; ofs += PAGE_SIZE) {
- u64 virt_addr = (u64)ipz_qeit_calc(queue, ofs);
- page = virt_to_page(virt_addr);
- ret = vm_insert_page(vma, start, page);
- if (unlikely(ret)) {
- ehca_gen_err("vm_insert_page() failed rc=%i", ret);
- return ret;
- }
- start += PAGE_SIZE;
- }
- vma->vm_private_data = mm_count;
- (*mm_count)++;
- vma->vm_ops = &vm_ops;
-
- return 0;
-}
-
-static int ehca_mmap_cq(struct vm_area_struct *vma, struct ehca_cq *cq,
- u32 rsrc_type)
-{
- int ret;
-
- switch (rsrc_type) {
- case 0: /* galpa fw handle */
- ehca_dbg(cq->ib_cq.device, "cq_num=%x fw", cq->cq_number);
- ret = ehca_mmap_fw(vma, &cq->galpas, &cq->mm_count_galpa);
- if (unlikely(ret)) {
- ehca_err(cq->ib_cq.device,
- "ehca_mmap_fw() failed rc=%i cq_num=%x",
- ret, cq->cq_number);
- return ret;
- }
- break;
-
- case 1: /* cq queue_addr */
- ehca_dbg(cq->ib_cq.device, "cq_num=%x queue", cq->cq_number);
- ret = ehca_mmap_queue(vma, &cq->ipz_queue, &cq->mm_count_queue);
- if (unlikely(ret)) {
- ehca_err(cq->ib_cq.device,
- "ehca_mmap_queue() failed rc=%i cq_num=%x",
- ret, cq->cq_number);
- return ret;
- }
- break;
-
- default:
- ehca_err(cq->ib_cq.device, "bad resource type=%x cq_num=%x",
- rsrc_type, cq->cq_number);
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int ehca_mmap_qp(struct vm_area_struct *vma, struct ehca_qp *qp,
- u32 rsrc_type)
-{
- int ret;
-
- switch (rsrc_type) {
- case 0: /* galpa fw handle */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x fw", qp->ib_qp.qp_num);
- ret = ehca_mmap_fw(vma, &qp->galpas, &qp->mm_count_galpa);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "remap_pfn_range() failed ret=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return -ENOMEM;
- }
- break;
-
- case 1: /* qp rqueue_addr */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x rq", qp->ib_qp.qp_num);
- ret = ehca_mmap_queue(vma, &qp->ipz_rqueue,
- &qp->mm_count_rqueue);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "ehca_mmap_queue(rq) failed rc=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return ret;
- }
- break;
-
- case 2: /* qp squeue_addr */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x sq", qp->ib_qp.qp_num);
- ret = ehca_mmap_queue(vma, &qp->ipz_squeue,
- &qp->mm_count_squeue);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "ehca_mmap_queue(sq) failed rc=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return ret;
- }
- break;
-
- default:
- ehca_err(qp->ib_qp.device, "bad resource type=%x qp=num=%x",
- rsrc_type, qp->ib_qp.qp_num);
- return -EINVAL;
- }
-
- return 0;
-}
-
-int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
-{
- u64 fileoffset = vma->vm_pgoff;
- u32 idr_handle = fileoffset & 0x1FFFFFF;
- u32 q_type = (fileoffset >> 27) & 0x1; /* CQ, QP,... */
- u32 rsrc_type = (fileoffset >> 25) & 0x3; /* sq,rq,cmnd_window */
- u32 ret;
- struct ehca_cq *cq;
- struct ehca_qp *qp;
- struct ib_uobject *uobject;
-
- switch (q_type) {
- case 0: /* CQ */
- read_lock(&ehca_cq_idr_lock);
- cq = idr_find(&ehca_cq_idr, idr_handle);
- read_unlock(&ehca_cq_idr_lock);
-
- /* make sure this mmap really belongs to the authorized user */
- if (!cq)
- return -EINVAL;
-
- if (!cq->ib_cq.uobject || cq->ib_cq.uobject->context != context)
- return -EINVAL;
-
- ret = ehca_mmap_cq(vma, cq, rsrc_type);
- if (unlikely(ret)) {
- ehca_err(cq->ib_cq.device,
- "ehca_mmap_cq() failed rc=%i cq_num=%x",
- ret, cq->cq_number);
- return ret;
- }
- break;
-
- case 1: /* QP */
- read_lock(&ehca_qp_idr_lock);
- qp = idr_find(&ehca_qp_idr, idr_handle);
- read_unlock(&ehca_qp_idr_lock);
-
- /* make sure this mmap really belongs to the authorized user */
- if (!qp)
- return -EINVAL;
-
- uobject = IS_SRQ(qp) ? qp->ib_srq.uobject : qp->ib_qp.uobject;
- if (!uobject || uobject->context != context)
- return -EINVAL;
-
- ret = ehca_mmap_qp(vma, qp, rsrc_type);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "ehca_mmap_qp() failed rc=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return ret;
- }
- break;
-
- default:
- ehca_gen_err("bad queue type %x", q_type);
- return -EINVAL;
- }
-
- return 0;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Firmware Infiniband Interface code for POWER
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- * Gerd Bayer <gerd.bayer@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <asm/hvcall.h>
-#include "ehca_tools.h"
-#include "hcp_if.h"
-#include "hcp_phyp.h"
-#include "hipz_fns.h"
-#include "ipz_pt_fn.h"
-
-#define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11)
-#define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12)
-#define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15)
-#define H_ALL_RES_QP_STORAGE EHCA_BMASK_IBM(16, 17)
-#define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18)
-#define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21)
-#define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23)
-#define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31)
-#define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35)
-#define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39)
-#define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63)
-
-#define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15)
-#define H_ALL_RES_QP_MAX_OUTST_RECV_WR EHCA_BMASK_IBM(16, 31)
-#define H_ALL_RES_QP_MAX_SEND_SGE EHCA_BMASK_IBM(32, 39)
-#define H_ALL_RES_QP_MAX_RECV_SGE EHCA_BMASK_IBM(40, 47)
-
-#define H_ALL_RES_QP_UD_AV_LKEY EHCA_BMASK_IBM(32, 63)
-#define H_ALL_RES_QP_SRQ_QP_TOKEN EHCA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_SRQ_QP_HANDLE EHCA_BMASK_IBM(0, 64)
-#define H_ALL_RES_QP_SRQ_LIMIT EHCA_BMASK_IBM(48, 63)
-#define H_ALL_RES_QP_SRQ_QPN EHCA_BMASK_IBM(40, 63)
-
-#define H_ALL_RES_QP_ACT_OUTST_SEND_WR EHCA_BMASK_IBM(16, 31)
-#define H_ALL_RES_QP_ACT_OUTST_RECV_WR EHCA_BMASK_IBM(48, 63)
-#define H_ALL_RES_QP_ACT_SEND_SGE EHCA_BMASK_IBM(8, 15)
-#define H_ALL_RES_QP_ACT_RECV_SGE EHCA_BMASK_IBM(24, 31)
-
-#define H_ALL_RES_QP_SQUEUE_SIZE_PAGES EHCA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_RQUEUE_SIZE_PAGES EHCA_BMASK_IBM(32, 63)
-
-#define H_MP_INIT_TYPE EHCA_BMASK_IBM(44, 47)
-#define H_MP_SHUTDOWN EHCA_BMASK_IBM(48, 48)
-#define H_MP_RESET_QKEY_CTR EHCA_BMASK_IBM(49, 49)
-
-#define HCALL4_REGS_FORMAT "r4=%lx r5=%lx r6=%lx r7=%lx"
-#define HCALL7_REGS_FORMAT HCALL4_REGS_FORMAT " r8=%lx r9=%lx r10=%lx"
-#define HCALL9_REGS_FORMAT HCALL7_REGS_FORMAT " r11=%lx r12=%lx"
-
-static DEFINE_SPINLOCK(hcall_lock);
-
-static long ehca_plpar_hcall_norets(unsigned long opcode,
- unsigned long arg1,
- unsigned long arg2,
- unsigned long arg3,
- unsigned long arg4,
- unsigned long arg5,
- unsigned long arg6,
- unsigned long arg7)
-{
- long ret;
- int i, sleep_msecs;
- unsigned long flags = 0;
-
- if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("opcode=%lx " HCALL7_REGS_FORMAT,
- opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
-
- for (i = 0; i < 5; i++) {
- /* serialize hCalls to work around firmware issue */
- if (ehca_lock_hcalls)
- spin_lock_irqsave(&hcall_lock, flags);
-
- ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
- arg5, arg6, arg7);
-
- if (ehca_lock_hcalls)
- spin_unlock_irqrestore(&hcall_lock, flags);
-
- if (H_IS_LONG_BUSY(ret)) {
- sleep_msecs = get_longbusy_msecs(ret);
- msleep_interruptible(sleep_msecs);
- continue;
- }
-
- if (ret < H_SUCCESS)
- ehca_gen_err("opcode=%lx ret=%li " HCALL7_REGS_FORMAT,
- opcode, ret, arg1, arg2, arg3,
- arg4, arg5, arg6, arg7);
- else
- if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("opcode=%lx ret=%li", opcode, ret);
-
- return ret;
- }
-
- return H_BUSY;
-}
-
-static long ehca_plpar_hcall9(unsigned long opcode,
- unsigned long *outs, /* array of 9 outputs */
- unsigned long arg1,
- unsigned long arg2,
- unsigned long arg3,
- unsigned long arg4,
- unsigned long arg5,
- unsigned long arg6,
- unsigned long arg7,
- unsigned long arg8,
- unsigned long arg9)
-{
- long ret;
- int i, sleep_msecs;
- unsigned long flags = 0;
-
- if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT, opcode,
- arg1, arg2, arg3, arg4, arg5,
- arg6, arg7, arg8, arg9);
-
- for (i = 0; i < 5; i++) {
- /* serialize hCalls to work around firmware issue */
- if (ehca_lock_hcalls)
- spin_lock_irqsave(&hcall_lock, flags);
-
- ret = plpar_hcall9(opcode, outs,
- arg1, arg2, arg3, arg4, arg5,
- arg6, arg7, arg8, arg9);
-
- if (ehca_lock_hcalls)
- spin_unlock_irqrestore(&hcall_lock, flags);
-
- if (H_IS_LONG_BUSY(ret)) {
- sleep_msecs = get_longbusy_msecs(ret);
- msleep_interruptible(sleep_msecs);
- continue;
- }
-
- if (ret < H_SUCCESS) {
- ehca_gen_err("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT,
- opcode, arg1, arg2, arg3, arg4, arg5,
- arg6, arg7, arg8, arg9);
- ehca_gen_err("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
- ret, outs[0], outs[1], outs[2], outs[3],
- outs[4], outs[5], outs[6], outs[7],
- outs[8]);
- } else if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
- ret, outs[0], outs[1], outs[2], outs[3],
- outs[4], outs[5], outs[6], outs[7],
- outs[8]);
- return ret;
- }
-
- return H_BUSY;
-}
-
-u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_pfeq *pfeq,
- const u32 neq_control,
- const u32 number_of_entries,
- struct ipz_eq_handle *eq_handle,
- u32 *act_nr_of_entries,
- u32 *act_pages,
- u32 *eq_ist)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
- u64 allocate_controls;
-
- /* resource type */
- allocate_controls = 3ULL;
-
- /* ISN is associated */
- if (neq_control != 1)
- allocate_controls = (1ULL << (63 - 7)) | allocate_controls;
- else /* notification event queue */
- allocate_controls = (1ULL << 63) | allocate_controls;
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- allocate_controls, /* r5 */
- number_of_entries, /* r6 */
- 0, 0, 0, 0, 0, 0);
- eq_handle->handle = outs[0];
- *act_nr_of_entries = (u32)outs[3];
- *act_pages = (u32)outs[4];
- *eq_ist = (u32)outs[5];
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resource - ret=%lli ", ret);
-
- return ret;
-}
-
-u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
- struct ipz_eq_handle eq_handle,
- const u64 event_mask)
-{
- return ehca_plpar_hcall_norets(H_RESET_EVENTS,
- adapter_handle.handle, /* r4 */
- eq_handle.handle, /* r5 */
- event_mask, /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- struct ehca_alloc_cq_parms *param)
-{
- int rc;
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- 2, /* r5 */
- param->eq_handle.handle, /* r6 */
- cq->token, /* r7 */
- param->nr_cqe, /* r8 */
- 0, 0, 0, 0);
- cq->ipz_cq_handle.handle = outs[0];
- param->act_nr_of_entries = (u32)outs[3];
- param->act_pages = (u32)outs[4];
-
- if (ret == H_SUCCESS) {
- rc = hcp_galpas_ctor(&cq->galpas, 0, outs[5], outs[6]);
- if (rc) {
- ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
- rc, outs[5]);
-
- ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- cq->ipz_cq_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
- ret = H_NO_MEM;
- }
- }
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resources. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_alloc_qp_parms *parms, int is_user)
-{
- int rc;
- u64 ret;
- u64 allocate_controls, max_r10_reg, r11, r12;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- allocate_controls =
- EHCA_BMASK_SET(H_ALL_RES_QP_ENHANCED_OPS, parms->ext_type)
- | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
- | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE,
- parms->squeue.page_size)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE,
- parms->rqueue.page_size)
- | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
- !!(parms->ll_comp_flags & LLQP_RECV_COMP))
- | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
- !!(parms->ll_comp_flags & LLQP_SEND_COMP))
- | EHCA_BMASK_SET(H_ALL_RES_QP_UD_AV_LKEY_CTRL,
- parms->ud_av_l_key_ctl)
- | EHCA_BMASK_SET(H_ALL_RES_QP_RESOURCE_TYPE, 1);
-
- max_r10_reg =
- EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
- parms->squeue.max_wr + 1)
- | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
- parms->rqueue.max_wr + 1)
- | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
- parms->squeue.max_sge)
- | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
- parms->rqueue.max_sge);
-
- r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token);
-
- if (parms->ext_type == EQPT_SRQ)
- r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_LIMIT, parms->srq_limit);
- else
- r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QPN, parms->srq_qpn);
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- allocate_controls, /* r5 */
- parms->send_cq_handle.handle,
- parms->recv_cq_handle.handle,
- parms->eq_handle.handle,
- ((u64)parms->token << 32) | parms->pd.value,
- max_r10_reg, r11, r12);
-
- parms->qp_handle.handle = outs[0];
- parms->real_qp_num = (u32)outs[1];
- parms->squeue.act_nr_wqes =
- (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
- parms->rqueue.act_nr_wqes =
- (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
- parms->squeue.act_nr_sges =
- (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
- parms->rqueue.act_nr_sges =
- (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
- parms->squeue.queue_size =
- (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
- parms->rqueue.queue_size =
- (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
-
- if (ret == H_SUCCESS) {
- rc = hcp_galpas_ctor(&parms->galpas, is_user, outs[6], outs[6]);
- if (rc) {
- ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
- rc, outs[6]);
-
- ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- parms->qp_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
- ret = H_NO_MEM;
- }
- }
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resources. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id,
- struct hipz_query_port *query_port_response_block)
-{
- u64 ret;
- u64 r_cb = __pa(query_port_response_block);
-
- if (r_cb & (EHCA_PAGESIZE-1)) {
- ehca_gen_err("response block not page aligned");
- return H_PARAMETER;
- }
-
- ret = ehca_plpar_hcall_norets(H_QUERY_PORT,
- adapter_handle.handle, /* r4 */
- port_id, /* r5 */
- r_cb, /* r6 */
- 0, 0, 0, 0);
-
- if (ehca_debug_level >= 2)
- ehca_dmp(query_port_response_block, 64, "response_block");
-
- return ret;
-}
-
-u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id, const u32 port_cap,
- const u8 init_type, const int modify_mask)
-{
- u64 port_attributes = port_cap;
-
- if (modify_mask & IB_PORT_SHUTDOWN)
- port_attributes |= EHCA_BMASK_SET(H_MP_SHUTDOWN, 1);
- if (modify_mask & IB_PORT_INIT_TYPE)
- port_attributes |= EHCA_BMASK_SET(H_MP_INIT_TYPE, init_type);
- if (modify_mask & IB_PORT_RESET_QKEY_CNTR)
- port_attributes |= EHCA_BMASK_SET(H_MP_RESET_QKEY_CTR, 1);
-
- return ehca_plpar_hcall_norets(H_MODIFY_PORT,
- adapter_handle.handle, /* r4 */
- port_id, /* r5 */
- port_attributes, /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
- struct hipz_query_hca *query_hca_rblock)
-{
- u64 r_cb = __pa(query_hca_rblock);
-
- if (r_cb & (EHCA_PAGESIZE-1)) {
- ehca_gen_err("response_block=%p not page aligned",
- query_hca_rblock);
- return H_PARAMETER;
- }
-
- return ehca_plpar_hcall_norets(H_QUERY_HCA,
- adapter_handle.handle, /* r4 */
- r_cb, /* r5 */
- 0, 0, 0, 0, 0);
-}
-
-u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
- const u8 pagesize,
- const u8 queue_type,
- const u64 resource_handle,
- const u64 logical_address_of_page,
- u64 count)
-{
- return ehca_plpar_hcall_norets(H_REGISTER_RPAGES,
- adapter_handle.handle, /* r4 */
- (u64)queue_type | ((u64)pagesize) << 8,
- /* r5 */
- resource_handle, /* r6 */
- logical_address_of_page, /* r7 */
- count, /* r8 */
- 0, 0);
-}
-
-u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_eq_handle eq_handle,
- struct ehca_pfeq *pfeq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count)
-{
- if (count != 1) {
- ehca_gen_err("Ppage counter=%llx", count);
- return H_PARAMETER;
- }
- return hipz_h_register_rpage(adapter_handle,
- pagesize,
- queue_type,
- eq_handle.handle,
- logical_address_of_page, count);
-}
-
-u64 hipz_h_query_int_state(const struct ipz_adapter_handle adapter_handle,
- u32 ist)
-{
- u64 ret;
- ret = ehca_plpar_hcall_norets(H_QUERY_INT_STATE,
- adapter_handle.handle, /* r4 */
- ist, /* r5 */
- 0, 0, 0, 0, 0);
-
- if (ret != H_SUCCESS && ret != H_BUSY)
- ehca_gen_err("Could not query interrupt state.");
-
- return ret;
-}
-
-u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_cq_handle cq_handle,
- struct ehca_pfcq *pfcq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa gal)
-{
- if (count != 1) {
- ehca_gen_err("Page counter=%llx", count);
- return H_PARAMETER;
- }
-
- return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
- cq_handle.handle, logical_address_of_page,
- count);
-}
-
-u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa galpa)
-{
- if (count > 1) {
- ehca_gen_err("Page counter=%llx", count);
- return H_PARAMETER;
- }
-
- return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
- qp_handle.handle, logical_address_of_page,
- count);
-}
-
-u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- void **log_addr_next_sq_wqe2processed,
- void **log_addr_next_rq_wqe2processed,
- int dis_and_get_function_code)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
- adapter_handle.handle, /* r4 */
- dis_and_get_function_code, /* r5 */
- qp_handle.handle, /* r6 */
- 0, 0, 0, 0, 0, 0);
- if (log_addr_next_sq_wqe2processed)
- *log_addr_next_sq_wqe2processed = (void *)outs[0];
- if (log_addr_next_rq_wqe2processed)
- *log_addr_next_rq_wqe2processed = (void *)outs[1];
-
- return ret;
-}
-
-u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u64 update_mask,
- struct hcp_modify_qp_control_block *mqpcb,
- struct h_galpa gal)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
- ret = ehca_plpar_hcall9(H_MODIFY_QP, outs,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- update_mask, /* r6 */
- __pa(mqpcb), /* r7 */
- 0, 0, 0, 0, 0);
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Insufficient resources ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- struct hcp_modify_qp_control_block *qqpcb,
- struct h_galpa gal)
-{
- return ehca_plpar_hcall_norets(H_QUERY_QP,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- __pa(qqpcb), /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_qp *qp)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = hcp_galpas_dtor(&qp->galpas);
- if (ret) {
- ehca_gen_err("Could not destruct qp->galpas");
- return H_RESOURCE;
- }
- ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
- adapter_handle.handle, /* r4 */
- /* function code */
- 1, /* r5 */
- qp->ipz_qp_handle.handle, /* r6 */
- 0, 0, 0, 0, 0, 0);
- if (ret == H_HARDWARE)
- ehca_gen_err("HCA not operational. ret=%lli", ret);
-
- ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- qp->ipz_qp_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-
- if (ret == H_RESOURCE)
- ehca_gen_err("Resource still in use. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port)
-{
- return ehca_plpar_hcall_norets(H_DEFINE_AQP0,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- port, /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port, u32 * pma_qp_nr,
- u32 * bma_qp_nr)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_DEFINE_AQP1, outs,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- port, /* r6 */
- 0, 0, 0, 0, 0, 0);
- *pma_qp_nr = (u32)outs[0];
- *bma_qp_nr = (u32)outs[1];
-
- if (ret == H_ALIAS_EXIST)
- ehca_gen_err("AQP1 already exists. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id)
-{
- u64 ret;
-
- ret = ehca_plpar_hcall_norets(H_ATTACH_MCQP,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- mcg_dlid, /* r6 */
- interface_id, /* r7 */
- subnet_prefix, /* r8 */
- 0, 0);
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resources. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id)
-{
- return ehca_plpar_hcall_norets(H_DETACH_MCQP,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- mcg_dlid, /* r6 */
- interface_id, /* r7 */
- subnet_prefix, /* r8 */
- 0, 0);
-}
-
-u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- u8 force_flag)
-{
- u64 ret;
-
- ret = hcp_galpas_dtor(&cq->galpas);
- if (ret) {
- ehca_gen_err("Could not destruct cp->galpas");
- return H_RESOURCE;
- }
-
- ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- cq->ipz_cq_handle.handle, /* r5 */
- force_flag != 0 ? 1L : 0L, /* r6 */
- 0, 0, 0, 0);
-
- if (ret == H_RESOURCE)
- ehca_gen_err("H_FREE_RESOURCE failed ret=%lli ", ret);
-
- return ret;
-}
-
-u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_eq *eq)
-{
- u64 ret;
-
- ret = hcp_galpas_dtor(&eq->galpas);
- if (ret) {
- ehca_gen_err("Could not destruct eq->galpas");
- return H_RESOURCE;
- }
-
- ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- eq->ipz_eq_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-
- if (ret == H_RESOURCE)
- ehca_gen_err("Resource in use. ret=%lli ", ret);
-
- return ret;
-}
-
-u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- 5, /* r5 */
- vaddr, /* r6 */
- length, /* r7 */
- (((u64)access_ctrl) << 32ULL), /* r8 */
- pd.value, /* r9 */
- 0, 0, 0);
- outparms->handle.handle = outs[0];
- outparms->lkey = (u32)outs[2];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count)
-{
- u64 ret;
-
- if (unlikely(ehca_debug_level >= 3)) {
- if (count > 1) {
- u64 *kpage;
- int i;
- kpage = __va(logical_address_of_page);
- for (i = 0; i < count; i++)
- ehca_gen_dbg("kpage[%d]=%p",
- i, (void *)kpage[i]);
- } else
- ehca_gen_dbg("kpage=%p",
- (void *)logical_address_of_page);
- }
-
- if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
- ehca_gen_err("logical_address_of_page not on a 4k boundary "
- "adapter_handle=%llx mr=%p mr_handle=%llx "
- "pagesize=%x queue_type=%x "
- "logical_address_of_page=%llx count=%llx",
- adapter_handle.handle, mr,
- mr->ipz_mr_handle.handle, pagesize, queue_type,
- logical_address_of_page, count);
- ret = H_PARAMETER;
- } else
- ret = hipz_h_register_rpage(adapter_handle, pagesize,
- queue_type,
- mr->ipz_mr_handle.handle,
- logical_address_of_page, count);
- return ret;
-}
-
-u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_QUERY_MR, outs,
- adapter_handle.handle, /* r4 */
- mr->ipz_mr_handle.handle, /* r5 */
- 0, 0, 0, 0, 0, 0, 0);
- outparms->len = outs[0];
- outparms->vaddr = outs[1];
- outparms->acl = outs[4] >> 32;
- outparms->lkey = (u32)(outs[5] >> 32);
- outparms->rkey = (u32)(outs[5] & (0xffffffff));
-
- return ret;
-}
-
-u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr)
-{
- return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- mr->ipz_mr_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-}
-
-u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr_in,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- const u64 mr_addr_cb,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_REREGISTER_PMR, outs,
- adapter_handle.handle, /* r4 */
- mr->ipz_mr_handle.handle, /* r5 */
- vaddr_in, /* r6 */
- length, /* r7 */
- /* r8 */
- ((((u64)access_ctrl) << 32ULL) | pd.value),
- mr_addr_cb, /* r9 */
- 0, 0, 0);
- outparms->vaddr = outs[1];
- outparms->lkey = (u32)outs[2];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const struct ehca_mr *orig_mr,
- const u64 vaddr_in,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_REGISTER_SMR, outs,
- adapter_handle.handle, /* r4 */
- orig_mr->ipz_mr_handle.handle, /* r5 */
- vaddr_in, /* r6 */
- (((u64)access_ctrl) << 32ULL), /* r7 */
- pd.value, /* r8 */
- 0, 0, 0, 0);
- outparms->handle.handle = outs[0];
- outparms->lkey = (u32)outs[2];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- const struct ipz_pd pd,
- struct ehca_mw_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- 6, /* r5 */
- pd.value, /* r6 */
- 0, 0, 0, 0, 0, 0);
- outparms->handle.handle = outs[0];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- struct ehca_mw_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_QUERY_MW, outs,
- adapter_handle.handle, /* r4 */
- mw->ipz_mw_handle.handle, /* r5 */
- 0, 0, 0, 0, 0, 0, 0);
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw)
-{
- return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- mw->ipz_mw_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-}
-
-u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
- const u64 ressource_handle,
- void *rblock,
- unsigned long *byte_count)
-{
- u64 r_cb = __pa(rblock);
-
- if (r_cb & (EHCA_PAGESIZE-1)) {
- ehca_gen_err("rblock not page aligned.");
- return H_PARAMETER;
- }
-
- return ehca_plpar_hcall_norets(H_ERROR_DATA,
- adapter_handle.handle,
- ressource_handle,
- r_cb,
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_eoi(int irq)
-{
- unsigned long xirr;
-
- iosync();
- xirr = (0xffULL << 24) | irq;
-
- return plpar_hcall_norets(H_EOI, xirr);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Firmware Infiniband Interface code for POWER
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Gerd Bayer <gerd.bayer@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HCP_IF_H__
-#define __HCP_IF_H__
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "hipz_hw.h"
-
-/*
- * hipz_h_alloc_resource_eq allocates EQ resources in HW and FW, initialize
- * resources, create the empty EQPT (ring).
- */
-u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_pfeq *pfeq,
- const u32 neq_control,
- const u32 number_of_entries,
- struct ipz_eq_handle *eq_handle,
- u32 * act_nr_of_entries,
- u32 * act_pages,
- u32 * eq_ist);
-
-u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
- struct ipz_eq_handle eq_handle,
- const u64 event_mask);
-/*
- * hipz_h_allocate_resource_cq allocates CQ resources in HW and FW, initialize
- * resources, create the empty CQPT (ring).
- */
-u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- struct ehca_alloc_cq_parms *param);
-
-
-/*
- * hipz_h_alloc_resource_qp allocates QP resources in HW and FW,
- * initialize resources, create empty QPPTs (2 rings).
- */
-u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_alloc_qp_parms *parms, int is_user);
-
-u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id,
- struct hipz_query_port *query_port_response_block);
-
-u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id, const u32 port_cap,
- const u8 init_type, const int modify_mask);
-
-u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
- struct hipz_query_hca *query_hca_rblock);
-
-/*
- * hipz_h_register_rpage internal function in hcp_if.h for all
- * hcp_H_REGISTER_RPAGE calls.
- */
-u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
- const u8 pagesize,
- const u8 queue_type,
- const u64 resource_handle,
- const u64 logical_address_of_page,
- u64 count);
-
-u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_eq_handle eq_handle,
- struct ehca_pfeq *pfeq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count);
-
-u64 hipz_h_query_int_state(const struct ipz_adapter_handle
- hcp_adapter_handle,
- u32 ist);
-
-u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_cq_handle cq_handle,
- struct ehca_pfcq *pfcq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa gal);
-
-u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa galpa);
-
-u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- void **log_addr_next_sq_wqe_tb_processed,
- void **log_addr_next_rq_wqe_tb_processed,
- int dis_and_get_function_code);
-enum hcall_sigt {
- HCALL_SIGT_NO_CQE = 0,
- HCALL_SIGT_BY_WQE = 1,
- HCALL_SIGT_EVERY = 2
-};
-
-u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u64 update_mask,
- struct hcp_modify_qp_control_block *mqpcb,
- struct h_galpa gal);
-
-u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- struct hcp_modify_qp_control_block *qqpcb,
- struct h_galpa gal);
-
-u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_qp *qp);
-
-u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port);
-
-u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port, u32 * pma_qp_nr,
- u32 * bma_qp_nr);
-
-u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id);
-
-u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id);
-
-u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- u8 force_flag);
-
-u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_eq *eq);
-
-/*
- * hipz_h_alloc_resource_mr allocates MR resources in HW and FW, initialize
- * resources.
- */
-u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms);
-
-/* hipz_h_register_rpage_mr registers MR resource pages in HW and FW */
-u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count);
-
-/* hipz_h_query_mr queries MR in HW and FW */
-u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- struct ehca_mr_hipzout_parms *outparms);
-
-/* hipz_h_free_resource_mr frees MR resources in HW and FW */
-u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr);
-
-/* hipz_h_reregister_pmr reregisters MR in HW and FW */
-u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr_in,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- const u64 mr_addr_cb,
- struct ehca_mr_hipzout_parms *outparms);
-
-/* hipz_h_register_smr register shared MR in HW and FW */
-u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const struct ehca_mr *orig_mr,
- const u64 vaddr_in,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms);
-
-/*
- * hipz_h_alloc_resource_mw allocates MW resources in HW and FW, initialize
- * resources.
- */
-u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- const struct ipz_pd pd,
- struct ehca_mw_hipzout_parms *outparms);
-
-/* hipz_h_query_mw queries MW in HW and FW */
-u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- struct ehca_mw_hipzout_parms *outparms);
-
-/* hipz_h_free_resource_mw frees MW resources in HW and FW */
-u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw);
-
-u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
- const u64 ressource_handle,
- void *rblock,
- unsigned long *byte_count);
-u64 hipz_h_eoi(int irq);
-
-#endif /* __HCP_IF_H__ */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * load store abstraction for ehca register access with tracing
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "ehca_classes.h"
-#include "hipz_hw.h"
-
-u64 hcall_map_page(u64 physaddr)
-{
- return (u64)ioremap(physaddr, EHCA_PAGESIZE);
-}
-
-int hcall_unmap_page(u64 mapaddr)
-{
- iounmap((volatile void __iomem *) mapaddr);
- return 0;
-}
-
-int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
- u64 paddr_kernel, u64 paddr_user)
-{
- if (!is_user) {
- galpas->kernel.fw_handle = hcall_map_page(paddr_kernel);
- if (!galpas->kernel.fw_handle)
- return -ENOMEM;
- } else
- galpas->kernel.fw_handle = 0;
-
- galpas->user.fw_handle = paddr_user;
-
- return 0;
-}
-
-int hcp_galpas_dtor(struct h_galpas *galpas)
-{
- if (galpas->kernel.fw_handle) {
- int ret = hcall_unmap_page(galpas->kernel.fw_handle);
- if (ret)
- return ret;
- }
-
- galpas->user.fw_handle = galpas->kernel.fw_handle = 0;
-
- return 0;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Firmware calls
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Gerd Bayer <gerd.bayer@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HCP_PHYP_H__
-#define __HCP_PHYP_H__
-
-
-/*
- * eHCA page (mapped into memory)
- * resource to access eHCA register pages in CPU address space
-*/
-struct h_galpa {
- u64 fw_handle;
- /* for pSeries this is a 64bit memory address where
- I/O memory is mapped into CPU address space (kv) */
-};
-
-/*
- * resource to access eHCA address space registers, all types
- */
-struct h_galpas {
- u32 pid; /*PID of userspace galpa checking */
- struct h_galpa user; /* user space accessible resource,
- set to 0 if unused */
- struct h_galpa kernel; /* kernel space accessible resource,
- set to 0 if unused */
-};
-
-static inline u64 hipz_galpa_load(struct h_galpa galpa, u32 offset)
-{
- u64 addr = galpa.fw_handle + offset;
- return *(volatile u64 __force *)addr;
-}
-
-static inline void hipz_galpa_store(struct h_galpa galpa, u32 offset, u64 value)
-{
- u64 addr = galpa.fw_handle + offset;
- *(volatile u64 __force *)addr = value;
-}
-
-int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
- u64 paddr_kernel, u64 paddr_user);
-
-int hcp_galpas_dtor(struct h_galpas *galpas);
-
-u64 hcall_map_page(u64 physaddr);
-
-int hcall_unmap_page(u64 mapaddr);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * HW abstraction register functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIPZ_FNS_H__
-#define __HIPZ_FNS_H__
-
-#include "ehca_classes.h"
-#include "hipz_hw.h"
-
-#include "hipz_fns_core.h"
-
-#define hipz_galpa_store_eq(gal, offset, value) \
- hipz_galpa_store(gal, EQTEMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_eq(gal, offset) \
- hipz_galpa_load(gal, EQTEMM_OFFSET(offset))
-
-#define hipz_galpa_store_qped(gal, offset, value) \
- hipz_galpa_store(gal, QPEDMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_qped(gal, offset) \
- hipz_galpa_load(gal, QPEDMM_OFFSET(offset))
-
-#define hipz_galpa_store_mrmw(gal, offset, value) \
- hipz_galpa_store(gal, MRMWMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_mrmw(gal, offset) \
- hipz_galpa_load(gal, MRMWMM_OFFSET(offset))
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * HW abstraction register functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIPZ_FNS_CORE_H__
-#define __HIPZ_FNS_CORE_H__
-
-#include "hcp_phyp.h"
-#include "hipz_hw.h"
-
-#define hipz_galpa_store_cq(gal, offset, value) \
- hipz_galpa_store(gal, CQTEMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_cq(gal, offset) \
- hipz_galpa_load(gal, CQTEMM_OFFSET(offset))
-
-#define hipz_galpa_store_qp(gal, offset, value) \
- hipz_galpa_store(gal, QPTEMM_OFFSET(offset), value)
-#define hipz_galpa_load_qp(gal, offset) \
- hipz_galpa_load(gal, QPTEMM_OFFSET(offset))
-
-static inline void hipz_update_sqa(struct ehca_qp *qp, u16 nr_wqes)
-{
- /* ringing doorbell :-) */
- hipz_galpa_store_qp(qp->galpas.kernel, qpx_sqa,
- EHCA_BMASK_SET(QPX_SQADDER, nr_wqes));
-}
-
-static inline void hipz_update_rqa(struct ehca_qp *qp, u16 nr_wqes)
-{
- /* ringing doorbell :-) */
- hipz_galpa_store_qp(qp->galpas.kernel, qpx_rqa,
- EHCA_BMASK_SET(QPX_RQADDER, nr_wqes));
-}
-
-static inline void hipz_update_feca(struct ehca_cq *cq, u32 nr_cqes)
-{
- hipz_galpa_store_cq(cq->galpas.kernel, cqx_feca,
- EHCA_BMASK_SET(CQX_FECADDER, nr_cqes));
-}
-
-static inline void hipz_set_cqx_n0(struct ehca_cq *cq, u32 value)
-{
- u64 cqx_n0_reg;
-
- hipz_galpa_store_cq(cq->galpas.kernel, cqx_n0,
- EHCA_BMASK_SET(CQX_N0_GENERATE_SOLICITED_COMP_EVENT,
- value));
- cqx_n0_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n0);
-}
-
-static inline void hipz_set_cqx_n1(struct ehca_cq *cq, u32 value)
-{
- u64 cqx_n1_reg;
-
- hipz_galpa_store_cq(cq->galpas.kernel, cqx_n1,
- EHCA_BMASK_SET(CQX_N1_GENERATE_COMP_EVENT, value));
- cqx_n1_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n1);
-}
-
-#endif /* __HIPZ_FNC_CORE_H__ */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * eHCA register definitions
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIPZ_HW_H__
-#define __HIPZ_HW_H__
-
-#include "ehca_tools.h"
-
-#define EHCA_MAX_MTU 4
-
-/* QP Table Entry Memory Map */
-struct hipz_qptemm {
- u64 qpx_hcr;
- u64 qpx_c;
- u64 qpx_herr;
- u64 qpx_aer;
-/* 0x20*/
- u64 qpx_sqa;
- u64 qpx_sqc;
- u64 qpx_rqa;
- u64 qpx_rqc;
-/* 0x40*/
- u64 qpx_st;
- u64 qpx_pmstate;
- u64 qpx_pmfa;
- u64 qpx_pkey;
-/* 0x60*/
- u64 qpx_pkeya;
- u64 qpx_pkeyb;
- u64 qpx_pkeyc;
- u64 qpx_pkeyd;
-/* 0x80*/
- u64 qpx_qkey;
- u64 qpx_dqp;
- u64 qpx_dlidp;
- u64 qpx_portp;
-/* 0xa0*/
- u64 qpx_slidp;
- u64 qpx_slidpp;
- u64 qpx_dlida;
- u64 qpx_porta;
-/* 0xc0*/
- u64 qpx_slida;
- u64 qpx_slidpa;
- u64 qpx_slvl;
- u64 qpx_ipd;
-/* 0xe0*/
- u64 qpx_mtu;
- u64 qpx_lato;
- u64 qpx_rlimit;
- u64 qpx_rnrlimit;
-/* 0x100*/
- u64 qpx_t;
- u64 qpx_sqhp;
- u64 qpx_sqptp;
- u64 qpx_nspsn;
-/* 0x120*/
- u64 qpx_nspsnhwm;
- u64 reserved1;
- u64 qpx_sdsi;
- u64 qpx_sdsbc;
-/* 0x140*/
- u64 qpx_sqwsize;
- u64 qpx_sqwts;
- u64 qpx_lsn;
- u64 qpx_nssn;
-/* 0x160 */
- u64 qpx_mor;
- u64 qpx_cor;
- u64 qpx_sqsize;
- u64 qpx_erc;
-/* 0x180*/
- u64 qpx_rnrrc;
- u64 qpx_ernrwt;
- u64 qpx_rnrresp;
- u64 qpx_lmsna;
-/* 0x1a0 */
- u64 qpx_sqhpc;
- u64 qpx_sqcptp;
- u64 qpx_sigt;
- u64 qpx_wqecnt;
-/* 0x1c0*/
- u64 qpx_rqhp;
- u64 qpx_rqptp;
- u64 qpx_rqsize;
- u64 qpx_nrr;
-/* 0x1e0*/
- u64 qpx_rdmac;
- u64 qpx_nrpsn;
- u64 qpx_lapsn;
- u64 qpx_lcr;
-/* 0x200*/
- u64 qpx_rwc;
- u64 qpx_rwva;
- u64 qpx_rdsi;
- u64 qpx_rdsbc;
-/* 0x220*/
- u64 qpx_rqwsize;
- u64 qpx_crmsn;
- u64 qpx_rdd;
- u64 qpx_larpsn;
-/* 0x240*/
- u64 qpx_pd;
- u64 qpx_scqn;
- u64 qpx_rcqn;
- u64 qpx_aeqn;
-/* 0x260*/
- u64 qpx_aaelog;
- u64 qpx_ram;
- u64 qpx_rdmaqe0;
- u64 qpx_rdmaqe1;
-/* 0x280*/
- u64 qpx_rdmaqe2;
- u64 qpx_rdmaqe3;
- u64 qpx_nrpsnhwm;
-/* 0x298*/
- u64 reserved[(0x400 - 0x298) / 8];
-/* 0x400 extended data */
- u64 reserved_ext[(0x500 - 0x400) / 8];
-/* 0x500 */
- u64 reserved2[(0x1000 - 0x500) / 8];
-/* 0x1000 */
-};
-
-#define QPX_SQADDER EHCA_BMASK_IBM(48, 63)
-#define QPX_RQADDER EHCA_BMASK_IBM(48, 63)
-#define QPX_AAELOG_RESET_SRQ_LIMIT EHCA_BMASK_IBM(3, 3)
-
-#define QPTEMM_OFFSET(x) offsetof(struct hipz_qptemm, x)
-
-/* MRMWPT Entry Memory Map */
-struct hipz_mrmwmm {
- /* 0x00 */
- u64 mrx_hcr;
-
- u64 mrx_c;
- u64 mrx_herr;
- u64 mrx_aer;
- /* 0x20 */
- u64 mrx_pp;
- u64 reserved1;
- u64 reserved2;
- u64 reserved3;
- /* 0x40 */
- u64 reserved4[(0x200 - 0x40) / 8];
- /* 0x200 */
- u64 mrx_ctl[64];
-
-};
-
-#define MRMWMM_OFFSET(x) offsetof(struct hipz_mrmwmm, x)
-
-struct hipz_qpedmm {
- /* 0x00 */
- u64 reserved0[(0x400) / 8];
- /* 0x400 */
- u64 qpedx_phh;
- u64 qpedx_ppsgp;
- /* 0x410 */
- u64 qpedx_ppsgu;
- u64 qpedx_ppdgp;
- /* 0x420 */
- u64 qpedx_ppdgu;
- u64 qpedx_aph;
- /* 0x430 */
- u64 qpedx_apsgp;
- u64 qpedx_apsgu;
- /* 0x440 */
- u64 qpedx_apdgp;
- u64 qpedx_apdgu;
- /* 0x450 */
- u64 qpedx_apav;
- u64 qpedx_apsav;
- /* 0x460 */
- u64 qpedx_hcr;
- u64 reserved1[4];
- /* 0x488 */
- u64 qpedx_rrl0;
- /* 0x490 */
- u64 qpedx_rrrkey0;
- u64 qpedx_rrva0;
- /* 0x4a0 */
- u64 reserved2;
- u64 qpedx_rrl1;
- /* 0x4b0 */
- u64 qpedx_rrrkey1;
- u64 qpedx_rrva1;
- /* 0x4c0 */
- u64 reserved3;
- u64 qpedx_rrl2;
- /* 0x4d0 */
- u64 qpedx_rrrkey2;
- u64 qpedx_rrva2;
- /* 0x4e0 */
- u64 reserved4;
- u64 qpedx_rrl3;
- /* 0x4f0 */
- u64 qpedx_rrrkey3;
- u64 qpedx_rrva3;
-};
-
-#define QPEDMM_OFFSET(x) offsetof(struct hipz_qpedmm, x)
-
-/* CQ Table Entry Memory Map */
-struct hipz_cqtemm {
- u64 cqx_hcr;
- u64 cqx_c;
- u64 cqx_herr;
- u64 cqx_aer;
-/* 0x20 */
- u64 cqx_ptp;
- u64 cqx_tp;
- u64 cqx_fec;
- u64 cqx_feca;
-/* 0x40 */
- u64 cqx_ep;
- u64 cqx_eq;
-/* 0x50 */
- u64 reserved1;
- u64 cqx_n0;
-/* 0x60 */
- u64 cqx_n1;
- u64 reserved2[(0x1000 - 0x60) / 8];
-/* 0x1000 */
-};
-
-#define CQX_FEC_CQE_CNT EHCA_BMASK_IBM(32, 63)
-#define CQX_FECADDER EHCA_BMASK_IBM(32, 63)
-#define CQX_N0_GENERATE_SOLICITED_COMP_EVENT EHCA_BMASK_IBM(0, 0)
-#define CQX_N1_GENERATE_COMP_EVENT EHCA_BMASK_IBM(0, 0)
-
-#define CQTEMM_OFFSET(x) offsetof(struct hipz_cqtemm, x)
-
-/* EQ Table Entry Memory Map */
-struct hipz_eqtemm {
- u64 eqx_hcr;
- u64 eqx_c;
-
- u64 eqx_herr;
- u64 eqx_aer;
-/* 0x20 */
- u64 eqx_ptp;
- u64 eqx_tp;
- u64 eqx_ssba;
- u64 eqx_psba;
-
-/* 0x40 */
- u64 eqx_cec;
- u64 eqx_meql;
- u64 eqx_xisbi;
- u64 eqx_xisc;
-/* 0x60 */
- u64 eqx_it;
-
-};
-
-#define EQTEMM_OFFSET(x) offsetof(struct hipz_eqtemm, x)
-
-/* access control defines for MR/MW */
-#define HIPZ_ACCESSCTRL_L_WRITE 0x00800000
-#define HIPZ_ACCESSCTRL_R_WRITE 0x00400000
-#define HIPZ_ACCESSCTRL_R_READ 0x00200000
-#define HIPZ_ACCESSCTRL_R_ATOMIC 0x00100000
-#define HIPZ_ACCESSCTRL_MW_BIND 0x00080000
-
-/* query hca response block */
-struct hipz_query_hca {
- u32 cur_reliable_dg;
- u32 cur_qp;
- u32 cur_cq;
- u32 cur_eq;
- u32 cur_mr;
- u32 cur_mw;
- u32 cur_ee_context;
- u32 cur_mcast_grp;
- u32 cur_qp_attached_mcast_grp;
- u32 reserved1;
- u32 cur_ipv6_qp;
- u32 cur_eth_qp;
- u32 cur_hp_mr;
- u32 reserved2[3];
- u32 max_rd_domain;
- u32 max_qp;
- u32 max_cq;
- u32 max_eq;
- u32 max_mr;
- u32 max_hp_mr;
- u32 max_mw;
- u32 max_mrwpte;
- u32 max_special_mrwpte;
- u32 max_rd_ee_context;
- u32 max_mcast_grp;
- u32 max_total_mcast_qp_attach;
- u32 max_mcast_qp_attach;
- u32 max_raw_ipv6_qp;
- u32 max_raw_ethy_qp;
- u32 internal_clock_frequency;
- u32 max_pd;
- u32 max_ah;
- u32 max_cqe;
- u32 max_wqes_wq;
- u32 max_partitions;
- u32 max_rr_ee_context;
- u32 max_rr_qp;
- u32 max_rr_hca;
- u32 max_act_wqs_ee_context;
- u32 max_act_wqs_qp;
- u32 max_sge;
- u32 max_sge_rd;
- u32 memory_page_size_supported;
- u64 max_mr_size;
- u32 local_ca_ack_delay;
- u32 num_ports;
- u32 vendor_id;
- u32 vendor_part_id;
- u32 hw_ver;
- u64 node_guid;
- u64 hca_cap_indicators;
- u32 data_counter_register_size;
- u32 max_shared_rq;
- u32 max_isns_eq;
- u32 max_neq;
-} __attribute__ ((packed));
-
-#define HCA_CAP_AH_PORT_NR_CHECK EHCA_BMASK_IBM( 0, 0)
-#define HCA_CAP_ATOMIC EHCA_BMASK_IBM( 1, 1)
-#define HCA_CAP_AUTO_PATH_MIG EHCA_BMASK_IBM( 2, 2)
-#define HCA_CAP_BAD_P_KEY_CTR EHCA_BMASK_IBM( 3, 3)
-#define HCA_CAP_SQD_RTS_PORT_CHANGE EHCA_BMASK_IBM( 4, 4)
-#define HCA_CAP_CUR_QP_STATE_MOD EHCA_BMASK_IBM( 5, 5)
-#define HCA_CAP_INIT_TYPE EHCA_BMASK_IBM( 6, 6)
-#define HCA_CAP_PORT_ACTIVE_EVENT EHCA_BMASK_IBM( 7, 7)
-#define HCA_CAP_Q_KEY_VIOL_CTR EHCA_BMASK_IBM( 8, 8)
-#define HCA_CAP_WQE_RESIZE EHCA_BMASK_IBM( 9, 9)
-#define HCA_CAP_RAW_PACKET_MCAST EHCA_BMASK_IBM(10, 10)
-#define HCA_CAP_SHUTDOWN_PORT EHCA_BMASK_IBM(11, 11)
-#define HCA_CAP_RC_LL_QP EHCA_BMASK_IBM(12, 12)
-#define HCA_CAP_SRQ EHCA_BMASK_IBM(13, 13)
-#define HCA_CAP_UD_LL_QP EHCA_BMASK_IBM(16, 16)
-#define HCA_CAP_RESIZE_MR EHCA_BMASK_IBM(17, 17)
-#define HCA_CAP_MINI_QP EHCA_BMASK_IBM(18, 18)
-#define HCA_CAP_H_ALLOC_RES_SYNC EHCA_BMASK_IBM(19, 19)
-
-/* query port response block */
-struct hipz_query_port {
- u32 state;
- u32 bad_pkey_cntr;
- u32 lmc;
- u32 lid;
- u32 subnet_timeout;
- u32 qkey_viol_cntr;
- u32 sm_sl;
- u32 sm_lid;
- u32 capability_mask;
- u32 init_type_reply;
- u32 pkey_tbl_len;
- u32 gid_tbl_len;
- u64 gid_prefix;
- u32 port_nr;
- u16 pkey_entries[16];
- u8 reserved1[32];
- u32 trent_size;
- u32 trbuf_size;
- u64 max_msg_sz;
- u32 max_mtu;
- u32 vl_cap;
- u32 phys_pstate;
- u32 phys_state;
- u32 phys_speed;
- u32 phys_width;
- u8 reserved2[1884];
- u64 guid_entries[255];
-} __attribute__ ((packed));
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * internal queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_tools.h"
-#include "ipz_pt_fn.h"
-#include "ehca_classes.h"
-
-#define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT)
-
-struct kmem_cache *small_qp_cache;
-
-void *ipz_qpageit_get_inc(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- queue->current_q_offset += queue->pagesize;
- if (queue->current_q_offset > queue->queue_length) {
- queue->current_q_offset -= queue->pagesize;
- ret = NULL;
- }
- if (((u64)ret) % queue->pagesize) {
- ehca_gen_err("ERROR!! not at PAGE-Boundary");
- return NULL;
- }
- return ret;
-}
-
-void *ipz_qeit_eq_get_inc(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- u64 last_entry_in_q = queue->queue_length - queue->qe_size;
-
- queue->current_q_offset += queue->qe_size;
- if (queue->current_q_offset > last_entry_in_q) {
- queue->current_q_offset = 0;
- queue->toggle_state = (~queue->toggle_state) & 1;
- }
-
- return ret;
-}
-
-int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset)
-{
- int i;
- for (i = 0; i < queue->queue_length / queue->pagesize; i++) {
- u64 page = __pa(queue->queue_pages[i]);
- if (addr >= page && addr < page + queue->pagesize) {
- *q_offset = addr - page + i * queue->pagesize;
- return 0;
- }
- }
- return -EINVAL;
-}
-
-#if PAGE_SHIFT < EHCA_PAGESHIFT
-#error Kernel pages must be at least as large than eHCA pages (4K) !
-#endif
-
-/*
- * allocate pages for queue:
- * outer loop allocates whole kernel pages (page aligned) and
- * inner loop divides a kernel page into smaller hca queue pages
- */
-static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages)
-{
- int k, f = 0;
- u8 *kpage;
-
- while (f < nr_of_pages) {
- kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
- if (!kpage)
- goto out;
-
- for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) {
- queue->queue_pages[f] = (struct ipz_page *)kpage;
- kpage += EHCA_PAGESIZE;
- f++;
- }
- }
- return 1;
-
-out:
- for (f = 0; f < nr_of_pages && queue->queue_pages[f];
- f += PAGES_PER_KPAGE)
- free_page((unsigned long)(queue->queue_pages)[f]);
- return 0;
-}
-
-static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
-{
- int order = ilog2(queue->pagesize) - 9;
- struct ipz_small_queue_page *page;
- unsigned long bit;
-
- mutex_lock(&pd->lock);
-
- if (!list_empty(&pd->free[order]))
- page = list_entry(pd->free[order].next,
- struct ipz_small_queue_page, list);
- else {
- page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL);
- if (!page)
- goto out;
-
- page->page = get_zeroed_page(GFP_KERNEL);
- if (!page->page) {
- kmem_cache_free(small_qp_cache, page);
- goto out;
- }
-
- list_add(&page->list, &pd->free[order]);
- }
-
- bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order);
- __set_bit(bit, page->bitmap);
- page->fill++;
-
- if (page->fill == IPZ_SPAGE_PER_KPAGE >> order)
- list_move(&page->list, &pd->full[order]);
-
- mutex_unlock(&pd->lock);
-
- queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9)));
- queue->small_page = page;
- queue->offset = bit << (order + 9);
- return 1;
-
-out:
- ehca_err(pd->ib_pd.device, "failed to allocate small queue page");
- mutex_unlock(&pd->lock);
- return 0;
-}
-
-static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
-{
- int order = ilog2(queue->pagesize) - 9;
- struct ipz_small_queue_page *page = queue->small_page;
- unsigned long bit;
- int free_page = 0;
-
- bit = ((unsigned long)queue->queue_pages[0] & ~PAGE_MASK)
- >> (order + 9);
-
- mutex_lock(&pd->lock);
-
- __clear_bit(bit, page->bitmap);
- page->fill--;
-
- if (page->fill == 0) {
- list_del(&page->list);
- free_page = 1;
- }
-
- if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1)
- /* the page was full until we freed the chunk */
- list_move_tail(&page->list, &pd->free[order]);
-
- mutex_unlock(&pd->lock);
-
- if (free_page) {
- free_page(page->page);
- kmem_cache_free(small_qp_cache, page);
- }
-}
-
-int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
- const u32 nr_of_pages, const u32 pagesize,
- const u32 qe_size, const u32 nr_of_sg,
- int is_small)
-{
- if (pagesize > PAGE_SIZE) {
- ehca_gen_err("FATAL ERROR: pagesize=%x "
- "is greater than kernel page size", pagesize);
- return 0;
- }
-
- /* init queue fields */
- queue->queue_length = nr_of_pages * pagesize;
- queue->pagesize = pagesize;
- queue->qe_size = qe_size;
- queue->act_nr_of_sg = nr_of_sg;
- queue->current_q_offset = 0;
- queue->toggle_state = 1;
- queue->small_page = NULL;
-
- /* allocate queue page pointers */
- queue->queue_pages = kzalloc(nr_of_pages * sizeof(void *),
- GFP_KERNEL | __GFP_NOWARN);
- if (!queue->queue_pages) {
- queue->queue_pages = vzalloc(nr_of_pages * sizeof(void *));
- if (!queue->queue_pages) {
- ehca_gen_err("Couldn't allocate queue page list");
- return 0;
- }
- }
-
- /* allocate actual queue pages */
- if (is_small) {
- if (!alloc_small_queue_page(queue, pd))
- goto ipz_queue_ctor_exit0;
- } else
- if (!alloc_queue_pages(queue, nr_of_pages))
- goto ipz_queue_ctor_exit0;
-
- return 1;
-
-ipz_queue_ctor_exit0:
- ehca_gen_err("Couldn't alloc pages queue=%p "
- "nr_of_pages=%x", queue, nr_of_pages);
- kvfree(queue->queue_pages);
-
- return 0;
-}
-
-int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue)
-{
- int i, nr_pages;
-
- if (!queue || !queue->queue_pages) {
- ehca_gen_dbg("queue or queue_pages is NULL");
- return 0;
- }
-
- if (queue->small_page)
- free_small_queue_page(queue, pd);
- else {
- nr_pages = queue->queue_length / queue->pagesize;
- for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE)
- free_page((unsigned long)queue->queue_pages[i]);
- }
-
- kvfree(queue->queue_pages);
-
- return 1;
-}
-
-int ehca_init_small_qp_cache(void)
-{
- small_qp_cache = kmem_cache_create("ehca_cache_small_qp",
- sizeof(struct ipz_small_queue_page),
- 0, SLAB_HWCACHE_ALIGN, NULL);
- if (!small_qp_cache)
- return -ENOMEM;
-
- return 0;
-}
-
-void ehca_cleanup_small_qp_cache(void)
-{
- kmem_cache_destroy(small_qp_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * internal queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __IPZ_PT_FN_H__
-#define __IPZ_PT_FN_H__
-
-#define EHCA_PAGESHIFT 12
-#define EHCA_PAGESIZE 4096UL
-#define EHCA_PAGEMASK (~(EHCA_PAGESIZE-1))
-#define EHCA_PT_ENTRIES 512UL
-
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-
-struct ehca_pd;
-struct ipz_small_queue_page;
-
-extern struct kmem_cache *small_qp_cache;
-
-/* struct generic ehca page */
-struct ipz_page {
- u8 entries[EHCA_PAGESIZE];
-};
-
-#define IPZ_SPAGE_PER_KPAGE (PAGE_SIZE / 512)
-
-struct ipz_small_queue_page {
- unsigned long page;
- unsigned long bitmap[IPZ_SPAGE_PER_KPAGE / BITS_PER_LONG];
- int fill;
- void *mapped_addr;
- u32 mmap_count;
- struct list_head list;
-};
-
-/* struct generic queue in linux kernel virtual memory (kv) */
-struct ipz_queue {
- u64 current_q_offset; /* current queue entry */
-
- struct ipz_page **queue_pages; /* array of pages belonging to queue */
- u32 qe_size; /* queue entry size */
- u32 act_nr_of_sg;
- u32 queue_length; /* queue length allocated in bytes */
- u32 pagesize;
- u32 toggle_state; /* toggle flag - per page */
- u32 offset; /* save offset within page for small_qp */
- struct ipz_small_queue_page *small_page;
-};
-
-/*
- * return current Queue Entry for a certain q_offset
- * returns address (kv) of Queue Entry
- */
-static inline void *ipz_qeit_calc(struct ipz_queue *queue, u64 q_offset)
-{
- struct ipz_page *current_page;
- if (q_offset >= queue->queue_length)
- return NULL;
- current_page = (queue->queue_pages)[q_offset >> EHCA_PAGESHIFT];
- return ¤t_page->entries[q_offset & (EHCA_PAGESIZE - 1)];
-}
-
-/*
- * return current Queue Entry
- * returns address (kv) of Queue Entry
- */
-static inline void *ipz_qeit_get(struct ipz_queue *queue)
-{
- return ipz_qeit_calc(queue, queue->current_q_offset);
-}
-
-/*
- * return current Queue Page , increment Queue Page iterator from
- * page to page in struct ipz_queue, last increment will return 0! and
- * NOT wrap
- * returns address (kv) of Queue Page
- * warning don't use in parallel with ipz_QE_get_inc()
- */
-void *ipz_qpageit_get_inc(struct ipz_queue *queue);
-
-/*
- * return current Queue Entry, increment Queue Entry iterator by one
- * step in struct ipz_queue, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * warning don't use in parallel with ipz_qpageit_get_inc()
- */
-static inline void *ipz_qeit_get_inc(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- queue->current_q_offset += queue->qe_size;
- if (queue->current_q_offset >= queue->queue_length) {
- queue->current_q_offset = 0;
- /* toggle the valid flag */
- queue->toggle_state = (~queue->toggle_state) & 1;
- }
-
- return ret;
-}
-
-/*
- * return a bool indicating whether current Queue Entry is valid
- */
-static inline int ipz_qeit_is_valid(struct ipz_queue *queue)
-{
- struct ehca_cqe *cqe = ipz_qeit_get(queue);
- return ((cqe->cqe_flags >> 7) == (queue->toggle_state & 1));
-}
-
-/*
- * return current Queue Entry, increment Queue Entry iterator by one
- * step in struct ipz_queue, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * returns 0 and does not increment, if wrong valid state
- * warning don't use in parallel with ipz_qpageit_get_inc()
- */
-static inline void *ipz_qeit_get_inc_valid(struct ipz_queue *queue)
-{
- return ipz_qeit_is_valid(queue) ? ipz_qeit_get_inc(queue) : NULL;
-}
-
-/*
- * returns and resets Queue Entry iterator
- * returns address (kv) of first Queue Entry
- */
-static inline void *ipz_qeit_reset(struct ipz_queue *queue)
-{
- queue->current_q_offset = 0;
- return ipz_qeit_get(queue);
-}
-
-/*
- * return the q_offset corresponding to an absolute address
- */
-int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset);
-
-/*
- * return the next queue offset. don't modify the queue.
- */
-static inline u64 ipz_queue_advance_offset(struct ipz_queue *queue, u64 offset)
-{
- offset += queue->qe_size;
- if (offset >= queue->queue_length) offset = 0;
- return offset;
-}
-
-/* struct generic page table */
-struct ipz_pt {
- u64 entries[EHCA_PT_ENTRIES];
-};
-
-/* struct page table for a queue, only to be used in pf */
-struct ipz_qpt {
- /* queue page tables (kv), use u64 because we know the element length */
- u64 *qpts;
- u32 n_qpts;
- u32 n_ptes; /* number of page table entries */
- u64 *current_pte_addr;
-};
-
-/*
- * constructor for a ipz_queue_t, placement new for ipz_queue_t,
- * new for all dependent datastructors
- * all QP Tables are the same
- * flow:
- * allocate+pin queue
- * see ipz_qpt_ctor()
- * returns true if ok, false if out of memory
- */
-int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
- const u32 nr_of_pages, const u32 pagesize,
- const u32 qe_size, const u32 nr_of_sg,
- int is_small);
-
-/*
- * destructor for a ipz_queue_t
- * -# free queue
- * see ipz_queue_ctor()
- * returns true if ok, false if queue was NULL-ptr of free failed
- */
-int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue);
-
-/*
- * constructor for a ipz_qpt_t,
- * placement new for struct ipz_queue, new for all dependent datastructors
- * all QP Tables are the same,
- * flow:
- * -# allocate+pin queue
- * -# initialise ptcb
- * -# allocate+pin PTs
- * -# link PTs to a ring, according to HCA Arch, set bit62 id needed
- * -# the ring must have room for exactly nr_of_PTEs
- * see ipz_qpt_ctor()
- */
-void ipz_qpt_ctor(struct ipz_qpt *qpt,
- const u32 nr_of_qes,
- const u32 pagesize,
- const u32 qe_size,
- const u8 lowbyte, const u8 toggle,
- u32 * act_nr_of_QEs, u32 * act_nr_of_pages);
-
-/*
- * return current Queue Entry, increment Queue Entry iterator by one
- * step in struct ipz_queue, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * warning don't use in parallel with ipz_qpageit_get_inc()
- * warning unpredictable results may occur if steps>act_nr_of_queue_entries
- * fix EQ page problems
- */
-void *ipz_qeit_eq_get_inc(struct ipz_queue *queue);
-
-/*
- * return current Event Queue Entry, increment Queue Entry iterator
- * by one step in struct ipz_queue if valid, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * returns 0 and does not increment, if wrong valid state
- * warning don't use in parallel with ipz_queue_QPageit_get_inc()
- * warning unpredictable results may occur if steps>act_nr_of_queue_entries
- */
-static inline void *ipz_eqit_eq_get_inc_valid(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- u32 qe = *(u8 *)ret;
- if ((qe >> 7) != (queue->toggle_state & 1))
- return NULL;
- ipz_qeit_eq_get_inc(queue); /* this is a good one */
- return ret;
-}
-
-static inline void *ipz_eqit_eq_peek_valid(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- u32 qe = *(u8 *)ret;
- if ((qe >> 7) != (queue->toggle_state & 1))
- return NULL;
- return ret;
-}
-
-/* returns address (GX) of first queue entry */
-static inline u64 ipz_qpt_get_firstpage(struct ipz_qpt *qpt)
-{
- return be64_to_cpu(qpt->qpts[0]);
-}
-
-/* returns address (kv) of first page of queue page table */
-static inline void *ipz_qpt_get_qpt(struct ipz_qpt *qpt)
-{
- return qpt->qpts;
-}
-
-#endif /* __IPZ_PT_FN_H__ */
props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
if (MLX5_CAP_GEN(mdev, apm))
props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
- props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
if (MLX5_CAP_GEN(mdev, xrc))
props->device_cap_flags |= IB_DEVICE_XRC;
props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
return 0;
}
-static int alloc_pa_mkey(struct mlx5_ib_dev *dev, u32 *key, u32 pdn)
-{
- struct mlx5_create_mkey_mbox_in *in;
- struct mlx5_mkey_seg *seg;
- struct mlx5_core_mr mr;
- int err;
-
- in = kzalloc(sizeof(*in), GFP_KERNEL);
- if (!in)
- return -ENOMEM;
-
- seg = &in->seg;
- seg->flags = MLX5_PERM_LOCAL_READ | MLX5_ACCESS_MODE_PA;
- seg->flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
- seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
- seg->start_addr = 0;
-
- err = mlx5_core_create_mkey(dev->mdev, &mr, in, sizeof(*in),
- NULL, NULL, NULL);
- if (err) {
- mlx5_ib_warn(dev, "failed to create mkey, %d\n", err);
- goto err_in;
- }
-
- kfree(in);
- *key = mr.key;
-
- return 0;
-
-err_in:
- kfree(in);
-
- return err;
-}
-
-static void free_pa_mkey(struct mlx5_ib_dev *dev, u32 key)
-{
- struct mlx5_core_mr mr;
- int err;
-
- memset(&mr, 0, sizeof(mr));
- mr.key = key;
- err = mlx5_core_destroy_mkey(dev->mdev, &mr);
- if (err)
- mlx5_ib_warn(dev, "failed to destroy mkey 0x%x\n", key);
-}
-
static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
kfree(pd);
return ERR_PTR(-EFAULT);
}
- } else {
- err = alloc_pa_mkey(to_mdev(ibdev), &pd->pa_lkey, pd->pdn);
- if (err) {
- mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
- kfree(pd);
- return ERR_PTR(err);
- }
}
return &pd->ibpd;
struct mlx5_ib_dev *mdev = to_mdev(pd->device);
struct mlx5_ib_pd *mpd = to_mpd(pd);
- if (!pd->uobject)
- free_pa_mkey(mdev, mpd->pa_lkey);
-
mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
kfree(mpd);
struct ib_srq_init_attr attr;
struct mlx5_ib_dev *dev;
struct ib_cq_init_attr cq_attr = {.cqe = 1};
- u32 rsvd_lkey;
int ret = 0;
dev = container_of(devr, struct mlx5_ib_dev, devr);
- ret = mlx5_core_query_special_context(dev->mdev, &rsvd_lkey);
- if (ret) {
- pr_err("Failed to query special context %d\n", ret);
- return ret;
- }
- dev->ib_dev.local_dma_lkey = rsvd_lkey;
-
devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
if (IS_ERR(devr->p0)) {
ret = PTR_ERR(devr->p0);
strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX);
dev->ib_dev.owner = THIS_MODULE;
dev->ib_dev.node_type = RDMA_NODE_IB_CA;
+ dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
dev->num_ports = MLX5_CAP_GEN(mdev, num_ports);
dev->ib_dev.phys_port_cnt = dev->num_ports;
dev->ib_dev.num_comp_vectors =
struct mlx5_ib_pd {
struct ib_pd ibpd;
u32 pdn;
- u32 pa_lkey;
};
/* Use macros here so that don't have to duplicate
int uuarn;
int create_type;
- u32 pa_lkey;
/* Store signature errors */
bool signature_en;
err = create_kernel_qp(dev, init_attr, qp, &in, &inlen);
if (err)
mlx5_ib_dbg(dev, "err %d\n", err);
- else
- qp->pa_lkey = to_mpd(pd)->pa_lkey;
}
if (err)
mfrpl->mapped_page_list[i] = cpu_to_be64(page_list[i] | perm);
dseg->addr = cpu_to_be64(mfrpl->map);
dseg->byte_count = cpu_to_be32(ALIGN(sizeof(u64) * wr->wr.fast_reg.page_list_len, 64));
- dseg->lkey = cpu_to_be32(pd->pa_lkey);
+ dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
}
static __be32 send_ieth(struct ib_send_wr *wr)
IPOIB_NUM_WC = 4,
IPOIB_MAX_PATH_REC_QUEUE = 3,
- IPOIB_MAX_MCAST_QUEUE = 3,
+ IPOIB_MAX_MCAST_QUEUE = 64,
IPOIB_FLAG_OPER_UP = 0,
IPOIB_FLAG_INITIALIZED = 1,
int ipoib_mcast_attach(struct net_device *dev, u16 mlid,
union ib_gid *mgid, int set_qkey);
+int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast);
+struct ipoib_mcast *__ipoib_mcast_find(struct net_device *dev, void *mgid);
int ipoib_init_qp(struct net_device *dev);
int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca);
unsigned long dt;
unsigned long flags;
int i;
+ LIST_HEAD(remove_list);
+ struct ipoib_mcast *mcast, *tmcast;
+ struct net_device *dev = priv->dev;
if (test_bit(IPOIB_STOP_NEIGH_GC, &priv->flags))
return;
lockdep_is_held(&priv->lock))) != NULL) {
/* was the neigh idle for two GC periods */
if (time_after(neigh_obsolete, neigh->alive)) {
+ u8 *mgid = neigh->daddr + 4;
+
+ /* Is this multicast ? */
+ if (*mgid == 0xff) {
+ mcast = __ipoib_mcast_find(dev, mgid);
+
+ if (mcast && test_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags)) {
+ list_del(&mcast->list);
+ rb_erase(&mcast->rb_node, &priv->multicast_tree);
+ list_add_tail(&mcast->list, &remove_list);
+ }
+ }
+
rcu_assign_pointer(*np,
rcu_dereference_protected(neigh->hnext,
lockdep_is_held(&priv->lock)));
out_unlock:
spin_unlock_irqrestore(&priv->lock, flags);
+ list_for_each_entry_safe(mcast, tmcast, &remove_list, list)
+ ipoib_mcast_leave(dev, mcast);
}
static void ipoib_reap_neigh(struct work_struct *work)
return mcast;
}
-static struct ipoib_mcast *__ipoib_mcast_find(struct net_device *dev, void *mgid)
+struct ipoib_mcast *__ipoib_mcast_find(struct net_device *dev, void *mgid)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
struct rb_node *n = priv->multicast_tree.rb_node;
rec.hop_limit = priv->broadcast->mcmember.hop_limit;
/*
- * Historically Linux IPoIB has never properly supported SEND
- * ONLY join. It emulated it by not providing all the required
- * attributes, which is enough to prevent group creation and
- * detect if there are full members or not. A major problem
- * with supporting SEND ONLY is detecting when the group is
- * auto-destroyed as IPoIB will cache the MLID..
+ * Send-only IB Multicast joins do not work at the core
+ * IB layer yet, so we can't use them here. However,
+ * we are emulating an Ethernet multicast send, which
+ * does not require a multicast subscription and will
+ * still send properly. The most appropriate thing to
+ * do is to create the group if it doesn't exist as that
+ * most closely emulates the behavior, from a user space
+ * application perspecitive, of Ethernet multicast
+ * operation. For now, we do a full join, maybe later
+ * when the core IB layers support send only joins we
+ * will use them.
*/
-#if 1
- if (test_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags))
- comp_mask &= ~IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
-#else
+#if 0
if (test_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags))
rec.join_state = 4;
#endif
return 0;
}
-static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
+int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
int ret = 0;
module_param_named(max_sectors, iser_max_sectors, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(max_sectors, "Max number of sectors in a single scsi command (default:1024");
+bool iser_always_reg = true;
+module_param_named(always_register, iser_always_reg, bool, S_IRUGO);
+MODULE_PARM_DESC(always_register,
+ "Always register memory, even for continuous memory regions (default:true)");
+
bool iser_pi_enable = false;
module_param_named(pi_enable, iser_pi_enable, bool, S_IRUGO);
MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)");
extern bool iser_pi_enable;
extern int iser_pi_guard;
extern unsigned int iser_max_sectors;
+extern bool iser_always_reg;
int iser_assign_reg_ops(struct iser_device *device);
iser_reg_prot_sg(struct iscsi_iser_task *task,
struct iser_data_buf *mem,
struct iser_fr_desc *desc,
+ bool use_dma_key,
struct iser_mem_reg *reg)
{
struct iser_device *device = task->iser_conn->ib_conn.device;
- if (mem->dma_nents == 1)
+ if (use_dma_key)
return iser_reg_dma(device, mem, reg);
return device->reg_ops->reg_mem(task, mem, &desc->pi_ctx->rsc, reg);
iser_reg_data_sg(struct iscsi_iser_task *task,
struct iser_data_buf *mem,
struct iser_fr_desc *desc,
+ bool use_dma_key,
struct iser_mem_reg *reg)
{
struct iser_device *device = task->iser_conn->ib_conn.device;
- if (mem->dma_nents == 1)
+ if (use_dma_key)
return iser_reg_dma(device, mem, reg);
return device->reg_ops->reg_mem(task, mem, &desc->rsc, reg);
struct iser_mem_reg *reg = &task->rdma_reg[dir];
struct iser_mem_reg *data_reg;
struct iser_fr_desc *desc = NULL;
+ bool use_dma_key;
int err;
err = iser_handle_unaligned_buf(task, mem, dir);
if (unlikely(err))
return err;
- if (mem->dma_nents != 1 ||
- scsi_get_prot_op(task->sc) != SCSI_PROT_NORMAL) {
+ use_dma_key = (mem->dma_nents == 1 && !iser_always_reg &&
+ scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL);
+
+ if (!use_dma_key) {
desc = device->reg_ops->reg_desc_get(ib_conn);
reg->mem_h = desc;
}
else
data_reg = &task->desc.data_reg;
- err = iser_reg_data_sg(task, mem, desc, data_reg);
+ err = iser_reg_data_sg(task, mem, desc, use_dma_key, data_reg);
if (unlikely(err))
goto err_reg;
if (unlikely(err))
goto err_reg;
- err = iser_reg_prot_sg(task, mem, desc, prot_reg);
+ err = iser_reg_prot_sg(task, mem, desc,
+ use_dma_key, prot_reg);
if (unlikely(err))
goto err_reg;
}
(unsigned long)comp);
}
- device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE |
- IB_ACCESS_REMOTE_WRITE |
- IB_ACCESS_REMOTE_READ);
- if (IS_ERR(device->mr))
- goto dma_mr_err;
+ if (!iser_always_reg) {
+ int access = IB_ACCESS_LOCAL_WRITE |
+ IB_ACCESS_REMOTE_WRITE |
+ IB_ACCESS_REMOTE_READ;
+
+ device->mr = ib_get_dma_mr(device->pd, access);
+ if (IS_ERR(device->mr))
+ goto dma_mr_err;
+ }
INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device,
iser_event_handler);
return 0;
handler_err:
- ib_dereg_mr(device->mr);
+ if (device->mr)
+ ib_dereg_mr(device->mr);
dma_mr_err:
for (i = 0; i < device->comps_used; i++)
tasklet_kill(&device->comps[i].tasklet);
static void iser_free_device_ib_res(struct iser_device *device)
{
int i;
- BUG_ON(device->mr == NULL);
for (i = 0; i < device->comps_used; i++) {
struct iser_comp *comp = &device->comps[i];
}
(void)ib_unregister_event_handler(&device->event_handler);
- (void)ib_dereg_mr(device->mr);
+ if (device->mr)
+ (void)ib_dereg_mr(device->mr);
ib_dealloc_pd(device->pd);
kfree(device->comps);
rx_sg->lkey = device->pd->local_dma_lkey;
}
- isert_conn->rx_desc_head = 0;
-
return 0;
dma_map_fail:
isert_init_conn(struct isert_conn *isert_conn)
{
isert_conn->state = ISER_CONN_INIT;
- INIT_LIST_HEAD(&isert_conn->accept_node);
+ INIT_LIST_HEAD(&isert_conn->node);
init_completion(&isert_conn->login_comp);
init_completion(&isert_conn->login_req_comp);
init_completion(&isert_conn->wait);
ret = isert_rdma_post_recvl(isert_conn);
if (ret)
goto out_conn_dev;
- /*
- * Obtain the second reference now before isert_rdma_accept() to
- * ensure that any initiator generated REJECT CM event that occurs
- * asynchronously won't drop the last reference until the error path
- * in iscsi_target_login_sess_out() does it's ->iscsit_free_conn() ->
- * isert_free_conn() -> isert_put_conn() -> kref_put().
- */
- if (!kref_get_unless_zero(&isert_conn->kref)) {
- isert_warn("conn %p connect_release is running\n", isert_conn);
- goto out_conn_dev;
- }
ret = isert_rdma_accept(isert_conn);
if (ret)
goto out_conn_dev;
- mutex_lock(&isert_np->np_accept_mutex);
- list_add_tail(&isert_conn->accept_node, &isert_np->np_accept_list);
- mutex_unlock(&isert_np->np_accept_mutex);
+ mutex_lock(&isert_np->mutex);
+ list_add_tail(&isert_conn->node, &isert_np->accepted);
+ mutex_unlock(&isert_np->mutex);
- isert_info("np %p: Allow accept_np to continue\n", np);
- up(&isert_np->np_sem);
return 0;
out_conn_dev:
isert_connected_handler(struct rdma_cm_id *cma_id)
{
struct isert_conn *isert_conn = cma_id->qp->qp_context;
+ struct isert_np *isert_np = cma_id->context;
isert_info("conn %p\n", isert_conn);
mutex_lock(&isert_conn->mutex);
- if (isert_conn->state != ISER_CONN_FULL_FEATURE)
- isert_conn->state = ISER_CONN_UP;
+ isert_conn->state = ISER_CONN_UP;
+ kref_get(&isert_conn->kref);
mutex_unlock(&isert_conn->mutex);
+
+ mutex_lock(&isert_np->mutex);
+ list_move_tail(&isert_conn->node, &isert_np->pending);
+ mutex_unlock(&isert_np->mutex);
+
+ isert_info("np %p: Allow accept_np to continue\n", isert_np);
+ up(&isert_np->sem);
}
static void
switch (event) {
case RDMA_CM_EVENT_DEVICE_REMOVAL:
- isert_np->np_cm_id = NULL;
+ isert_np->cm_id = NULL;
break;
case RDMA_CM_EVENT_ADDR_CHANGE:
- isert_np->np_cm_id = isert_setup_id(isert_np);
- if (IS_ERR(isert_np->np_cm_id)) {
+ isert_np->cm_id = isert_setup_id(isert_np);
+ if (IS_ERR(isert_np->cm_id)) {
isert_err("isert np %p setup id failed: %ld\n",
- isert_np, PTR_ERR(isert_np->np_cm_id));
- isert_np->np_cm_id = NULL;
+ isert_np, PTR_ERR(isert_np->cm_id));
+ isert_np->cm_id = NULL;
}
break;
default:
struct isert_conn *isert_conn;
bool terminating = false;
- if (isert_np->np_cm_id == cma_id)
+ if (isert_np->cm_id == cma_id)
return isert_np_cma_handler(cma_id->context, event);
isert_conn = cma_id->qp->qp_context;
if (terminating)
goto out;
- mutex_lock(&isert_np->np_accept_mutex);
- if (!list_empty(&isert_conn->accept_node)) {
- list_del_init(&isert_conn->accept_node);
+ mutex_lock(&isert_np->mutex);
+ if (!list_empty(&isert_conn->node)) {
+ list_del_init(&isert_conn->node);
isert_put_conn(isert_conn);
queue_work(isert_release_wq, &isert_conn->release_work);
}
- mutex_unlock(&isert_np->np_accept_mutex);
+ mutex_unlock(&isert_np->mutex);
out:
return 0;
{
struct isert_conn *isert_conn = cma_id->qp->qp_context;
+ list_del_init(&isert_conn->node);
isert_conn->cm_id = NULL;
isert_put_conn(isert_conn);
}
static int
-isert_post_recv(struct isert_conn *isert_conn, u32 count)
+isert_post_recvm(struct isert_conn *isert_conn, u32 count)
{
struct ib_recv_wr *rx_wr, *rx_wr_failed;
int i, ret;
- unsigned int rx_head = isert_conn->rx_desc_head;
struct iser_rx_desc *rx_desc;
for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
- rx_desc = &isert_conn->rx_descs[rx_head];
- rx_wr->wr_id = (uintptr_t)rx_desc;
- rx_wr->sg_list = &rx_desc->rx_sg;
- rx_wr->num_sge = 1;
- rx_wr->next = rx_wr + 1;
- rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
+ rx_desc = &isert_conn->rx_descs[i];
+ rx_wr->wr_id = (uintptr_t)rx_desc;
+ rx_wr->sg_list = &rx_desc->rx_sg;
+ rx_wr->num_sge = 1;
+ rx_wr->next = rx_wr + 1;
}
-
rx_wr--;
rx_wr->next = NULL; /* mark end of work requests list */
isert_conn->post_recv_buf_count += count;
ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr,
- &rx_wr_failed);
+ &rx_wr_failed);
if (ret) {
isert_err("ib_post_recv() failed with ret: %d\n", ret);
isert_conn->post_recv_buf_count -= count;
- } else {
- isert_dbg("Posted %d RX buffers\n", count);
- isert_conn->rx_desc_head = rx_head;
}
+
+ return ret;
+}
+
+static int
+isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
+{
+ struct ib_recv_wr *rx_wr_failed, rx_wr;
+ int ret;
+
+ rx_wr.wr_id = (uintptr_t)rx_desc;
+ rx_wr.sg_list = &rx_desc->rx_sg;
+ rx_wr.num_sge = 1;
+ rx_wr.next = NULL;
+
+ isert_conn->post_recv_buf_count++;
+ ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed);
+ if (ret) {
+ isert_err("ib_post_recv() failed with ret: %d\n", ret);
+ isert_conn->post_recv_buf_count--;
+ }
+
return ret;
}
if (ret)
return ret;
- ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
+ ret = isert_post_recvm(isert_conn,
+ ISERT_QP_MAX_RECV_DTOS);
if (ret)
return ret;
}
static struct iscsi_cmd
-*isert_allocate_cmd(struct iscsi_conn *conn)
+*isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc)
{
struct isert_conn *isert_conn = conn->context;
struct isert_cmd *isert_cmd;
isert_cmd = iscsit_priv_cmd(cmd);
isert_cmd->conn = isert_conn;
isert_cmd->iscsi_cmd = cmd;
+ isert_cmd->rx_desc = rx_desc;
return cmd;
}
{
struct iscsi_conn *conn = isert_conn->conn;
struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
- struct scatterlist *sg;
int imm_data, imm_data_len, unsol_data, sg_nents, rc;
bool dump_payload = false;
+ unsigned int data_len;
rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
if (rc < 0)
imm_data = cmd->immediate_data;
imm_data_len = cmd->first_burst_len;
unsol_data = cmd->unsolicited_data;
+ data_len = cmd->se_cmd.data_length;
+ if (imm_data && imm_data_len == data_len)
+ cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
if (rc < 0) {
return 0;
if (!imm_data)
return 0;
- sg = &cmd->se_cmd.t_data_sg[0];
- sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
-
- isert_dbg("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
- sg, sg_nents, &rx_desc->data[0], imm_data_len);
-
- sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);
+ if (imm_data_len != data_len) {
+ sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
+ sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,
+ &rx_desc->data[0], imm_data_len);
+ isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n",
+ sg_nents, imm_data_len);
+ } else {
+ sg_init_table(&isert_cmd->sg, 1);
+ cmd->se_cmd.t_data_sg = &isert_cmd->sg;
+ cmd->se_cmd.t_data_nents = 1;
+ sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len);
+ isert_dbg("Transfer Immediate imm_data_len: %d\n",
+ imm_data_len);
+ }
cmd->write_data_done += imm_data_len;
if (rc < 0)
return rc;
+ /*
+ * multiple data-outs on the same command can arrive -
+ * so post the buffer before hand
+ */
+ rc = isert_post_recv(isert_conn, rx_desc);
+ if (rc) {
+ isert_err("ib_post_recv failed with %d\n", rc);
+ return rc;
+ }
return 0;
}
switch (opcode) {
case ISCSI_OP_SCSI_CMD:
- cmd = isert_allocate_cmd(conn);
+ cmd = isert_allocate_cmd(conn, rx_desc);
if (!cmd)
break;
rx_desc, (unsigned char *)hdr);
break;
case ISCSI_OP_NOOP_OUT:
- cmd = isert_allocate_cmd(conn);
+ cmd = isert_allocate_cmd(conn, rx_desc);
if (!cmd)
break;
(unsigned char *)hdr);
break;
case ISCSI_OP_SCSI_TMFUNC:
- cmd = isert_allocate_cmd(conn);
+ cmd = isert_allocate_cmd(conn, rx_desc);
if (!cmd)
break;
(unsigned char *)hdr);
break;
case ISCSI_OP_LOGOUT:
- cmd = isert_allocate_cmd(conn);
+ cmd = isert_allocate_cmd(conn, rx_desc);
if (!cmd)
break;
ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
break;
case ISCSI_OP_TEXT:
- if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) {
+ if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF)
cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
- if (!cmd)
- break;
- } else {
- cmd = isert_allocate_cmd(conn);
- if (!cmd)
- break;
- }
+ else
+ cmd = isert_allocate_cmd(conn, rx_desc);
+
+ if (!cmd)
+ break;
isert_cmd = iscsit_priv_cmd(cmd);
ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
struct ib_device *ib_dev = isert_conn->cm_id->device;
struct iscsi_hdr *hdr;
u64 rx_dma;
- int rx_buflen, outstanding;
+ int rx_buflen;
if ((char *)desc == isert_conn->login_req_buf) {
rx_dma = isert_conn->login_req_dma;
DMA_FROM_DEVICE);
isert_conn->post_recv_buf_count--;
- isert_dbg("Decremented post_recv_buf_count: %d\n",
- isert_conn->post_recv_buf_count);
-
- if ((char *)desc == isert_conn->login_req_buf)
- return;
-
- outstanding = isert_conn->post_recv_buf_count;
- if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
- int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
- ISERT_MIN_POSTED_RX);
- err = isert_post_recv(isert_conn, count);
- if (err) {
- isert_err("isert_post_recv() count: %d failed, %d\n",
- count, err);
- }
- }
}
static int
struct ib_send_wr *wr_failed;
int ret;
+ ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);
+ if (ret) {
+ isert_err("ib_post_recv failed with %d\n", ret);
+ return ret;
+ }
+
ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr,
&wr_failed);
if (ret) {
&isert_cmd->tx_desc.send_wr);
isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr;
wr->send_wr_num += 1;
+
+ rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);
+ if (rc) {
+ isert_err("ib_post_recv failed with %d\n", rc);
+ return rc;
+ }
}
rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed);
static int
isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
{
- int ret;
+ struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
+ int ret = 0;
switch (state) {
+ case ISTATE_REMOVE:
+ spin_lock_bh(&conn->cmd_lock);
+ list_del_init(&cmd->i_conn_node);
+ spin_unlock_bh(&conn->cmd_lock);
+ isert_put_cmd(isert_cmd, true);
+ break;
case ISTATE_SEND_NOPIN_WANT_RESPONSE:
ret = isert_put_nopin(cmd, conn, false);
break;
isert_err("Unable to allocate struct isert_np\n");
return -ENOMEM;
}
- sema_init(&isert_np->np_sem, 0);
- mutex_init(&isert_np->np_accept_mutex);
- INIT_LIST_HEAD(&isert_np->np_accept_list);
- init_completion(&isert_np->np_login_comp);
+ sema_init(&isert_np->sem, 0);
+ mutex_init(&isert_np->mutex);
+ INIT_LIST_HEAD(&isert_np->accepted);
+ INIT_LIST_HEAD(&isert_np->pending);
isert_np->np = np;
/*
goto out;
}
- isert_np->np_cm_id = isert_lid;
+ isert_np->cm_id = isert_lid;
np->np_context = isert_np;
return 0;
int ret;
accept_wait:
- ret = down_interruptible(&isert_np->np_sem);
+ ret = down_interruptible(&isert_np->sem);
if (ret)
return -ENODEV;
}
spin_unlock_bh(&np->np_thread_lock);
- mutex_lock(&isert_np->np_accept_mutex);
- if (list_empty(&isert_np->np_accept_list)) {
- mutex_unlock(&isert_np->np_accept_mutex);
+ mutex_lock(&isert_np->mutex);
+ if (list_empty(&isert_np->pending)) {
+ mutex_unlock(&isert_np->mutex);
goto accept_wait;
}
- isert_conn = list_first_entry(&isert_np->np_accept_list,
- struct isert_conn, accept_node);
- list_del_init(&isert_conn->accept_node);
- mutex_unlock(&isert_np->np_accept_mutex);
+ isert_conn = list_first_entry(&isert_np->pending,
+ struct isert_conn, node);
+ list_del_init(&isert_conn->node);
+ mutex_unlock(&isert_np->mutex);
conn->context = isert_conn;
isert_conn->conn = conn;
struct isert_np *isert_np = np->np_context;
struct isert_conn *isert_conn, *n;
- if (isert_np->np_cm_id)
- rdma_destroy_id(isert_np->np_cm_id);
+ if (isert_np->cm_id)
+ rdma_destroy_id(isert_np->cm_id);
/*
* FIXME: At this point we don't have a good way to insure
* that at this point we don't have hanging connections that
* completed RDMA establishment but didn't start iscsi login
* process. So work-around this by cleaning up what ever piled
- * up in np_accept_list.
+ * up in accepted and pending lists.
*/
- mutex_lock(&isert_np->np_accept_mutex);
- if (!list_empty(&isert_np->np_accept_list)) {
- isert_info("Still have isert connections, cleaning up...\n");
+ mutex_lock(&isert_np->mutex);
+ if (!list_empty(&isert_np->pending)) {
+ isert_info("Still have isert pending connections\n");
+ list_for_each_entry_safe(isert_conn, n,
+ &isert_np->pending,
+ node) {
+ isert_info("cleaning isert_conn %p state (%d)\n",
+ isert_conn, isert_conn->state);
+ isert_connect_release(isert_conn);
+ }
+ }
+
+ if (!list_empty(&isert_np->accepted)) {
+ isert_info("Still have isert accepted connections\n");
list_for_each_entry_safe(isert_conn, n,
- &isert_np->np_accept_list,
- accept_node) {
+ &isert_np->accepted,
+ node) {
isert_info("cleaning isert_conn %p state (%d)\n",
isert_conn, isert_conn->state);
isert_connect_release(isert_conn);
}
}
- mutex_unlock(&isert_np->np_accept_mutex);
+ mutex_unlock(&isert_np->mutex);
np->np_context = NULL;
kfree(isert_np);
wait_for_completion(&isert_conn->wait_comp_err);
}
+/**
+ * isert_put_unsol_pending_cmds() - Drop commands waiting for
+ * unsolicitate dataout
+ * @conn: iscsi connection
+ *
+ * We might still have commands that are waiting for unsolicited
+ * dataouts messages. We must put the extra reference on those
+ * before blocking on the target_wait_for_session_cmds
+ */
+static void
+isert_put_unsol_pending_cmds(struct iscsi_conn *conn)
+{
+ struct iscsi_cmd *cmd, *tmp;
+ static LIST_HEAD(drop_cmd_list);
+
+ spin_lock_bh(&conn->cmd_lock);
+ list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
+ if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&
+ (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&
+ (cmd->write_data_done < cmd->se_cmd.data_length))
+ list_move_tail(&cmd->i_conn_node, &drop_cmd_list);
+ }
+ spin_unlock_bh(&conn->cmd_lock);
+
+ list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {
+ list_del_init(&cmd->i_conn_node);
+ if (cmd->i_state != ISTATE_REMOVE) {
+ struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
+
+ isert_info("conn %p dropping cmd %p\n", conn, cmd);
+ isert_put_cmd(isert_cmd, true);
+ }
+ }
+}
+
static void isert_wait_conn(struct iscsi_conn *conn)
{
struct isert_conn *isert_conn = conn->context;
isert_conn_terminate(isert_conn);
mutex_unlock(&isert_conn->mutex);
- isert_wait4cmds(conn);
isert_wait4flush(isert_conn);
+ isert_put_unsol_pending_cmds(conn);
+ isert_wait4cmds(conn);
isert_wait4logout(isert_conn);
queue_work(isert_release_wq, &isert_conn->release_work);
};
struct isert_rdma_wr {
- struct list_head wr_list;
struct isert_cmd *isert_cmd;
enum iser_ib_op_code iser_ib_op;
struct ib_sge *ib_sge;
uint64_t write_va;
u64 pdu_buf_dma;
u32 pdu_buf_len;
- u32 read_va_off;
- u32 write_va_off;
- u32 rdma_wr_num;
struct isert_conn *conn;
struct iscsi_cmd *iscsi_cmd;
struct iser_tx_desc tx_desc;
+ struct iser_rx_desc *rx_desc;
struct isert_rdma_wr rdma_wr;
struct work_struct comp_work;
+ struct scatterlist sg;
};
struct isert_device;
u64 login_req_dma;
int login_req_len;
u64 login_rsp_dma;
- unsigned int rx_desc_head;
struct iser_rx_desc *rx_descs;
- struct ib_recv_wr rx_wr[ISERT_MIN_POSTED_RX];
+ struct ib_recv_wr rx_wr[ISERT_QP_MAX_RECV_DTOS];
struct iscsi_conn *conn;
- struct list_head accept_node;
+ struct list_head node;
struct completion login_comp;
struct completion login_req_comp;
struct iser_tx_desc login_tx_desc;
struct isert_np {
struct iscsi_np *np;
- struct semaphore np_sem;
- struct rdma_cm_id *np_cm_id;
- struct mutex np_accept_mutex;
- struct list_head np_accept_list;
- struct completion np_login_comp;
+ struct semaphore sem;
+ struct rdma_cm_id *cm_id;
+ struct mutex mutex;
+ struct list_head accepted;
+ struct list_head pending;
};
config JOYSTICK_ZHENHUA
tristate "5-byte Zhenhua RC transmitter"
select SERIO
+ select BITREVERSE
help
Say Y here if you have a Zhen Hua PPM-4CH transmitter which is
supplied with a ready to fly micro electric indoor helicopters
if (w->counter == 24) { /* full frame */
walkera0701_parse_frame(w);
w->counter = NO_SYNC;
- if (abs(pulse_time - SYNC_PULSE) < RESERVE) /* new frame sync */
+ if (abs64(pulse_time - SYNC_PULSE) < RESERVE) /* new frame sync */
w->counter = 0;
} else {
if ((pulse_time > (ANALOG_MIN_PULSE - RESERVE)
} else
w->counter = NO_SYNC;
}
- } else if (abs(pulse_time - SYNC_PULSE - BIN0_PULSE) <
+ } else if (abs64(pulse_time - SYNC_PULSE - BIN0_PULSE) <
RESERVE + BIN1_PULSE - BIN0_PULSE) /* frame sync .. */
w->counter = 0;
error = omap4_keypad_parse_dt(&pdev->dev, keypad_data);
if (error)
- return error;
+ goto err_free_keypad;
res = request_mem_region(res->start, resource_size(res), pdev->name);
if (!res) {
default:
reset_type = PON_PS_HOLD_TYPE_HARD_RESET;
break;
- };
+ }
error = regmap_update_bits(pwrkey->regmap,
pwrkey->baseaddr + PON_PS_HOLD_RST_CTL,
dev->id.product = user_dev->id.product;
dev->id.version = user_dev->id.version;
- for_each_set_bit(i, dev->absbit, ABS_CNT) {
+ for (i = 0; i < ABS_CNT; i++) {
input_abs_set_max(dev, i, user_dev->absmax[i]);
input_abs_set_min(dev, i, user_dev->absmin[i]);
input_abs_set_fuzz(dev, i, user_dev->absfuzz[i]);
int (*get_sm_version)(struct i2c_client *client,
u8* ic_type, u8 *version);
int (*get_checksum)(struct i2c_client *client, bool iap, u16 *csum);
- int (*get_product_id)(struct i2c_client *client, u8 *id);
+ int (*get_product_id)(struct i2c_client *client, u16 *id);
int (*get_max)(struct i2c_client *client,
unsigned int *max_x, unsigned int *max_y);
#include "elan_i2c.h"
#define DRIVER_NAME "elan_i2c"
-#define ELAN_DRIVER_VERSION "1.6.0"
+#define ELAN_DRIVER_VERSION "1.6.1"
#define ETP_MAX_PRESSURE 255
#define ETP_FWIDTH_REDUCE 90
#define ETP_FINGER_WIDTH 15
unsigned int x_res;
unsigned int y_res;
- u8 product_id;
+ u16 product_id;
u8 fw_version;
u8 sm_version;
u8 iap_version;
u16 *signature_address)
{
switch (iap_version) {
+ case 0x00:
+ case 0x06:
case 0x08:
*validpage_count = 512;
break;
+ case 0x03:
+ case 0x07:
case 0x09:
+ case 0x0A:
+ case 0x0B:
+ case 0x0C:
*validpage_count = 768;
break;
case 0x0D:
*validpage_count = 896;
break;
+ case 0x0E:
+ *validpage_count = 640;
+ break;
default:
/* unknown ic type clear value */
*validpage_count = 0;
error = elan_get_fwinfo(data->iap_version, &data->fw_validpage_count,
&data->fw_signature_address);
- if (error) {
- dev_err(&data->client->dev,
- "unknown iap version %d\n", data->iap_version);
- return error;
- }
+ if (error)
+ dev_warn(&data->client->dev,
+ "unexpected iap version %#04x (ic type: %#04x), firmware update will not work\n",
+ data->iap_version, data->ic_type);
return 0;
}
const u8 *fw_signature;
static const u8 signature[] = {0xAA, 0x55, 0xCC, 0x33, 0xFF, 0xFF};
+ if (data->fw_validpage_count == 0)
+ return -EINVAL;
+
/* Look for a firmware with the product id appended. */
fw_name = kasprintf(GFP_KERNEL, ETP_FW_NAME, data->product_id);
if (!fw_name) {
return 0;
}
-static int elan_i2c_get_product_id(struct i2c_client *client, u8 *id)
+static int elan_i2c_get_product_id(struct i2c_client *client, u16 *id)
{
int error;
u8 val[3];
return error;
}
- *id = val[0];
+ *id = le16_to_cpup((__le16 *)val);
return 0;
}
return 0;
}
-static int elan_smbus_get_product_id(struct i2c_client *client, u8 *id)
+static int elan_smbus_get_product_id(struct i2c_client *client, u16 *id)
{
int error;
u8 val[3];
return error;
}
- *id = val[1];
+ *id = be16_to_cpup((__be16 *)val);
return 0;
}
struct synaptics_data *priv = psmouse->private;
priv->mode = 0;
-
- if (priv->absolute_mode) {
+ if (priv->absolute_mode)
priv->mode |= SYN_BIT_ABSOLUTE_MODE;
- if (SYN_CAP_EXTENDED(priv->capabilities))
- priv->mode |= SYN_BIT_W_MODE;
- }
-
- if (!SYN_MODE_WMODE(priv->mode) && priv->disable_gesture)
+ if (priv->disable_gesture)
priv->mode |= SYN_BIT_DISABLE_GESTURE;
-
if (psmouse->rate >= 80)
priv->mode |= SYN_BIT_HIGH_RATE;
+ if (SYN_CAP_EXTENDED(priv->capabilities))
+ priv->mode |= SYN_BIT_W_MODE;
if (synaptics_mode_cmd(psmouse, priv->mode))
return -1;
* time before the ACK arrives.
*/
if (ps2_sendbyte(ps2dev, command & 0xff,
- command == PS2_CMD_RESET_BAT ? 1000 : 200))
- goto out;
+ command == PS2_CMD_RESET_BAT ? 1000 : 200)) {
+ serio_pause_rx(ps2dev->serio);
+ goto out_reset_flags;
+ }
- for (i = 0; i < send; i++)
- if (ps2_sendbyte(ps2dev, param[i], 200))
- goto out;
+ for (i = 0; i < send; i++) {
+ if (ps2_sendbyte(ps2dev, param[i], 200)) {
+ serio_pause_rx(ps2dev->serio);
+ goto out_reset_flags;
+ }
+ }
/*
* The reset command takes a long time to execute.
!(ps2dev->flags & PS2_FLAG_CMD), timeout);
}
+ serio_pause_rx(ps2dev->serio);
+
if (param)
for (i = 0; i < receive; i++)
param[i] = ps2dev->cmdbuf[(receive - 1) - i];
if (ps2dev->cmdcnt && (command != PS2_CMD_RESET_BAT || ps2dev->cmdcnt != 1))
- goto out;
+ goto out_reset_flags;
rc = 0;
- out:
- serio_pause_rx(ps2dev->serio);
+ out_reset_flags:
ps2dev->flags = 0;
serio_continue_rx(ps2dev->serio);
parkbd_port = parkbd_allocate_serio();
if (!parkbd_port) {
parport_release(parkbd_dev);
+ parport_unregister_device(parkbd_dev);
return -ENOMEM;
}
* TSC module need ADC to get the measure value. So
* before config TSC, we should initialize ADC module.
*/
-static void imx6ul_adc_init(struct imx6ul_tsc *tsc)
+static int imx6ul_adc_init(struct imx6ul_tsc *tsc)
{
int adc_hc = 0;
int adc_gc;
timeout = wait_for_completion_timeout
(&tsc->completion, ADC_TIMEOUT);
- if (timeout == 0)
+ if (timeout == 0) {
dev_err(tsc->dev, "Timeout for adc calibration\n");
+ return -ETIMEDOUT;
+ }
adc_gs = readl(tsc->adc_regs + REG_ADC_GS);
- if (adc_gs & ADC_CALF)
+ if (adc_gs & ADC_CALF) {
dev_err(tsc->dev, "ADC calibration failed\n");
+ return -EINVAL;
+ }
/* TSC need the ADC work in hardware trigger */
adc_cfg = readl(tsc->adc_regs + REG_ADC_CFG);
adc_cfg |= ADC_HARDWARE_TRIGGER;
writel(adc_cfg, tsc->adc_regs + REG_ADC_CFG);
+
+ return 0;
}
/*
writel(start, tsc->tsc_regs + REG_TSC_FLOW_CONTROL);
}
-static void imx6ul_tsc_init(struct imx6ul_tsc *tsc)
+static int imx6ul_tsc_init(struct imx6ul_tsc *tsc)
{
- imx6ul_adc_init(tsc);
+ int err;
+
+ err = imx6ul_adc_init(tsc);
+ if (err)
+ return err;
imx6ul_tsc_channel_config(tsc);
imx6ul_tsc_set(tsc);
+
+ return 0;
}
static void imx6ul_tsc_disable(struct imx6ul_tsc *tsc)
return err;
}
- imx6ul_tsc_init(tsc);
-
- return 0;
+ return imx6ul_tsc_init(tsc);
}
static void imx6ul_tsc_close(struct input_dev *input_dev)
int tsc_irq;
int adc_irq;
- tsc = devm_kzalloc(&pdev->dev, sizeof(struct imx6ul_tsc), GFP_KERNEL);
+ tsc = devm_kzalloc(&pdev->dev, sizeof(*tsc), GFP_KERNEL);
if (!tsc)
return -ENOMEM;
if (!input_dev)
return -ENOMEM;
- input_dev->name = "iMX6UL TouchScreen Controller";
+ input_dev->name = "iMX6UL Touchscreen Controller";
input_dev->id.bustype = BUS_HOST;
input_dev->open = imx6ul_tsc_open;
}
adc_irq = platform_get_irq(pdev, 1);
- if (adc_irq <= 0) {
+ if (adc_irq < 0) {
dev_err(&pdev->dev, "no adc irq resource?\n");
return adc_irq;
}
goto out;
}
- imx6ul_tsc_init(tsc);
+ retval = imx6ul_tsc_init(tsc);
}
out:
if (of_property_read_u32(np, "x-size", &pdata->x_size)) {
dev_err(dev, "failed to get x-size property\n");
return NULL;
- };
+ }
if (of_property_read_u32(np, "y-size", &pdata->y_size)) {
dev_err(dev, "failed to get y-size property\n");
return NULL;
- };
+ }
of_property_read_u32(np, "contact-threshold",
&pdata->contact_threshold);
endmenu
config IOMMU_IOVA
- bool
+ tristate
config OF_IOMMU
def_bool y
/* Restrict dma_mask to the width that the iommu can handle */
dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
+ /* Ensure we reserve the whole size-aligned region */
+ nrpages = __roundup_pow_of_two(nrpages);
if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
/*
static int __init iommu_init_mempool(void)
{
int ret;
- ret = iommu_iova_cache_init();
+ ret = iova_cache_get();
if (ret)
return ret;
kmem_cache_destroy(iommu_domain_cache);
domain_error:
- iommu_iova_cache_destroy();
+ iova_cache_put();
return -ENOMEM;
}
{
kmem_cache_destroy(iommu_devinfo_cache);
kmem_cache_destroy(iommu_domain_cache);
- iommu_iova_cache_destroy();
+ iova_cache_put();
}
static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
*/
#include <linux/iova.h>
+#include <linux/module.h>
#include <linux/slab.h>
-static struct kmem_cache *iommu_iova_cache;
-
-int iommu_iova_cache_init(void)
-{
- int ret = 0;
-
- iommu_iova_cache = kmem_cache_create("iommu_iova",
- sizeof(struct iova),
- 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!iommu_iova_cache) {
- pr_err("Couldn't create iova cache\n");
- ret = -ENOMEM;
- }
-
- return ret;
-}
-
-void iommu_iova_cache_destroy(void)
-{
- kmem_cache_destroy(iommu_iova_cache);
-}
-
-struct iova *alloc_iova_mem(void)
-{
- return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
-}
-
-void free_iova_mem(struct iova *iova)
-{
- kmem_cache_free(iommu_iova_cache, iova);
-}
-
void
init_iova_domain(struct iova_domain *iovad, unsigned long granule,
unsigned long start_pfn, unsigned long pfn_32bit)
iovad->start_pfn = start_pfn;
iovad->dma_32bit_pfn = pfn_32bit;
}
+EXPORT_SYMBOL_GPL(init_iova_domain);
static struct rb_node *
__get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)
}
}
-/* Computes the padding size required, to make the
- * the start address naturally aligned on its size
+/*
+ * Computes the padding size required, to make the start address
+ * naturally aligned on the power-of-two order of its size
*/
-static int
-iova_get_pad_size(int size, unsigned int limit_pfn)
+static unsigned int
+iova_get_pad_size(unsigned int size, unsigned int limit_pfn)
{
- unsigned int pad_size = 0;
- unsigned int order = ilog2(size);
-
- if (order)
- pad_size = (limit_pfn + 1) % (1 << order);
-
- return pad_size;
+ return (limit_pfn + 1 - size) & (__roundup_pow_of_two(size) - 1);
}
static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
rb_insert_color(&iova->node, root);
}
+static struct kmem_cache *iova_cache;
+static unsigned int iova_cache_users;
+static DEFINE_MUTEX(iova_cache_mutex);
+
+struct iova *alloc_iova_mem(void)
+{
+ return kmem_cache_alloc(iova_cache, GFP_ATOMIC);
+}
+EXPORT_SYMBOL(alloc_iova_mem);
+
+void free_iova_mem(struct iova *iova)
+{
+ kmem_cache_free(iova_cache, iova);
+}
+EXPORT_SYMBOL(free_iova_mem);
+
+int iova_cache_get(void)
+{
+ mutex_lock(&iova_cache_mutex);
+ if (!iova_cache_users) {
+ iova_cache = kmem_cache_create(
+ "iommu_iova", sizeof(struct iova), 0,
+ SLAB_HWCACHE_ALIGN, NULL);
+ if (!iova_cache) {
+ mutex_unlock(&iova_cache_mutex);
+ printk(KERN_ERR "Couldn't create iova cache\n");
+ return -ENOMEM;
+ }
+ }
+
+ iova_cache_users++;
+ mutex_unlock(&iova_cache_mutex);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(iova_cache_get);
+
+void iova_cache_put(void)
+{
+ mutex_lock(&iova_cache_mutex);
+ if (WARN_ON(!iova_cache_users)) {
+ mutex_unlock(&iova_cache_mutex);
+ return;
+ }
+ iova_cache_users--;
+ if (!iova_cache_users)
+ kmem_cache_destroy(iova_cache);
+ mutex_unlock(&iova_cache_mutex);
+}
+EXPORT_SYMBOL_GPL(iova_cache_put);
+
/**
* alloc_iova - allocates an iova
* @iovad: - iova domain in question
if (!new_iova)
return NULL;
- /* If size aligned is set then round the size to
- * to next power of two.
- */
- if (size_aligned)
- size = __roundup_pow_of_two(size);
-
ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn,
new_iova, size_aligned);
return new_iova;
}
+EXPORT_SYMBOL_GPL(alloc_iova);
/**
* find_iova - find's an iova for a given pfn
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
return NULL;
}
+EXPORT_SYMBOL_GPL(find_iova);
/**
* __free_iova - frees the given iova
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
free_iova_mem(iova);
}
+EXPORT_SYMBOL_GPL(__free_iova);
/**
* free_iova - finds and frees the iova for a given pfn
__free_iova(iovad, iova);
}
+EXPORT_SYMBOL_GPL(free_iova);
/**
* put_iova_domain - destroys the iova doamin
}
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
}
+EXPORT_SYMBOL_GPL(put_iova_domain);
static int
__is_range_overlap(struct rb_node *node,
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
return iova;
}
+EXPORT_SYMBOL_GPL(reserve_iova);
/**
* copy_reserved_iova - copies the reserved between domains
}
spin_unlock_irqrestore(&from->iova_rbtree_lock, flags);
}
+EXPORT_SYMBOL_GPL(copy_reserved_iova);
struct iova *
split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,
free_iova_mem(prev);
return NULL;
}
+
+MODULE_AUTHOR("Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>");
+MODULE_LICENSE("GPL");
__raw_writel(mask, combiner_base(data) + COMBINER_ENABLE_SET);
}
-static void combiner_handle_cascade_irq(unsigned int __irq,
- struct irq_desc *desc)
+static void combiner_handle_cascade_irq(struct irq_desc *desc)
{
struct combiner_chip_data *chip_data = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
- unsigned int irq = irq_desc_get_irq(desc);
unsigned int cascade_irq, combiner_irq;
unsigned long status;
cascade_irq = irq_find_mapping(combiner_irq_domain, combiner_irq);
if (unlikely(!cascade_irq))
- handle_bad_irq(irq, desc);
+ handle_bad_irq(desc);
else
generic_handle_irq(cascade_irq);
irq_set_chip_and_handler(irq, &combiner_chip, handle_level_irq);
irq_set_chip_data(irq, &combiner_data[hw >> 3]);
- set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(irq);
return 0;
}
{
irq_set_chip_and_handler(virq, &armada_370_xp_msi_irq_chip,
handle_simple_irq);
- set_irq_flags(virq, IRQF_VALID);
return 0;
}
irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
handle_level_irq);
}
- set_irq_flags(virq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(virq);
return 0;
}
static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {}
#endif
-static void armada_370_xp_mpic_handle_cascade_irq(unsigned int irq,
- struct irq_desc *desc)
+static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned long irqmap, irqn, irqsrc, cpuid;
{
struct irq_domain *domain = d->domain;
struct irq_domain_chip_generic *dgc = domain->gc;
- struct irq_chip_generic *gc = dgc->gc[0];
+ struct irq_chip_generic *bgc = dgc->gc[0];
+ struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
- /* Disable interrupt on AIC5 */
- irq_gc_lock(gc);
+ /*
+ * Disable interrupt on AIC5. We always take the lock of the
+ * first irq chip as all chips share the same registers.
+ */
+ irq_gc_lock(bgc);
irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR);
irq_reg_writel(gc, 1, AT91_AIC5_IDCR);
gc->mask_cache &= ~d->mask;
- irq_gc_unlock(gc);
+ irq_gc_unlock(bgc);
}
static void aic5_unmask(struct irq_data *d)
{
struct irq_domain *domain = d->domain;
struct irq_domain_chip_generic *dgc = domain->gc;
- struct irq_chip_generic *gc = dgc->gc[0];
+ struct irq_chip_generic *bgc = dgc->gc[0];
+ struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
- /* Enable interrupt on AIC5 */
- irq_gc_lock(gc);
+ /*
+ * Enable interrupt on AIC5. We always take the lock of the
+ * first irq chip as all chips share the same registers.
+ */
+ irq_gc_lock(bgc);
irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR);
irq_reg_writel(gc, 1, AT91_AIC5_IECR);
gc->mask_cache |= d->mask;
- irq_gc_unlock(gc);
+ irq_gc_unlock(bgc);
}
static int aic5_retrigger(struct irq_data *d)
static struct armctrl_ic intc __read_mostly;
static void __exception_irq_entry bcm2835_handle_irq(
struct pt_regs *regs);
-static void bcm2836_chained_handle_irq(unsigned int irq, struct irq_desc *desc);
+static void bcm2836_chained_handle_irq(struct irq_desc *desc);
static void armctrl_mask_irq(struct irq_data *d)
{
BUG_ON(irq <= 0);
irq_set_chip_and_handler(irq, &armctrl_chip,
handle_level_irq);
- set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(irq);
}
}
handle_IRQ(irq_linear_revmap(intc.domain, hwirq), regs);
}
-static void bcm2836_chained_handle_irq(unsigned int irq, struct irq_desc *desc)
+static void bcm2836_chained_handle_irq(struct irq_desc *desc)
{
u32 hwirq;
writel(val, reg);
}
-static void bcm7038_l1_irq_handle(unsigned int irq, struct irq_desc *desc)
+static void bcm7038_l1_irq_handle(struct irq_desc *desc)
{
struct bcm7038_l1_chip *intc = irq_desc_get_handler_data(desc);
struct bcm7038_l1_cpu *cpu;
const __be32 *map_mask_prop;
};
-static void bcm7120_l2_intc_irq_handle(unsigned int irq, struct irq_desc *desc)
+static void bcm7120_l2_intc_irq_handle(struct irq_desc *desc)
{
struct bcm7120_l1_intc_data *data = irq_desc_get_handler_data(desc);
struct bcm7120_l2_intc_data *b = data->b;
u32 saved_mask; /* for suspend/resume */
};
-static void brcmstb_l2_intc_irq_handle(unsigned int __irq,
- struct irq_desc *desc)
+static void brcmstb_l2_intc_irq_handle(struct irq_desc *desc)
{
struct brcmstb_l2_intc_data *b = irq_desc_get_handler_data(desc);
struct irq_chip_generic *gc = irq_get_domain_generic_chip(b->domain, 0);
struct irq_chip *chip = irq_desc_get_chip(desc);
- unsigned int irq = irq_desc_get_irq(desc);
+ unsigned int irq;
u32 status;
chained_irq_enter(chip, desc);
if (status == 0) {
raw_spin_lock(&desc->lock);
- handle_bad_irq(irq, desc);
+ handle_bad_irq(desc);
raw_spin_unlock(&desc->lock);
goto out;
}
irq_hw_number_t hw)
{
irq_flow_handler_t handler = handle_level_irq;
- unsigned int flags = IRQF_VALID | IRQF_PROBE;
+ unsigned int flags = 0;
if (!clps711x_irqs[hw].flags)
return 0;
if (clps711x_irqs[hw].flags & CLPS711X_FLAG_FIQ) {
handler = handle_bad_irq;
- flags |= IRQF_NOAUTOEN;
+ flags |= IRQ_NOAUTOEN;
} else if (clps711x_irqs[hw].eoi) {
handler = handle_fasteoi_irq;
}
writel_relaxed(0, clps711x_intc->base + clps711x_irqs[hw].eoi);
irq_set_chip_and_handler(virq, &clps711x_intc_chip, handler);
- set_irq_flags(virq, flags);
+ irq_modify_status(virq, IRQ_NOPROBE, flags);
return 0;
}
#define APB_INT_FINALSTATUS_H 0x34
#define APB_INT_BASE_OFFSET 0x04
-static void dw_apb_ictl_handler(unsigned int irq, struct irq_desc *desc)
+static void dw_apb_ictl_handler(struct irq_desc *desc)
{
struct irq_domain *d = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
struct v2m_data *v2m = irq_data_get_irq_chip_data(data);
phys_addr_t addr = v2m->res.start + V2M_MSI_SETSPI_NS;
- msg->address_hi = (u32) (addr >> 32);
- msg->address_lo = (u32) (addr);
+ msg->address_hi = upper_32_bits(addr);
+ msg->address_lo = lower_32_bits(addr);
msg->data = data->hwirq;
}
dev_alias->dev_id = alias;
if (pdev != dev_alias->pdev)
- dev_alias->count += its_pci_msi_vec_count(dev_alias->pdev);
+ dev_alias->count += its_pci_msi_vec_count(pdev);
return 0;
}
out:
spin_unlock(&lpi_lock);
+ if (!bitmap)
+ *base = *nr_ids = 0;
+
return bitmap;
}
* non-cacheable as well.
*/
shr = tmp & GITS_BASER_SHAREABILITY_MASK;
- if (!shr)
+ if (!shr) {
cache = GITS_BASER_nC;
+ __flush_dcache_area(base, alloc_size);
+ }
goto retry_baser;
}
return NULL;
}
+ __flush_dcache_area(itt, sz);
+
dev->its = its;
dev->itt = itt;
dev->nr_ites = nr_ites;
return gic_irq(d) < 32;
}
-static inline bool forwarded_irq(struct irq_data *d)
-{
- return d->handler_data != NULL;
-}
-
static inline void __iomem *gic_dist_base(struct irq_data *d)
{
if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */
* disabled/masked will not get "stuck", because there is
* noone to deactivate it (guest is being terminated).
*/
- if (forwarded_irq(d))
+ if (irqd_is_forwarded_to_vcpu(d))
gic_poke_irq(d, GICD_ICACTIVER);
}
* No need to deactivate an LPI, or an interrupt that
* is is getting forwarded to a vcpu.
*/
- if (gic_irq(d) >= 8192 || forwarded_irq(d))
+ if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
return;
gic_write_dir(gic_irq(d));
}
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
{
- d->handler_data = vcpu;
+ if (vcpu)
+ irqd_set_forwarded_to_vcpu(d);
+ else
+ irqd_clr_forwarded_to_vcpu(d);
return 0;
}
irq_set_percpu_devid(irq);
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_percpu_devid_irq, NULL, NULL);
- set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN);
+ irq_set_status_flags(irq, IRQ_NOAUTOEN);
}
/* SPIs */
if (hw >= 32 && hw < gic_data.irq_nr) {
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
- set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(irq);
}
/* LPIs */
if (hw >= 8192 && hw < GIC_ID_NR) {
return -EPERM;
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
- set_irq_flags(irq, IRQF_VALID);
}
return 0;
void *data = irq_data_get_irq_handler_data(d);
/*
- * If handler_data pointing to one of the secondary GICs, then
- * this is a cascading interrupt, and it cannot possibly be
- * forwarded.
+ * If handler_data is set, this is a cascading interrupt, and
+ * it cannot possibly be forwarded.
*/
- if (data >= (void *)(gic_data + 1) &&
- data < (void *)(gic_data + MAX_GIC_NR))
- return true;
-
- return false;
-}
-
-static inline bool forwarded_irq(struct irq_data *d)
-{
- /*
- * A forwarded interrupt:
- * - is on the primary GIC
- * - has its handler_data set to a value
- * - that isn't a secondary GIC
- */
- if (d->handler_data && !cascading_gic_irq(d))
- return true;
-
- return false;
+ return data != NULL;
}
/*
* disabled/masked will not get "stuck", because there is
* noone to deactivate it (guest is being terminated).
*/
- if (forwarded_irq(d))
+ if (irqd_is_forwarded_to_vcpu(d))
gic_poke_irq(d, GIC_DIST_ACTIVE_CLEAR);
}
static void gic_eoimode1_eoi_irq(struct irq_data *d)
{
/* Do not deactivate an IRQ forwarded to a vcpu. */
- if (forwarded_irq(d))
+ if (irqd_is_forwarded_to_vcpu(d))
return;
writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_DEACTIVATE);
if (cascading_gic_irq(d))
return -EINVAL;
- d->handler_data = vcpu;
+ if (vcpu)
+ irqd_set_forwarded_to_vcpu(d);
+ else
+ irqd_clr_forwarded_to_vcpu(d);
return 0;
}
} while (1);
}
-static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc)
+static void gic_handle_cascade_irq(struct irq_desc *desc)
{
struct gic_chip_data *chip_data = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
cascade_irq = irq_find_mapping(chip_data->domain, gic_irq);
if (unlikely(gic_irq < 32 || gic_irq > 1020))
- handle_bad_irq(cascade_irq, desc);
+ handle_bad_irq(desc);
else
generic_handle_irq(cascade_irq);
irq_set_percpu_devid(irq);
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_percpu_devid_irq, NULL, NULL);
- set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN);
+ irq_set_status_flags(irq, IRQ_NOAUTOEN);
} else {
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
- set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(irq);
}
return 0;
}
#ifdef CONFIG_OF
static int gic_cnt __initdata;
+static bool gic_check_eoimode(struct device_node *node, void __iomem **base)
+{
+ struct resource cpuif_res;
+
+ of_address_to_resource(node, 1, &cpuif_res);
+
+ if (!is_hyp_mode_available())
+ return false;
+ if (resource_size(&cpuif_res) < SZ_8K)
+ return false;
+ if (resource_size(&cpuif_res) == SZ_128K) {
+ u32 val_low, val_high;
+
+ /*
+ * Verify that we have the first 4kB of a GIC400
+ * aliased over the first 64kB by checking the
+ * GICC_IIDR register on both ends.
+ */
+ val_low = readl_relaxed(*base + GIC_CPU_IDENT);
+ val_high = readl_relaxed(*base + GIC_CPU_IDENT + 0xf000);
+ if ((val_low & 0xffff0fff) != 0x0202043B ||
+ val_low != val_high)
+ return false;
+
+ /*
+ * Move the base up by 60kB, so that we have a 8kB
+ * contiguous region, which allows us to use GICC_DIR
+ * at its normal offset. Please pass me that bucket.
+ */
+ *base += 0xf000;
+ cpuif_res.start += 0xf000;
+ pr_warn("GIC: Adjusting CPU interface base to %pa",
+ &cpuif_res.start);
+ }
+
+ return true;
+}
+
static int __init
gic_of_init(struct device_node *node, struct device_node *parent)
{
void __iomem *cpu_base;
void __iomem *dist_base;
- struct resource cpu_res;
u32 percpu_offset;
int irq;
cpu_base = of_iomap(node, 1);
WARN(!cpu_base, "unable to map gic cpu registers\n");
- of_address_to_resource(node, 1, &cpu_res);
-
/*
* Disable split EOI/Deactivate if either HYP is not available
* or the CPU interface is too small.
*/
- if (gic_cnt == 0 && (!is_hyp_mode_available() ||
- resource_size(&cpu_res) < SZ_8K))
+ if (gic_cnt == 0 && !gic_check_eoimode(node, &cpu_base))
static_key_slow_dec(&supports_deactivate);
if (of_property_read_u32(node, "cpu-offset", &percpu_offset))
irq_set_percpu_devid(irq);
irq_set_chip_and_handler(irq, &hip04_irq_chip,
handle_percpu_devid_irq);
- set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN);
+ irq_set_status_flags(irq, IRQ_NOAUTOEN);
} else {
irq_set_chip_and_handler(irq, &hip04_irq_chip,
handle_fasteoi_irq);
- set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(irq);
}
irq_set_chip_data(irq, d->host_data);
return 0;
__init_i8259_irqs(NULL);
}
-static void i8259_irq_dispatch(unsigned int __irq, struct irq_desc *desc)
+static void i8259_irq_dispatch(struct irq_desc *desc)
{
struct irq_domain *domain = irq_desc_get_handler_data(desc);
int hwirq = i8259_irq();
return 0;
}
-static void pdc_intc_perip_isr(unsigned int __irq, struct irq_desc *desc)
+static void pdc_intc_perip_isr(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct pdc_intc_priv *priv;
generic_handle_irq(irq_no);
}
-static void pdc_intc_syswake_isr(unsigned int irq, struct irq_desc *desc)
+static void pdc_intc_syswake_isr(struct irq_desc *desc)
{
struct pdc_intc_priv *priv;
unsigned int syswake, irq_no;
/* nothing to do here */
}
-static void keystone_irq_handler(unsigned __irq, struct irq_desc *desc)
+static void keystone_irq_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct keystone_irq_device *kirq = irq_desc_get_handler_data(desc);
irq_set_chip_data(virq, kirq);
irq_set_chip_and_handler(virq, &kirq->chip, handle_level_irq);
- set_irq_flags(virq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(virq);
return 0;
}
* Whilst using TR2 to detect external interrupts is a software convention it is
* (hopefully) unlikely to change.
*/
-static void meta_intc_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void meta_intc_irq_demux(struct irq_desc *desc)
{
struct meta_intc_priv *priv = &meta_intc_priv;
irq_hw_number_t hw;
* occurred. It is this function's job to demux this irq and
* figure out exactly which trigger needs servicing.
*/
-static void metag_internal_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void metag_internal_irq_demux(struct irq_desc *desc)
{
struct metag_internal_irq_priv *priv = irq_desc_get_handler_data(desc);
irq_hw_number_t hw;
intrmask[i] = gic_read(intrmask_reg);
pending_reg += gic_reg_step;
intrmask_reg += gic_reg_step;
+
+ if (!config_enabled(CONFIG_64BIT) || mips_cm_is64)
+ continue;
+
+ pending[i] |= (u64)gic_read(pending_reg) << 32;
+ intrmask[i] |= (u64)gic_read(intrmask_reg) << 32;
+ pending_reg += gic_reg_step;
+ intrmask_reg += gic_reg_step;
}
bitmap_and(pending, pending, intrmask, gic_shared_intrs);
spin_lock_irqsave(&gic_lock, flags);
/* Re-route this IRQ */
- gic_map_to_vpe(irq, cpumask_first(&tmp));
+ gic_map_to_vpe(irq, mips_cm_vp_id(cpumask_first(&tmp)));
/* Update the pcpu_masks */
for (i = 0; i < NR_CPUS; i++)
gic_handle_shared_int(false);
}
-static void gic_irq_dispatch(unsigned int irq, struct irq_desc *desc)
+static void gic_irq_dispatch(struct irq_desc *desc)
{
gic_handle_local_int(true);
gic_handle_shared_int(true);
GIC_SHARED_TO_HWIRQ(intr));
int i;
- gic_map_to_vpe(intr, cpu);
+ gic_map_to_vpe(intr, mips_cm_vp_id(cpu));
for (i = 0; i < NR_CPUS; i++)
clear_bit(intr, pcpu_masks[i].pcpu_mask);
set_bit(intr, pcpu_masks[cpu].pcpu_mask);
.irq_unmask = icu_unmask_irq,
};
-static void icu_mux_irq_demux(unsigned int __irq, struct irq_desc *desc)
+static void icu_mux_irq_demux(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct irq_domain *domain;
irq_hw_number_t hw)
{
irq_set_chip_and_handler(irq, &icu_irq_chip, handle_level_irq);
- set_irq_flags(irq, IRQF_VALID);
return 0;
}
for (irq = 0; irq < 64; irq++) {
icu_mask_irq(irq_get_irq_data(irq));
irq_set_chip_and_handler(irq, &icu_irq_chip, handle_level_irq);
- set_irq_flags(irq, IRQF_VALID);
}
irq_set_default_host(icu_data[0].domain);
set_handle_irq(mmp_handle_irq);
irq_set_chip_and_handler(irq, &icu_irq_chip,
handle_level_irq);
}
- set_irq_flags(irq, IRQF_VALID);
}
irq_set_default_host(icu_data[0].domain);
set_handle_irq(mmp2_handle_irq);
irq_hw_number_t hw)
{
irq_set_chip_and_handler(virq, &mxs_icoll_chip, handle_level_irq);
- set_irq_flags(virq, IRQF_VALID);
return 0;
}
#define ORION_BRIDGE_IRQ_CAUSE 0x00
#define ORION_BRIDGE_IRQ_MASK 0x04
-static void orion_bridge_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void orion_bridge_irq_handler(struct irq_desc *desc)
{
struct irq_domain *d = irq_desc_get_handler_data(desc);
static int intc_irqpin_irq_set_wake(struct irq_data *d, unsigned int on)
{
struct intc_irqpin_priv *p = irq_data_get_irq_chip_data(d);
+ int hw_irq = irqd_to_hwirq(d);
+
+ irq_set_irq_wake(p->irq[hw_irq].requested_irq, on);
if (!p->clk)
return 0;
return status;
}
+/*
+ * This lock class tells lockdep that INTC External IRQ Pin irqs are in a
+ * different category than their parents, so it won't report false recursion.
+ */
+static struct lock_class_key intc_irqpin_irq_lock_class;
+
static int intc_irqpin_irq_domain_map(struct irq_domain *h, unsigned int virq,
irq_hw_number_t hw)
{
intc_irqpin_dbg(&p->irq[hw], "map");
irq_set_chip_data(virq, h->host_data);
+ irq_set_lockdep_class(virq, &intc_irqpin_irq_lock_class);
irq_set_chip_and_handler(virq, &p->irq_chip, handle_level_irq);
- set_irq_flags(virq, IRQF_VALID); /* kill me now */
return 0;
}
static int irqc_irq_set_wake(struct irq_data *d, unsigned int on)
{
struct irqc_priv *p = irq_data_get_irq_chip_data(d);
+ int hw_irq = irqd_to_hwirq(d);
+
+ irq_set_irq_wake(p->irq[hw_irq].requested_irq, on);
if (!p->clk)
return 0;
return IRQ_NONE;
}
+/*
+ * This lock class tells lockdep that IRQC irqs are in a different
+ * category than their parents, so it won't report false recursion.
+ */
+static struct lock_class_key irqc_irq_lock_class;
+
static int irqc_irq_domain_map(struct irq_domain *h, unsigned int virq,
irq_hw_number_t hw)
{
irqc_dbg(&p->irq[hw], "map");
irq_set_chip_data(virq, h->host_data);
+ irq_set_lockdep_class(virq, &irqc_irq_lock_class);
irq_set_chip_and_handler(virq, &p->irq_chip, handle_level_irq);
return 0;
}
.irq_set_type = s3c_irqext0_type,
};
-static void s3c_irq_demux(unsigned int __irq, struct irq_desc *desc)
+static void s3c_irq_demux(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct s3c_irq_data *irq_data = irq_desc_get_chip_data(desc);
irq_set_chip_data(virq, irq_data);
- set_irq_flags(virq, IRQF_VALID);
-
if (parent_intc && irq_data->type != S3C_IRQTYPE_NONE) {
if (irq_data->parent_irq > 31) {
pr_err("irq-s3c24xx: parent irq %lu is out of range\n",
irq_data->parent_irq);
- goto err;
+ return -EINVAL;
}
parent_irq_data = &parent_intc->irqs[irq_data->parent_irq];
if (!irqno) {
pr_err("irq-s3c24xx: could not find mapping for parent irq %lu\n",
irq_data->parent_irq);
- goto err;
+ return -EINVAL;
}
irq_set_chained_handler(irqno, s3c_irq_demux);
}
return 0;
-
-err:
- set_irq_flags(virq, 0);
-
- /* the only error can result from bad mapping data*/
- return -EINVAL;
}
static const struct irq_domain_ops s3c24xx_irq_ops = {
irq_set_chip_data(virq, irq_data);
- set_irq_flags(virq, IRQF_VALID);
-
return 0;
}
irq_hw_number_t hw)
{
irq_set_chip_and_handler(virq, &sun4i_irq_chip, handle_fasteoi_irq);
- set_irq_flags(virq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(virq);
return 0;
}
return irq_reg_readl(gc, off);
}
-static void sunxi_sc_nmi_handle_irq(unsigned int irq, struct irq_desc *desc)
+static void sunxi_sc_nmi_handle_irq(struct irq_desc *desc)
{
struct irq_domain *domain = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
return IRQ_SET_MASK_OK;
}
-static void tb10x_irq_cascade(unsigned int __irq, struct irq_desc *desc)
+static void tb10x_irq_cascade(struct irq_desc *desc)
{
struct irq_domain *domain = irq_desc_get_handler_data(desc);
unsigned int irq = irq_desc_get_irq(desc);
writel(mask, f->base + IRQ_ENABLE_SET);
}
-static void fpga_irq_handle(unsigned int __irq, struct irq_desc *desc)
+static void fpga_irq_handle(struct irq_desc *desc)
{
struct fpga_irq_data *f = irq_desc_get_handler_data(desc);
- unsigned int irq = irq_desc_get_irq(desc);
u32 status = readl(f->base + IRQ_STATUS);
if (status == 0) {
- do_bad_IRQ(irq, desc);
+ do_bad_IRQ(desc);
return;
}
do {
- irq = ffs(status) - 1;
+ unsigned int irq = ffs(status) - 1;
+
status &= ~(1 << irq);
generic_handle_irq(irq_find_mapping(f->domain, irq));
} while (status);
irq_set_chip_data(irq, f);
irq_set_chip_and_handler(irq, &f->chip,
handle_level_irq);
- set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(irq);
return 0;
}
return -EPERM;
irq_set_chip_and_handler(irq, &vic_chip, handle_level_irq);
irq_set_chip_data(irq, v->base);
- set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+ irq_set_probe(irq);
return 0;
}
return handled;
}
-static void vic_handle_irq_cascaded(unsigned int irq, struct irq_desc *desc)
+static void vic_handle_irq_cascaded(struct irq_desc *desc)
{
u32 stat, hwirq;
struct irq_chip *host_chip = irq_desc_get_chip(desc);
irq_hw_number_t hw)
{
irq_set_chip_and_handler(virq, &vt8500_irq_chip, handle_level_irq);
- set_irq_flags(virq, IRQF_VALID);
return 0;
}
&spear320_shirq_intrcomm_ras,
};
-static void shirq_handler(unsigned __irq, struct irq_desc *desc)
+static void shirq_handler(struct irq_desc *desc)
{
struct spear_shirq *shirq = irq_desc_get_handler_data(desc);
u32 pend;
for (i = 0; i < shirq->nr_irqs; i++) {
irq_set_chip_and_handler(shirq->virq_base + i,
shirq->irq_chip, handle_simple_irq);
- set_irq_flags(shirq->virq_base + i, IRQF_VALID);
irq_set_chip_data(shirq->virq_base + i, shirq);
}
}
config LEDS_IPAQ_MICRO
tristate "LED Support for the Compaq iPAQ h3xxx"
+ depends on LEDS_CLASS
depends on MFD_IPAQ_MICRO
help
Choose this option if you want to use the notification LED on
tristate "Common Driver for TI/National LP5521/5523/55231/5562/8501"
depends on LEDS_LP5521 || LEDS_LP5523 || LEDS_LP5562 || LEDS_LP8501
select FW_LOADER
- select FW_LOADER_USER_HELPER_FALLBACK
+ select FW_LOADER_USER_HELPER
help
This option supports common operations for LP5521/5523/55231/5562/8501
devices.
cfg->max_brightness = b + 1;
}
-int init_mm_current_scale(struct aat1290_led *led,
+static int init_mm_current_scale(struct aat1290_led *led,
struct aat1290_led_config_data *cfg)
{
int max_mm_current_percent[] = { 20, 22, 25, 28, 32, 36, 40, 45, 50, 56,
{ .compatible = "skyworks,aat1290" },
{},
};
+MODULE_DEVICE_TABLE(of, aat1290_led_dt_match);
static struct platform_driver aat1290_led_driver = {
.probe = aat1290_led_probe,
{ .compatible = "brcm,bcm6328-leds", },
{ },
};
+MODULE_DEVICE_TABLE(of, bcm6328_leds_of_match);
static struct platform_driver bcm6328_leds_driver = {
.probe = bcm6328_leds_probe,
{ .compatible = "brcm,bcm6358-leds", },
{ },
};
+MODULE_DEVICE_TABLE(of, bcm6358_leds_of_match);
static struct platform_driver bcm6358_leds_driver = {
.probe = bcm6358_leds_probe,
{ .compatible = "kinetic,ktd2692", },
{ /* sentinel */ },
};
+MODULE_DEVICE_TABLE(of, ktd2692_match);
static struct platform_driver ktd2692_driver = {
.driver = {
{ .compatible = "maxim,max77693-led" },
{},
};
+MODULE_DEVICE_TABLE(of, max77693_led_dt_match);
static struct platform_driver max77693_led_driver = {
.probe = max77693_led_probe,
{ .compatible = "lacie,ns2-leds", },
{},
};
+MODULE_DEVICE_TABLE(of, of_ns2_leds_match);
#endif /* CONFIG_OF_GPIO */
struct ns2_led_priv {
ret = -ENOTSUPP;
dev_err(&pdev->dev,
"IO mapped PCI devices are not supported\n");
- goto out_release;
+ goto out_iounmap;
}
pci_set_drvdata(pdev, priv);
ret = chameleon_parse_cells(priv->bus, priv->mapbase, priv->base);
if (ret < 0)
- goto out_iounmap;
+ goto out_mcb_bus;
num_cells = ret;
dev_dbg(&pdev->dev, "Found %d cells\n", num_cells);
return 0;
+out_mcb_bus:
+ mcb_release_bus(priv->bus);
out_iounmap:
iounmap(priv->base);
out_release:
if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
ret = bitmap_storage_alloc(&store, chunks,
!bitmap->mddev->bitmap_info.external,
- bitmap->cluster_slot);
+ mddev_is_clustered(bitmap->mddev)
+ ? bitmap->cluster_slot : 0);
if (ret)
goto err;
static struct dm_cache_policy_type wb_policy_type = {
.name = "cleaner",
.version = {1, 0, 0},
- .hint_size = 0,
+ .hint_size = 4,
.owner = THIS_MODULE,
.create = wb_create
};
/*
* Generate a new unfragmented bio with the given size
- * This should never violate the device limitations
+ * This should never violate the device limitations (but only because
+ * max_segment_size is being constrained to PAGE_SIZE).
*
* This function may be called concurrently. If we allocate from the mempool
* concurrently, there is a possibility of deadlock. For example, if we have
return fn(ti, cc->dev, cc->start, ti->len, data);
}
+static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+ /*
+ * Unfortunate constraint that is required to avoid the potential
+ * for exceeding underlying device's max_segments limits -- due to
+ * crypt_alloc_buffer() possibly allocating pages for the encryption
+ * bio that are not as physically contiguous as the original bio.
+ */
+ limits->max_segment_size = PAGE_SIZE;
+}
+
static struct target_type crypt_target = {
.name = "crypt",
- .version = {1, 14, 0},
+ .version = {1, 14, 1},
.module = THIS_MODULE,
.ctr = crypt_ctr,
.dtr = crypt_dtr,
.resume = crypt_resume,
.message = crypt_message,
.iterate_devices = crypt_iterate_devices,
+ .io_hints = crypt_io_hints,
};
static int __init dm_crypt_init(void)
return -EINVAL;
}
- tmp_store = kmalloc(sizeof(*tmp_store), GFP_KERNEL);
+ tmp_store = kzalloc(sizeof(*tmp_store), GFP_KERNEL);
if (!tmp_store) {
ti->error = "Exception store allocation failed";
return -ENOMEM;
else if (persistent == 'N')
type = get_type("N");
else {
- ti->error = "Persistent flag is not P or N";
+ ti->error = "Exception store type is not P or N";
r = -EINVAL;
goto bad_type;
}
if (r)
goto bad;
- r = type->ctr(tmp_store, 0, NULL);
+ r = type->ctr(tmp_store, (strlen(argv[0]) > 1 ? &argv[0][1] : NULL));
if (r) {
ti->error = "Exception store type constructor failed";
goto bad;
const char *name;
struct module *module;
- int (*ctr) (struct dm_exception_store *store,
- unsigned argc, char **argv);
+ int (*ctr) (struct dm_exception_store *store, char *options);
/*
* Destroys this object when you've finished with it.
unsigned chunk_shift;
void *context;
+
+ bool userspace_supports_overflow;
};
/*
*/
if (min_region_size > (1 << 13)) {
/* If not a power of 2, make it the next power of 2 */
- if (min_region_size & (min_region_size - 1))
- region_size = 1 << fls(region_size);
+ region_size = roundup_pow_of_two(min_region_size);
DMINFO("Choosing default region size of %lu sectors",
region_size);
} else {
#include "dm-exception-store.h"
+#include <linux/ctype.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/vmalloc.h>
DMWARN("write header failed");
}
-static int persistent_ctr(struct dm_exception_store *store,
- unsigned argc, char **argv)
+static int persistent_ctr(struct dm_exception_store *store, char *options)
{
struct pstore *ps;
return -ENOMEM;
}
+ if (options) {
+ char overflow = toupper(options[0]);
+ if (overflow == 'O')
+ store->userspace_supports_overflow = true;
+ else {
+ DMERR("Unsupported persistent store option: %s", options);
+ return -EINVAL;
+ }
+ }
+
store->context = ps;
return 0;
case STATUSTYPE_INFO:
break;
case STATUSTYPE_TABLE:
- DMEMIT(" P %llu", (unsigned long long)store->chunk_size);
+ DMEMIT(" %s %llu", store->userspace_supports_overflow ? "PO" : "P",
+ (unsigned long long)store->chunk_size);
}
return sz;
*metadata_sectors = 0;
}
-static int transient_ctr(struct dm_exception_store *store,
- unsigned argc, char **argv)
+static int transient_ctr(struct dm_exception_store *store, char *options)
{
struct transient_c *tc;
}
/*
- * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
+ * Construct a snapshot mapping: <origin_dev> <COW-dev> <p|po|n> <chunk-size>
*/
static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
{
u.store_swap = snap_dest->store;
snap_dest->store = snap_src->store;
+ snap_dest->store->userspace_supports_overflow = u.store_swap->userspace_supports_overflow;
snap_src->store = u.store_swap;
snap_dest->store->snap = snap_dest;
pe = __find_pending_exception(s, pe, chunk);
if (!pe) {
- s->snapshot_overflowed = 1;
- DMERR("Snapshot overflowed: Unable to allocate exception.");
+ if (s->store->userspace_supports_overflow) {
+ s->snapshot_overflowed = 1;
+ DMERR("Snapshot overflowed: Unable to allocate exception.");
+ } else
+ __invalidate_snapshot(s, -ENOMEM);
r = -EIO;
goto out_unlock;
}
static struct target_type snapshot_target = {
.name = "snapshot",
- .version = {1, 14, 0},
+ .version = {1, 15, 0},
.module = THIS_MODULE,
.ctr = snapshot_ctr,
.dtr = snapshot_dtr,
static struct target_type merge_target = {
.name = dm_snapshot_merge_target_name,
- .version = {1, 3, 0},
+ .version = {1, 4, 0},
.module = THIS_MODULE,
.ctr = snapshot_ctr,
.dtr = snapshot_dtr,
{
struct thin_c *tc = ti->private;
struct pool *pool = tc->pool;
+ struct queue_limits *pool_limits = dm_get_queue_limits(pool->pool_md);
+
+ if (!pool_limits->discard_granularity)
+ return; /* pool's discard support is disabled */
limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */
struct dm_rq_target_io *tio = info->tio;
struct bio *bio = info->orig;
unsigned int nr_bytes = info->orig->bi_iter.bi_size;
+ int error = clone->bi_error;
bio_put(clone);
* the remainder.
*/
return;
- else if (bio->bi_error) {
+ else if (error) {
/*
* Don't notice the error to the upper layer yet.
* The error handling decision is made by the target driver,
* when the request is completed.
*/
- tio->error = bio->bi_error;
+ tio->error = error;
return;
}
might_sleep();
- map = dm_get_live_table(md, &srcu_idx);
-
spin_lock(&_minor_lock);
idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
set_bit(DMF_FREEING, &md->flags);
* do not race with internal suspend.
*/
mutex_lock(&md->suspend_lock);
+ map = dm_get_live_table(md, &srcu_idx);
if (!dm_suspended_md(md)) {
dm_table_presuspend_targets(map);
dm_table_postsuspend_targets(map);
}
- mutex_unlock(&md->suspend_lock);
-
/* dm_put_live_table must be before msleep, otherwise deadlock is possible */
dm_put_live_table(md, srcu_idx);
+ mutex_unlock(&md->suspend_lock);
/*
* Rare, but there may be I/O requests still going to complete,
* which will now never happen */
wake_up_process(mddev->sync_thread->tsk);
+ if (mddev->external && test_bit(MD_CHANGE_PENDING, &mddev->flags))
+ return -EBUSY;
mddev_unlock(mddev);
wait_event(resync_wait, !test_bit(MD_RECOVERY_RUNNING,
&mddev->recovery));
+ wait_event(mddev->sb_wait,
+ !test_bit(MD_CHANGE_PENDING, &mddev->flags));
mddev_lock_nointr(mddev);
mutex_lock(&mddev->open_mutex);
md_reap_sync_thread(mddev);
clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ clear_bit(MD_CHANGE_PENDING, &mddev->flags);
goto unlock;
}
return 0;
out_free_conf:
- if (conf->pool)
- mempool_destroy(conf->pool);
+ mempool_destroy(conf->pool);
kfree(conf->multipaths);
kfree(conf);
mddev->private = NULL;
struct md_rdev *rdev;
bool discard_supported = false;
- rdev_for_each(rdev, mddev) {
- disk_stack_limits(mddev->gendisk, rdev->bdev,
- rdev->data_offset << 9);
- if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
- discard_supported = true;
- }
blk_queue_max_hw_sectors(mddev->queue, mddev->chunk_sectors);
blk_queue_max_write_same_sectors(mddev->queue, mddev->chunk_sectors);
blk_queue_max_discard_sectors(mddev->queue, mddev->chunk_sectors);
blk_queue_io_opt(mddev->queue,
(mddev->chunk_sectors << 9) * mddev->raid_disks);
+ rdev_for_each(rdev, mddev) {
+ disk_stack_limits(mddev->gendisk, rdev->bdev,
+ rdev->data_offset << 9);
+ if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
+ discard_supported = true;
+ }
if (!discard_supported)
queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
else
}
if (bio && bio_data_dir(bio) == WRITE) {
- if (bio->bi_iter.bi_sector >=
- conf->mddev->curr_resync_completed) {
+ if (bio->bi_iter.bi_sector >= conf->next_resync) {
if (conf->start_next_window == MaxSector)
conf->start_next_window =
conf->next_resync +
conf->r1buf_pool = NULL;
spin_lock_irq(&conf->resync_lock);
- conf->next_resync = 0;
+ conf->next_resync = MaxSector - 2 * NEXT_NORMALIO_DISTANCE;
conf->start_next_window = MaxSector;
conf->current_window_requests +=
conf->next_window_requests;
}
spin_unlock_irqrestore(&conf->device_lock, flags);
while (!list_empty(&tmp)) {
- r1_bio = list_first_entry(&conf->bio_end_io_list,
- struct r1bio, retry_list);
+ r1_bio = list_first_entry(&tmp, struct r1bio,
+ retry_list);
list_del(&r1_bio->retry_list);
raid_end_bio_io(r1_bio);
}
abort:
if (conf) {
- if (conf->r1bio_pool)
- mempool_destroy(conf->r1bio_pool);
+ mempool_destroy(conf->r1bio_pool);
kfree(conf->mirrors);
safe_put_page(conf->tmppage);
kfree(conf->poolinfo);
{
struct r1conf *conf = priv;
- if (conf->r1bio_pool)
- mempool_destroy(conf->r1bio_pool);
+ mempool_destroy(conf->r1bio_pool);
kfree(conf->mirrors);
safe_put_page(conf->tmppage);
kfree(conf->poolinfo);
}
spin_unlock_irqrestore(&conf->device_lock, flags);
while (!list_empty(&tmp)) {
- r10_bio = list_first_entry(&conf->bio_end_io_list,
- struct r10bio, retry_list);
+ r10_bio = list_first_entry(&tmp, struct r10bio,
+ retry_list);
list_del(&r10_bio->retry_list);
raid_end_bio_io(r10_bio);
}
printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
mdname(mddev));
if (conf) {
- if (conf->r10bio_pool)
- mempool_destroy(conf->r10bio_pool);
+ mempool_destroy(conf->r10bio_pool);
kfree(conf->mirrors);
safe_put_page(conf->tmppage);
kfree(conf);
out_free_conf:
md_unregister_thread(&mddev->thread);
- if (conf->r10bio_pool)
- mempool_destroy(conf->r10bio_pool);
+ mempool_destroy(conf->r10bio_pool);
safe_put_page(conf->tmppage);
kfree(conf->mirrors);
kfree(conf);
{
struct r10conf *conf = priv;
- if (conf->r10bio_pool)
- mempool_destroy(conf->r10bio_pool);
+ mempool_destroy(conf->r10bio_pool);
safe_put_page(conf->tmppage);
kfree(conf->mirrors);
kfree(conf->mirrors_old);
drop_one_stripe(conf))
;
- if (conf->slab_cache)
- kmem_cache_destroy(conf->slab_cache);
+ kmem_cache_destroy(conf->slab_cache);
conf->slab_cache = NULL;
}
spin_unlock_irq(&sh->stripe_lock);
if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
wake_up(&conf->wait_for_overlap);
+ if (bi)
+ s->to_read--;
while (bi && bi->bi_iter.bi_sector <
sh->dev[i].sector + STRIPE_SECTORS) {
struct bio *nextbi =
*/
clear_bit(R5_LOCKED, &sh->dev[i].flags);
}
+ s->to_write = 0;
+ s->written = 0;
if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
if (atomic_dec_and_test(&conf->pending_full_writes))
*/
return 0;
- for (i = 0; i < s->failed; i++) {
+ for (i = 0; i < s->failed && i < 2; i++) {
if (fdev[i]->towrite &&
!test_bit(R5_UPTODATE, &fdev[i]->flags) &&
!test_bit(R5_OVERWRITE, &fdev[i]->flags))
sh->sector < sh->raid_conf->mddev->recovery_cp)
/* reconstruct-write isn't being forced */
return 0;
- for (i = 0; i < s->failed; i++) {
+ for (i = 0; i < s->failed && i < 2; i++) {
if (s->failed_num[i] != sh->pd_idx &&
s->failed_num[i] != sh->qd_idx &&
!test_bit(R5_UPTODATE, &fdev[i]->flags) &&
spin_unlock_irqrestore(&asic->lock, flags);
}
-static void asic3_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void asic3_irq_demux(struct irq_desc *desc)
{
struct asic3 *asic = irq_desc_get_handler_data(desc);
struct irq_data *data = irq_desc_get_irq_data(desc);
} while (gpio_get_value(pdata->gpio));
}
-static void pcap_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void pcap_irq_handler(struct irq_desc *desc)
{
struct pcap_chip *pcap = irq_desc_get_handler_data(desc);
.irq_unmask = egpio_unmask,
};
-static void egpio_handler(unsigned int irq, struct irq_desc *desc)
+static void egpio_handler(struct irq_desc *desc)
{
struct egpio_info *ei = irq_desc_get_handler_data(desc);
int irqpin;
spinlock_t lock;
};
-static void jz4740_adc_irq_demux(unsigned int irq, struct irq_desc *desc)
+static void jz4740_adc_irq_demux(struct irq_desc *desc)
{
struct irq_chip_generic *gc = irq_desc_get_handler_data(desc);
uint8_t status;
return ret;
}
-static void pm8xxx_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void pm8xxx_irq_handler(struct irq_desc *desc)
{
struct pm_irq_chip *chip = irq_desc_get_handler_data(desc);
struct irq_chip *irq_chip = irq_desc_get_chip(desc);
/*--------------------------------------------------------------------------*/
/* Handle the T7L66XB interrupt mux */
-static void t7l66xb_irq(unsigned int irq, struct irq_desc *desc)
+static void t7l66xb_irq(struct irq_desc *desc)
{
struct t7l66xb *t7l66xb = irq_desc_get_handler_data(desc);
unsigned int isr;
/*--------------------------------------------------------------------------*/
-static void
-tc6393xb_irq(unsigned int irq, struct irq_desc *desc)
+static void tc6393xb_irq(struct irq_desc *desc)
{
struct tc6393xb *tc6393xb = irq_desc_get_handler_data(desc);
unsigned int isr;
* SIBCLK to talk to the chip. We leave the clock running until
* we have finished processing all interrupts from the chip.
*/
-static void ucb1x00_irq(unsigned int __irq, struct irq_desc *desc)
+static void ucb1x00_irq(struct irq_desc *desc)
{
struct ucb1x00 *ucb = irq_desc_get_handler_data(desc);
unsigned int isr, i;
-ccflags-y := -Werror
+ccflags-y := -Werror -Wno-unused-const-variable
cxl-y += main.o file.o irq.o fault.o native.o
cxl-y += context.o sysfs.o debugfs.o pci.o trace.o
int slice;
int rc;
- pci_dev_get(dev);
-
if (cxl_verbose)
dump_cxl_config_space(dev);
/* conditionally create the add the binary file for error info buffer */
if (afu->eb_len) {
+ sysfs_attr_init(&afu->attr_eb.attr);
+
afu->attr_eb.attr.name = "afu_err_buff";
afu->attr_eb.attr.mode = S_IRUGO;
afu->attr_eb.size = afu->eb_len;
phb = pci_bus_to_host(dev->bus);
afu = (struct cxl_afu *)phb->private_data;
+
+ if (!cxl_adapter_link_ok(afu->adapter)) {
+ dev_warn(&dev->dev, "%s: Device link is down, refusing to enable AFU\n", __func__);
+ return false;
+ }
+
set_dma_ops(&dev->dev, &dma_direct_ops);
set_dma_offset(&dev->dev, PAGE_OFFSET);
if (!dir)
return -ENOMEM;
+ dev->dbgfs_dir = dir;
+
f = debugfs_create_file("meclients", S_IRUSR, dir,
dev, &mei_dbgfs_fops_meclients);
if (!f) {
dev_err(dev->dev, "allow_fixed_address: registration failed\n");
goto err;
}
- dev->dbgfs_dir = dir;
return 0;
err:
mei_dbgfs_deregister(dev);
* after the host receives the enum_resp
* message clients may be added or removed
*/
- if (dev->hbm_state <= MEI_HBM_ENUM_CLIENTS &&
+ if (dev->hbm_state <= MEI_HBM_ENUM_CLIENTS ||
dev->hbm_state >= MEI_HBM_STOPPED) {
dev_err(dev->dev, "hbm: add client: state mismatch, [%d, %d]\n",
dev->dev_state, dev->hbm_state);
int err = cmd->error;
/* Flag re-tuning needed on CRC errors */
- if (err == -EILSEQ || (mrq->sbc && mrq->sbc->error == -EILSEQ) ||
+ if ((cmd->opcode != MMC_SEND_TUNING_BLOCK &&
+ cmd->opcode != MMC_SEND_TUNING_BLOCK_HS200) &&
+ (err == -EILSEQ || (mrq->sbc && mrq->sbc->error == -EILSEQ) ||
(mrq->data && mrq->data->error == -EILSEQ) ||
- (mrq->stop && mrq->stop->error == -EILSEQ))
+ (mrq->stop && mrq->stop->error == -EILSEQ)))
mmc_retune_needed(host);
if (err && cmd->retries && mmc_host_is_spi(host)) {
0, &cd_gpio_invert);
if (!ret)
dev_info(host->parent, "Got CD GPIO\n");
- else if (ret != -ENOENT)
+ else if (ret != -ENOENT && ret != -ENOSYS)
return ret;
/*
ret = mmc_gpiod_request_ro(host, "wp", 0, false, 0, &ro_gpio_invert);
if (!ret)
dev_info(host->parent, "Got WP GPIO\n");
- else if (ret != -ENOENT)
+ else if (ret != -ENOENT && ret != -ENOSYS)
return ret;
if (of_property_read_bool(np, "disable-wp"))
struct clk *fclk;
struct clk *dbclk;
struct regulator *pbias;
+ bool pbias_enabled;
void __iomem *base;
int vqmmc_enabled;
resource_size_t mapbase;
return ret;
}
- if (!regulator_is_enabled(host->pbias)) {
+ if (host->pbias_enabled == 0) {
ret = regulator_enable(host->pbias);
if (ret) {
dev_err(host->dev, "pbias reg enable fail\n");
return ret;
}
+ host->pbias_enabled = 1;
}
} else {
- if (regulator_is_enabled(host->pbias)) {
+ if (host->pbias_enabled == 1) {
ret = regulator_disable(host->pbias);
if (ret) {
dev_err(host->dev, "pbias reg disable fail\n");
return ret;
}
+ host->pbias_enabled = 0;
}
}
mmc->supply.vmmc = devm_regulator_get_optional(host->dev, "vmmc");
if (IS_ERR(mmc->supply.vmmc)) {
ret = PTR_ERR(mmc->supply.vmmc);
- if (ret != -ENODEV)
+ if ((ret != -ENODEV) && host->dev->of_node)
return ret;
dev_dbg(host->dev, "unable to get vmmc regulator %ld\n",
PTR_ERR(mmc->supply.vmmc));
mmc->supply.vqmmc = devm_regulator_get_optional(host->dev, "vmmc_aux");
if (IS_ERR(mmc->supply.vqmmc)) {
ret = PTR_ERR(mmc->supply.vqmmc);
- if (ret != -ENODEV)
+ if ((ret != -ENODEV) && host->dev->of_node)
return ret;
dev_dbg(host->dev, "unable to get vmmc_aux regulator %ld\n",
PTR_ERR(mmc->supply.vqmmc));
host->pbias = devm_regulator_get_optional(host->dev, "pbias");
if (IS_ERR(host->pbias)) {
ret = PTR_ERR(host->pbias);
- if (ret != -ENODEV)
+ if ((ret != -ENODEV) && host->dev->of_node)
return ret;
dev_dbg(host->dev, "unable to get pbias regulator %ld\n",
PTR_ERR(host->pbias));
host->base = base + pdata->reg_offset;
host->power_mode = MMC_POWER_OFF;
host->next_data.cookie = 1;
+ host->pbias_enabled = 0;
host->vqmmc_enabled = 0;
ret = omap_hsmmc_gpio_init(mmc, host, pdata);
#include <linux/clk.h>
#include <linux/err.h>
#include <linux/mmc/host.h>
+#include <linux/mmc/slot-gpio.h>
#include <linux/io.h>
#include <linux/regulator/consumer.h>
#include <linux/gpio.h>
{
struct pxamci_host *host = mmc_priv(mmc);
- if (host->pdata && gpio_is_valid(host->pdata->gpio_card_ro)) {
- if (host->pdata->gpio_card_ro_invert)
- return !gpio_get_value(host->pdata->gpio_card_ro);
- else
- return gpio_get_value(host->pdata->gpio_card_ro);
- }
+ if (host->pdata && gpio_is_valid(host->pdata->gpio_card_ro))
+ return mmc_gpio_get_ro(mmc);
if (host->pdata && host->pdata->get_ro)
return !!host->pdata->get_ro(mmc_dev(mmc));
/*
static const struct mmc_host_ops pxamci_ops = {
.request = pxamci_request,
+ .get_cd = mmc_gpio_get_cd,
.get_ro = pxamci_get_ro,
.set_ios = pxamci_set_ios,
.enable_sdio_irq = pxamci_enable_sdio_irq,
gpio_power = host->pdata->gpio_power;
}
if (gpio_is_valid(gpio_power)) {
- ret = gpio_request(gpio_power, "mmc card power");
+ ret = devm_gpio_request(&pdev->dev, gpio_power,
+ "mmc card power");
if (ret) {
- dev_err(&pdev->dev, "Failed requesting gpio_power %d\n", gpio_power);
+ dev_err(&pdev->dev, "Failed requesting gpio_power %d\n",
+ gpio_power);
goto out;
}
gpio_direction_output(gpio_power,
host->pdata->gpio_power_invert);
}
- if (gpio_is_valid(gpio_ro)) {
- ret = gpio_request(gpio_ro, "mmc card read only");
- if (ret) {
- dev_err(&pdev->dev, "Failed requesting gpio_ro %d\n", gpio_ro);
- goto err_gpio_ro;
- }
- gpio_direction_input(gpio_ro);
+ if (gpio_is_valid(gpio_ro))
+ ret = mmc_gpio_request_ro(mmc, gpio_ro);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed requesting gpio_ro %d\n", gpio_ro);
+ goto out;
+ } else {
+ mmc->caps |= host->pdata->gpio_card_ro_invert ?
+ MMC_CAP2_RO_ACTIVE_HIGH : 0;
}
- if (gpio_is_valid(gpio_cd)) {
- ret = gpio_request(gpio_cd, "mmc card detect");
- if (ret) {
- dev_err(&pdev->dev, "Failed requesting gpio_cd %d\n", gpio_cd);
- goto err_gpio_cd;
- }
- gpio_direction_input(gpio_cd);
- ret = request_irq(gpio_to_irq(gpio_cd), pxamci_detect_irq,
- IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING,
- "mmc card detect", mmc);
- if (ret) {
- dev_err(&pdev->dev, "failed to request card detect IRQ\n");
- goto err_request_irq;
- }
+ if (gpio_is_valid(gpio_cd))
+ ret = mmc_gpio_request_cd(mmc, gpio_cd, 0);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed requesting gpio_cd %d\n", gpio_cd);
+ goto out;
}
if (host->pdata && host->pdata->init)
return 0;
-err_request_irq:
- gpio_free(gpio_cd);
-err_gpio_cd:
- gpio_free(gpio_ro);
-err_gpio_ro:
- gpio_free(gpio_power);
- out:
+out:
if (host) {
if (host->dma_chan_rx)
dma_release_channel(host->dma_chan_rx);
gpio_ro = host->pdata->gpio_card_ro;
gpio_power = host->pdata->gpio_power;
}
- if (gpio_is_valid(gpio_cd)) {
- free_irq(gpio_to_irq(gpio_cd), mmc);
- gpio_free(gpio_cd);
- }
- if (gpio_is_valid(gpio_ro))
- gpio_free(gpio_ro);
- if (gpio_is_valid(gpio_power))
- gpio_free(gpio_power);
if (host->vcc)
regulator_put(host->vcc);
static const struct sdhci_pltfm_data soc_data_sama5d2 = {
.ops = &sdhci_at91_sama5d2_ops,
+ .quirks2 = SDHCI_QUIRK2_NEED_DELAY_AFTER_INT_CLK_RST,
};
static const struct of_device_id sdhci_at91_dt_match[] = {
struct sdhci_pxa *pxa = pltfm_host->priv;
struct resource *res;
+ host->quirks &= ~SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN;
host->quirks |= SDHCI_QUIRK_MISSING_CAPS;
res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"conf-sdio3");
uhs == MMC_TIMING_UHS_DDR50) {
reg_val &= ~SDIO3_CONF_CLK_INV;
reg_val |= SDIO3_CONF_SD_FB_CLK;
+ } else if (uhs == MMC_TIMING_MMC_HS) {
+ reg_val &= ~SDIO3_CONF_CLK_INV;
+ reg_val &= ~SDIO3_CONF_SD_FB_CLK;
} else {
reg_val |= SDIO3_CONF_CLK_INV;
reg_val &= ~SDIO3_CONF_SD_FB_CLK;
if (of_device_is_compatible(np, "marvell,armada-380-sdhci")) {
ret = armada_38x_quirks(pdev, host);
if (ret < 0)
- goto err_clk_get;
+ goto err_mbus_win;
ret = mv_conf_mbus_windows(pdev, mv_mbus_dram_info());
if (ret < 0)
goto err_mbus_win;
host->mmc->actual_clock = 0;
sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
+ if (host->quirks2 & SDHCI_QUIRK2_NEED_DELAY_AFTER_INT_CLK_RST)
+ mdelay(1);
if (clock == 0)
return;
#define SDHCI_QUIRK2_ACMD23_BROKEN (1<<14)
/* Broken Clock divider zero in controller */
#define SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN (1<<15)
+/*
+ * When internal clock is disabled, a delay is needed before modifying the
+ * SD clock frequency or enabling back the internal clock.
+ */
+#define SDHCI_QUIRK2_NEED_DELAY_AFTER_INT_CLK_RST (1<<16)
int irq; /* Device IRQ */
void __iomem *ioaddr; /* Mapped address */
#define SDXC_IDMAC_DES0_CES BIT(30) /* card error summary */
#define SDXC_IDMAC_DES0_OWN BIT(31) /* 1-idma owns it, 0-host owns it */
+#define SDXC_CLK_400K 0
+#define SDXC_CLK_25M 1
+#define SDXC_CLK_50M 2
+#define SDXC_CLK_50M_DDR 3
+
+struct sunxi_mmc_clk_delay {
+ u32 output;
+ u32 sample;
+};
+
struct sunxi_idma_des {
u32 config;
u32 buf_size;
struct clk *clk_mmc;
struct clk *clk_sample;
struct clk *clk_output;
+ const struct sunxi_mmc_clk_delay *clk_delays;
/* irq */
spinlock_t lock;
/* determine delays */
if (rate <= 400000) {
- oclk_dly = 180;
- sclk_dly = 42;
+ oclk_dly = host->clk_delays[SDXC_CLK_400K].output;
+ sclk_dly = host->clk_delays[SDXC_CLK_400K].sample;
} else if (rate <= 25000000) {
- oclk_dly = 180;
- sclk_dly = 75;
+ oclk_dly = host->clk_delays[SDXC_CLK_25M].output;
+ sclk_dly = host->clk_delays[SDXC_CLK_25M].sample;
} else if (rate <= 50000000) {
if (ios->timing == MMC_TIMING_UHS_DDR50) {
- oclk_dly = 60;
- sclk_dly = 120;
+ oclk_dly = host->clk_delays[SDXC_CLK_50M_DDR].output;
+ sclk_dly = host->clk_delays[SDXC_CLK_50M_DDR].sample;
} else {
- oclk_dly = 90;
- sclk_dly = 150;
+ oclk_dly = host->clk_delays[SDXC_CLK_50M].output;
+ sclk_dly = host->clk_delays[SDXC_CLK_50M].sample;
}
- } else if (rate <= 100000000) {
- oclk_dly = 6;
- sclk_dly = 24;
- } else if (rate <= 200000000) {
- oclk_dly = 3;
- sclk_dly = 12;
} else {
return -EINVAL;
}
static const struct of_device_id sunxi_mmc_of_match[] = {
{ .compatible = "allwinner,sun4i-a10-mmc", },
{ .compatible = "allwinner,sun5i-a13-mmc", },
+ { .compatible = "allwinner,sun9i-a80-mmc", },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, sunxi_mmc_of_match);
.hw_reset = sunxi_mmc_hw_reset,
};
+static const struct sunxi_mmc_clk_delay sunxi_mmc_clk_delays[] = {
+ [SDXC_CLK_400K] = { .output = 180, .sample = 180 },
+ [SDXC_CLK_25M] = { .output = 180, .sample = 75 },
+ [SDXC_CLK_50M] = { .output = 90, .sample = 120 },
+ [SDXC_CLK_50M_DDR] = { .output = 60, .sample = 120 },
+};
+
+static const struct sunxi_mmc_clk_delay sun9i_mmc_clk_delays[] = {
+ [SDXC_CLK_400K] = { .output = 180, .sample = 180 },
+ [SDXC_CLK_25M] = { .output = 180, .sample = 75 },
+ [SDXC_CLK_50M] = { .output = 150, .sample = 120 },
+ [SDXC_CLK_50M_DDR] = { .output = 90, .sample = 120 },
+};
+
static int sunxi_mmc_resource_request(struct sunxi_mmc_host *host,
struct platform_device *pdev)
{
else
host->idma_des_size_bits = 16;
+ if (of_device_is_compatible(np, "allwinner,sun9i-a80-mmc"))
+ host->clk_delays = sun9i_mmc_clk_delays;
+ else
+ host->clk_delays = sunxi_mmc_clk_delays;
+
ret = mmc_regulator_get_supply(host->mmc);
if (ret) {
if (ret != -EPROBE_DEFER)
oob_chunk_size);
/* the last chunk */
- memcpy16_toio(&s[oob_chunk_size * sparebuf_size],
+ memcpy16_toio(&s[i * sparebuf_size],
&d[i * oob_chunk_size],
host->used_oobsize - i * oob_chunk_size);
}
#define NFC_ECC_MODE GENMASK(15, 12)
#define NFC_RANDOM_SEED GENMASK(30, 16)
+/* NFC_USER_DATA helper macros */
+#define NFC_BUF_TO_USER_DATA(buf) ((buf)[0] | ((buf)[1] << 8) | \
+ ((buf)[2] << 16) | ((buf)[3] << 24))
+
#define NFC_DEFAULT_TIMEOUT_MS 1000
#define NFC_SRAM_SIZE 1024
offset = layout->eccpos[i * ecc->bytes] - 4 + mtd->writesize;
/* Fill OOB data in */
- if (oob_required) {
- tmp = 0xffffffff;
- memcpy_toio(nfc->regs + NFC_REG_USER_DATA_BASE, &tmp,
- 4);
- } else {
- memcpy_toio(nfc->regs + NFC_REG_USER_DATA_BASE,
- chip->oob_poi + offset - mtd->writesize,
- 4);
- }
+ writel(NFC_BUF_TO_USER_DATA(chip->oob_poi +
+ layout->oobfree[i].offset),
+ nfc->regs + NFC_REG_USER_DATA_BASE);
chip->cmdfunc(mtd, NAND_CMD_RNDIN, offset, -1);
offset += ecc->size;
/* Fill OOB data in */
- if (oob_required) {
- tmp = 0xffffffff;
- memcpy_toio(nfc->regs + NFC_REG_USER_DATA_BASE, &tmp,
- 4);
- } else {
- memcpy_toio(nfc->regs + NFC_REG_USER_DATA_BASE, oob,
- 4);
- }
+ writel(NFC_BUF_TO_USER_DATA(oob),
+ nfc->regs + NFC_REG_USER_DATA_BASE);
tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ACCESS_DIR |
(1 << 30);
node);
nand_release(&chip->mtd);
sunxi_nand_ecc_cleanup(&chip->nand.ecc);
+ list_del(&chip->node);
}
}
goto bad;
}
+ if (data_size > ubi->leb_size) {
+ ubi_err(ubi, "bad data_size");
+ goto bad;
+ }
+
if (vol_type == UBI_VID_STATIC) {
/*
* Although from high-level point of view static volumes may
if (ubi->corr_peb_count)
ubi_err(ubi, "%d PEBs are corrupted and not used",
ubi->corr_peb_count);
+ return -ENOSPC;
}
ubi->rsvd_pebs += reserved_pebs;
ubi->avail_pebs -= reserved_pebs;
if (ubi->corr_peb_count)
ubi_err(ubi, "%d PEBs are corrupted and not used",
ubi->corr_peb_count);
+ err = -ENOSPC;
goto out_free;
}
ubi->avail_pebs -= reserved_pebs;
dev->type = ARPHRD_ARCNET;
dev->netdev_ops = &arcnet_netdev_ops;
dev->header_ops = &arcnet_header_ops;
- dev->hard_header_len = sizeof(struct archdr);
+ dev->hard_header_len = sizeof(struct arc_hardware);
dev->mtu = choose_mtu();
dev->addr_len = ARCNET_ALEN;
*/
reg = _mv88e6xxx_reg_read(ds, REG_PORT(port), PORT_PCS_CTRL);
if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) {
+ reg &= ~PORT_PCS_CTRL_UNFORCED;
reg |= PORT_PCS_CTRL_FORCE_LINK |
PORT_PCS_CTRL_LINK_UP |
PORT_PCS_CTRL_DUPLEX_FULL |
reg |= PORT_CONTROL_FRAME_ETHER_TYPE_DSA;
else
reg |= PORT_CONTROL_FRAME_MODE_DSA;
+ reg |= PORT_CONTROL_FORWARD_UNKNOWN |
+ PORT_CONTROL_FORWARD_UNKNOWN_MC;
}
if (mv88e6xxx_6352_family(ds) || mv88e6xxx_6351_family(ds) ||
netdev_dbg(ndev, "No phy-handle found in DT\n");
return -ENODEV;
}
- pdata->phy_dev = of_phy_find_device(phy_np);
- }
- phy_dev = pdata->phy_dev;
+ phy_dev = of_phy_connect(ndev, phy_np, &xgene_enet_adjust_link,
+ 0, pdata->phy_mode);
+ if (!phy_dev) {
+ netdev_err(ndev, "Could not connect to PHY\n");
+ return -ENODEV;
+ }
+
+ pdata->phy_dev = phy_dev;
+ } else {
+ phy_dev = pdata->phy_dev;
- if (!phy_dev ||
- phy_connect_direct(ndev, phy_dev, &xgene_enet_adjust_link,
- pdata->phy_mode)) {
- netdev_err(ndev, "Could not connect to PHY\n");
- return -ENODEV;
+ if (!phy_dev ||
+ phy_connect_direct(ndev, phy_dev, &xgene_enet_adjust_link,
+ pdata->phy_mode)) {
+ netdev_err(ndev, "Could not connect to PHY\n");
+ return -ENODEV;
+ }
}
pdata->phy_speed = SPEED_UNKNOWN;
{ .compatible = "snps,arc-emac" },
{ /* Sentinel */ }
};
+MODULE_DEVICE_TABLE(of, emac_arc_dt_ids);
static struct platform_driver emac_arc_driver = {
.probe = emac_arc_probe,
{ .compatible = "brcm,systemport" },
{ /* sentinel */ }
};
+MODULE_DEVICE_TABLE(of, bcm_sysport_of_match);
static struct platform_driver bcm_sysport_driver = {
.probe = bcm_sysport_probe,
u16 vlan_cnt;
u16 vlan_credit;
u16 vxlan_dst_port;
+ u8 vxlan_dst_port_count;
bool accept_any_vlan;
};
void bnx2x_update_mfw_dump(struct bnx2x *bp)
{
- struct timeval epoc;
u32 drv_ver;
u32 valid_dump;
if (!SHMEM2_HAS(bp, drv_info))
return;
- /* Update Driver load time */
- do_gettimeofday(&epoc);
- SHMEM2_WR(bp, drv_info.epoc, epoc.tv_sec);
+ /* Update Driver load time, possibly broken in y2038 */
+ SHMEM2_WR(bp, drv_info.epoc, (u32)ktime_get_real_seconds());
drv_ver = bnx2x_update_mng_version_utility(DRV_MODULE_VERSION, true);
SHMEM2_WR(bp, drv_info.drv_ver, drv_ver);
if (!netif_running(bp->dev))
return;
- if (bp->vxlan_dst_port || !IS_PF(bp)) {
+ if (bp->vxlan_dst_port_count && bp->vxlan_dst_port == port) {
+ bp->vxlan_dst_port_count++;
+ return;
+ }
+
+ if (bp->vxlan_dst_port_count || !IS_PF(bp)) {
DP(BNX2X_MSG_SP, "Vxlan destination port limit reached\n");
return;
}
bp->vxlan_dst_port = port;
+ bp->vxlan_dst_port_count = 1;
bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_ADD_VXLAN_PORT, 0);
}
static void __bnx2x_del_vxlan_port(struct bnx2x *bp, u16 port)
{
- if (!bp->vxlan_dst_port || bp->vxlan_dst_port != port || !IS_PF(bp)) {
+ if (!bp->vxlan_dst_port_count || bp->vxlan_dst_port != port ||
+ !IS_PF(bp)) {
DP(BNX2X_MSG_SP, "Invalid vxlan port\n");
return;
}
+ bp->vxlan_dst_port--;
+ if (bp->vxlan_dst_port)
+ return;
if (netif_running(bp->dev)) {
bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_DEL_VXLAN_PORT, 0);
/* RSS keys */
if (test_bit(BNX2X_RSS_SET_SRCH, &p->rss_flags)) {
- memcpy(&data->rss_key[0], &p->rss_key[0],
- sizeof(data->rss_key));
+ u8 *dst = (u8 *)(data->rss_key) + sizeof(data->rss_key);
+ const u8 *src = (const u8 *)p->rss_key;
+ int i;
+
+ /* Apparently, bnx2x reads this array in reverse order
+ * We need to byte swap rss_key to comply with Toeplitz specs.
+ */
+ for (i = 0; i < sizeof(data->rss_key); i++)
+ *--dst = *src++;
+
caps |= ETH_RSS_UPDATE_RAMROD_DATA_UPDATE_RSS_KEY;
}
{ .compatible = "brcm,genet-v4", .data = (void *)GENET_V4 },
{ },
};
+MODULE_DEVICE_TABLE(of, bcmgenet_match);
static int bcmgenet_probe(struct platform_device *pdev)
{
}
/* Flush FLI data fifo. */
-static u32
+static int
bfa_flash_fifo_flush(void __iomem *pci_bar)
{
u32 i;
}
/* Read flash status. */
-static u32
+static int
bfa_flash_status_read(void __iomem *pci_bar)
{
union bfa_flash_dev_status_reg dev_status;
- u32 status;
+ int status;
u32 ret_status;
int i;
}
/* Start flash read operation. */
-static u32
+static int
bfa_flash_read_start(void __iomem *pci_bar, u32 offset, u32 len,
char *buf)
{
- u32 status;
+ int status;
/* len must be mutiple of 4 and not exceeding fifo size */
if (len == 0 || len > BFA_FLASH_FIFO_SIZE || (len & 0x03) != 0)
bfa_flash_raw_read(void __iomem *pci_bar, u32 offset, char *buf,
u32 len)
{
- u32 n, status;
+ u32 n;
+ int status;
u32 off, l, s, residue, fifo_sz;
residue = len;
q0->rcb->id = 0;
q0->rx_packets = q0->rx_bytes = 0;
q0->rx_packets_with_error = q0->rxbuf_alloc_failed = 0;
+ q0->rxbuf_map_failed = 0;
bna_rxq_qpt_setup(q0, rxp, dpage_count, PAGE_SIZE,
&dqpt_mem[i], &dsqpt_mem[i], &dpage_mem[i]);
: rx_cfg->q1_buf_size;
q1->rx_packets = q1->rx_bytes = 0;
q1->rx_packets_with_error = q1->rxbuf_alloc_failed = 0;
+ q1->rxbuf_map_failed = 0;
bna_rxq_qpt_setup(q1, rxp, hpage_count, PAGE_SIZE,
&hqpt_mem[i], &hsqpt_mem[i],
u64 rx_bytes;
u64 rx_packets_with_error;
u64 rxbuf_alloc_failed;
+ u64 rxbuf_map_failed;
};
/* RxQ pair */
}
dma_addr = dma_map_page(&bnad->pcidev->dev, page, page_offset,
- unmap_q->map_size, DMA_FROM_DEVICE);
+ unmap_q->map_size, DMA_FROM_DEVICE);
+ if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
+ put_page(page);
+ BNAD_UPDATE_CTR(bnad, rxbuf_map_failed);
+ rcb->rxq->rxbuf_map_failed++;
+ goto finishing;
+ }
unmap->page = page;
unmap->page_offset = page_offset;
rcb->rxq->rxbuf_alloc_failed++;
goto finishing;
}
+
dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data,
buff_sz, DMA_FROM_DEVICE);
+ if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
+ dev_kfree_skb_any(skb);
+ BNAD_UPDATE_CTR(bnad, rxbuf_map_failed);
+ rcb->rxq->rxbuf_map_failed++;
+ goto finishing;
+ }
unmap->skb = skb;
dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr);
unmap = head_unmap;
dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data,
len, DMA_TO_DEVICE);
+ if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
+ dev_kfree_skb_any(skb);
+ BNAD_UPDATE_CTR(bnad, tx_skb_map_failed);
+ return NETDEV_TX_OK;
+ }
BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[0].host_addr);
txqent->vector[0].length = htons(len);
dma_unmap_addr_set(&unmap->vectors[0], dma_addr, dma_addr);
dma_addr = skb_frag_dma_map(&bnad->pcidev->dev, frag,
0, size, DMA_TO_DEVICE);
+ if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
+ /* Undo the changes starting at tcb->producer_index */
+ bnad_tx_buff_unmap(bnad, unmap_q, q_depth,
+ tcb->producer_index);
+ dev_kfree_skb_any(skb);
+ BNAD_UPDATE_CTR(bnad, tx_skb_map_failed);
+ return NETDEV_TX_OK;
+ }
+
dma_unmap_len_set(&unmap->vectors[vect_id], dma_len, size);
BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[vect_id].host_addr);
txqent->vector[vect_id].length = htons(size);
u64 tx_skb_headlen_zero;
u64 tx_skb_frag_zero;
u64 tx_skb_len_mismatch;
+ u64 tx_skb_map_failed;
u64 hw_stats_updates;
u64 netif_rx_dropped;
u64 rx_unmap_q_alloc_failed;
u64 rxbuf_alloc_failed;
+ u64 rxbuf_map_failed;
};
/* Complete driver stats */
"tx_skb_headlen_zero",
"tx_skb_frag_zero",
"tx_skb_len_mismatch",
+ "tx_skb_map_failed",
"hw_stats_updates",
"netif_rx_dropped",
"tx_unmap_q_alloc_failed",
"rx_unmap_q_alloc_failed",
"rxbuf_alloc_failed",
+ "rxbuf_map_failed",
"mac_stats_clr_cnt",
"mac_frame_64",
rx_packets_with_error;
buf[bi++] = rcb->rxq->
rxbuf_alloc_failed;
+ buf[bi++] = rcb->rxq->rxbuf_map_failed;
buf[bi++] = rcb->producer_index;
buf[bi++] = rcb->consumer_index;
}
rx_packets_with_error;
buf[bi++] = rcb->rxq->
rxbuf_alloc_failed;
+ buf[bi++] = rcb->rxq->rxbuf_map_failed;
buf[bi++] = rcb->producer_index;
buf[bi++] = rcb->consumer_index;
}
CH_PCI_ID_TABLE_FENTRY(0x5090), /* Custom T540-CR */
CH_PCI_ID_TABLE_FENTRY(0x5091), /* Custom T522-CR */
CH_PCI_ID_TABLE_FENTRY(0x5092), /* Custom T520-CR */
+ CH_PCI_ID_TABLE_FENTRY(0x5093), /* Custom T580-LP-CR */
+ CH_PCI_ID_TABLE_FENTRY(0x5094), /* Custom T540-CR */
+ CH_PCI_ID_TABLE_FENTRY(0x5095), /* Custom T540-CR-SO */
+ CH_PCI_ID_TABLE_FENTRY(0x5096), /* Custom T580-CR */
+ CH_PCI_ID_TABLE_FENTRY(0x5097), /* Custom T520-KR */
/* T6 adapters:
*/
u16 pvid;
__be16 vxlan_port;
int vxlan_port_count;
+ int vxlan_port_aliases;
struct phy_info phy;
u8 wol_cap;
bool wol_en;
if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
return;
+ if (adapter->vxlan_port == port && adapter->vxlan_port_count) {
+ adapter->vxlan_port_aliases++;
+ return;
+ }
+
if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
dev_info(dev,
"Only one UDP port supported for VxLAN offloads\n");
if (adapter->vxlan_port != port)
goto done;
+ if (adapter->vxlan_port_aliases) {
+ adapter->vxlan_port_aliases--;
+ return;
+ }
+
be_disable_vxlan_offloads(adapter);
dev_info(&adapter->pdev->dev,
* everything for us? Resetting it takes the link down and requires
* several seconds for it to come back.
*/
- if (phy_read(tbiphy, MII_BMSR) & BMSR_LSTATUS)
+ if (phy_read(tbiphy, MII_BMSR) & BMSR_LSTATUS) {
+ put_device(&tbiphy->dev);
return;
+ }
/* Single clk mode, mii mode off(for serdes communication) */
phy_write(tbiphy, MII_TBICON, TBICON_CLK_SELECT);
phy_write(tbiphy, MII_BMCR,
BMCR_ANENABLE | BMCR_ANRESTART | BMCR_FULLDPLX |
BMCR_SPEED1000);
+
+ put_device(&tbiphy->dev);
}
static int __gfar_is_rx_idle(struct gfar_private *priv)
/* Install our interrupt handlers for Error,
* Transmit, and Receive
*/
- err = request_irq(gfar_irq(grp, ER)->irq, gfar_error,
- IRQF_NO_SUSPEND,
+ err = request_irq(gfar_irq(grp, ER)->irq, gfar_error, 0,
gfar_irq(grp, ER)->name, grp);
if (err < 0) {
netif_err(priv, intr, dev, "Can't get IRQ %d\n",
goto err_irq_fail;
}
+ enable_irq_wake(gfar_irq(grp, ER)->irq);
+
err = request_irq(gfar_irq(grp, TX)->irq, gfar_transmit, 0,
gfar_irq(grp, TX)->name, grp);
if (err < 0) {
goto rx_irq_fail;
}
} else {
- err = request_irq(gfar_irq(grp, TX)->irq, gfar_interrupt,
- IRQF_NO_SUSPEND,
+ err = request_irq(gfar_irq(grp, TX)->irq, gfar_interrupt, 0,
gfar_irq(grp, TX)->name, grp);
if (err < 0) {
netif_err(priv, intr, dev, "Can't get IRQ %d\n",
gfar_irq(grp, TX)->irq);
goto err_irq_fail;
}
+ enable_irq_wake(gfar_irq(grp, TX)->irq);
}
return 0;
{ .compatible = "fsl,etsec-ptp" },
{},
};
+MODULE_DEVICE_TABLE(of, match_table);
static struct platform_driver gianfar_ptp_driver = {
.driver = {
value = phy_read(tbiphy, ENET_TBI_MII_CR);
value &= ~0x1000; /* Turn off autonegotiation */
phy_write(tbiphy, ENET_TBI_MII_CR, value);
+
+ put_device(&tbiphy->dev);
}
init_check_frame_length_mode(ug_info->lengthCheckRx, &ug_regs->maccfg2);
* everything for us? Resetting it takes the link down and requires
* several seconds for it to come back.
*/
- if (phy_read(tbiphy, ENET_TBI_MII_SR) & TBISR_LSTATUS)
+ if (phy_read(tbiphy, ENET_TBI_MII_SR) & TBISR_LSTATUS) {
+ put_device(&tbiphy->dev);
return;
+ }
/* Single clk mode, mii mode off(for serdes communication) */
phy_write(tbiphy, ENET_TBI_MII_ANA, TBIANA_SETTINGS);
phy_write(tbiphy, ENET_TBI_MII_TBICON, TBICON_CLK_SELECT);
phy_write(tbiphy, ENET_TBI_MII_CR, TBICR_SETTINGS);
+
+ put_device(&tbiphy->dev);
}
/* Configure the PHY for dev.
struct net_device *ndev;
struct hip04_priv *priv;
struct resource *res;
- unsigned int irq;
+ int irq;
int ret;
ndev = alloc_etherdev(sizeof(struct hip04_priv));
u32 index;
};
-#define EMAC_ETHTOOL_REGS_VER 0
-#define EMAC4_ETHTOOL_REGS_VER 1
-#define EMAC4SYNC_ETHTOOL_REGS_VER 2
+#define EMAC_ETHTOOL_REGS_VER 3
+#define EMAC4_ETHTOOL_REGS_VER 4
+#define EMAC4SYNC_ETHTOOL_REGS_VER 5
#endif /* __IBM_NEWEMAC_CORE_H */
/* take the lock before we start messing with the ring */
mutex_lock(&hw->aq.arq_mutex);
+ if (hw->aq.arq.count == 0) {
+ i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
+ "AQRX: Admin queue not initialized.\n");
+ ret_code = I40E_ERR_QUEUE_EMPTY;
+ goto clean_arq_element_err;
+ }
+
/* set next_to_use to head */
ntu = (rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK);
if (ntu == ntc) {
/* Set pending if needed, unlock and return */
if (pending != NULL)
*pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);
+
+clean_arq_element_err:
mutex_unlock(&hw->aq.arq_mutex);
if (i40e_is_nvm_update_op(&e->desc)) {
rx_ctx.lrxqthresh = 2;
rx_ctx.crcstrip = 1;
rx_ctx.l2tsel = 1;
- rx_ctx.showiv = 1;
+ /* this controls whether VLAN is stripped from inner headers */
+ rx_ctx.showiv = 0;
#ifdef I40E_FCOE
rx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
#endif
/* take the lock before we start messing with the ring */
mutex_lock(&hw->aq.arq_mutex);
+ if (hw->aq.arq.count == 0) {
+ i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
+ "AQRX: Admin queue not initialized.\n");
+ ret_code = I40E_ERR_QUEUE_EMPTY;
+ goto clean_arq_element_err;
+ }
+
/* set next_to_use to head */
ntu = (rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK);
if (ntu == ntc) {
/* Set pending if needed, unlock and return */
if (pending != NULL)
*pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);
+
+clean_arq_element_err:
mutex_unlock(&hw->aq.arq_mutex);
return ret_code;
struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
struct sk_buff *skb;
unsigned char *data;
+ dma_addr_t phys_addr;
u32 rx_status;
int rx_bytes, err;
rx_status = rx_desc->status;
rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE);
data = (unsigned char *)rx_desc->buf_cookie;
+ phys_addr = rx_desc->buf_phys_addr;
if (!mvneta_rxq_desc_is_first_last(rx_status) ||
(rx_status & MVNETA_RXD_ERR_SUMMARY)) {
if (!skb)
goto err_drop_frame;
- dma_unmap_single(dev->dev.parent, rx_desc->buf_phys_addr,
+ dma_unmap_single(dev->dev.parent, phys_addr,
MVNETA_RX_BUF_SIZE(pp->pkt_size), DMA_FROM_DEVICE);
rcvd_pkts++;
struct phy_device *phy = of_phy_find_device(dn);
mvneta_fixed_link_update(pp, phy);
+
+ put_device(&phy->dev);
}
return 0;
/* If we used up all the quota - we're probably not done yet... */
if (done == budget) {
- int cpu_curr;
const struct cpumask *aff;
+ struct irq_data *idata;
+ int cpu_curr;
INC_PERF_COUNTER(priv->pstats.napi_quota);
cpu_curr = smp_processor_id();
- aff = irq_desc_get_irq_data(cq->irq_desc)->affinity;
+ idata = irq_desc_get_irq_data(cq->irq_desc);
+ aff = irq_data_get_affinity_mask(idata);
if (likely(cpumask_test_cpu(cpu_curr, aff)))
return budget;
rss_context->hash_fn = MLX4_RSS_HASH_TOP;
memcpy(rss_context->rss_key, priv->rss_key,
MLX4_EN_RSS_KEY_SIZE);
- netdev_rss_key_fill(rss_context->rss_key,
- MLX4_EN_RSS_KEY_SIZE);
} else {
en_err(priv, "Unknown RSS hash function requested\n");
err = -EINVAL;
if (prot == MLX4_PROT_ETH) {
/* manage the steering entry for promisc mode */
if (new_entry)
- new_steering_entry(dev, port, steer, index, qp->qpn);
+ err = new_steering_entry(dev, port, steer,
+ index, qp->qpn);
else
- existing_steering_entry(dev, port, steer,
- index, qp->qpn);
+ err = existing_steering_entry(dev, port, steer,
+ index, qp->qpn);
}
if (err && link && index != -1) {
if (index < dev->caps.num_mgms)
return err;
}
-
-int mlx5_core_query_special_context(struct mlx5_core_dev *dev, u32 *rsvd_lkey)
-{
- struct mlx5_cmd_query_special_contexts_mbox_in in;
- struct mlx5_cmd_query_special_contexts_mbox_out out;
- int err;
-
- memset(&in, 0, sizeof(in));
- memset(&out, 0, sizeof(out));
- in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
- err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
- if (err)
- return err;
-
- if (out.hdr.status)
- err = mlx5_cmd_status_to_err(&out.hdr);
-
- *rsvd_lkey = be32_to_cpu(out.resd_lkey);
-
- return err;
-}
-EXPORT_SYMBOL(mlx5_core_query_special_context);
{ .compatible = "micrel,ks8851" },
{ }
};
+MODULE_DEVICE_TABLE(of, ks8851_match_table);
static struct spi_driver ks8851_driver = {
.driver = {
{ .compatible = "moxa,moxart-mac" },
{ }
};
+MODULE_DEVICE_TABLE(of, moxart_mac_match);
static struct platform_driver moxart_mac_driver = {
.probe = moxart_mac_probe,
u8 extend_lb_time;
u8 phys_port_id[ETH_ALEN];
u8 lb_mode;
+ u8 vxlan_port_count;
u16 vxlan_port;
struct device *hwmon_dev;
u32 post_mode;
/* Adapter supports only one VXLAN port. Use very first port
* for enabling offload
*/
- if (!qlcnic_encap_rx_offload(adapter) || ahw->vxlan_port)
+ if (!qlcnic_encap_rx_offload(adapter))
return;
+ if (!ahw->vxlan_port_count) {
+ ahw->vxlan_port_count = 1;
+ ahw->vxlan_port = ntohs(port);
+ adapter->flags |= QLCNIC_ADD_VXLAN_PORT;
+ return;
+ }
+ if (ahw->vxlan_port == ntohs(port))
+ ahw->vxlan_port_count++;
- ahw->vxlan_port = ntohs(port);
- adapter->flags |= QLCNIC_ADD_VXLAN_PORT;
}
static void qlcnic_del_vxlan_port(struct net_device *netdev,
struct qlcnic_adapter *adapter = netdev_priv(netdev);
struct qlcnic_hardware_context *ahw = adapter->ahw;
- if (!qlcnic_encap_rx_offload(adapter) || !ahw->vxlan_port ||
+ if (!qlcnic_encap_rx_offload(adapter) || !ahw->vxlan_port_count ||
(ahw->vxlan_port != ntohs(port)))
return;
- adapter->flags |= QLCNIC_DEL_VXLAN_PORT;
+ ahw->vxlan_port_count--;
+ if (!ahw->vxlan_port_count)
+ adapter->flags |= QLCNIC_DEL_VXLAN_PORT;
}
static netdev_features_t qlcnic_features_check(struct sk_buff *skb,
NWayAdvert = 0x66, /* MII ADVERTISE */
NWayLPAR = 0x68, /* MII LPA */
NWayExpansion = 0x6A, /* MII Expansion */
+ TxDmaOkLowDesc = 0x82, /* Low 16 bit address of a Tx descriptor. */
Config5 = 0xD8, /* Config5 */
TxPoll = 0xD9, /* Tell chip to check Tx descriptors for work */
RxMaxSize = 0xDA, /* Max size of an Rx packet (8169 only) */
unsigned tx_tail;
struct cp_desc *tx_ring;
struct sk_buff *tx_skb[CP_TX_RING_SIZE];
+ u32 tx_opts[CP_TX_RING_SIZE];
unsigned rx_buf_sz;
unsigned wol_enabled : 1; /* Is Wake-on-LAN enabled? */
BUG_ON(!skb);
dma_unmap_single(&cp->pdev->dev, le64_to_cpu(txd->addr),
- le32_to_cpu(txd->opts1) & 0xffff,
+ cp->tx_opts[tx_tail] & 0xffff,
PCI_DMA_TODEVICE);
if (status & LastFrag) {
{
struct cp_private *cp = netdev_priv(dev);
unsigned entry;
- u32 eor, flags;
+ u32 eor, opts1;
unsigned long intr_flags;
__le32 opts2;
int mss = 0;
mss = skb_shinfo(skb)->gso_size;
opts2 = cpu_to_le32(cp_tx_vlan_tag(skb));
+ opts1 = DescOwn;
+ if (mss)
+ opts1 |= LargeSend | ((mss & MSSMask) << MSSShift);
+ else if (skb->ip_summed == CHECKSUM_PARTIAL) {
+ const struct iphdr *ip = ip_hdr(skb);
+ if (ip->protocol == IPPROTO_TCP)
+ opts1 |= IPCS | TCPCS;
+ else if (ip->protocol == IPPROTO_UDP)
+ opts1 |= IPCS | UDPCS;
+ else {
+ WARN_ONCE(1,
+ "Net bug: asked to checksum invalid Legacy IP packet\n");
+ goto out_dma_error;
+ }
+ }
if (skb_shinfo(skb)->nr_frags == 0) {
struct cp_desc *txd = &cp->tx_ring[entry];
txd->addr = cpu_to_le64(mapping);
wmb();
- flags = eor | len | DescOwn | FirstFrag | LastFrag;
-
- if (mss)
- flags |= LargeSend | ((mss & MSSMask) << MSSShift);
- else if (skb->ip_summed == CHECKSUM_PARTIAL) {
- const struct iphdr *ip = ip_hdr(skb);
- if (ip->protocol == IPPROTO_TCP)
- flags |= IPCS | TCPCS;
- else if (ip->protocol == IPPROTO_UDP)
- flags |= IPCS | UDPCS;
- else
- WARN_ON(1); /* we need a WARN() */
- }
+ opts1 |= eor | len | FirstFrag | LastFrag;
- txd->opts1 = cpu_to_le32(flags);
+ txd->opts1 = cpu_to_le32(opts1);
wmb();
cp->tx_skb[entry] = skb;
- entry = NEXT_TX(entry);
+ cp->tx_opts[entry] = opts1;
+ netif_dbg(cp, tx_queued, cp->dev, "tx queued, slot %d, skblen %d\n",
+ entry, skb->len);
} else {
struct cp_desc *txd;
- u32 first_len, first_eor;
+ u32 first_len, first_eor, ctrl;
dma_addr_t first_mapping;
int frag, first_entry = entry;
- const struct iphdr *ip = ip_hdr(skb);
/* We must give this initial chunk to the device last.
* Otherwise we could race with the device.
goto out_dma_error;
cp->tx_skb[entry] = skb;
- entry = NEXT_TX(entry);
for (frag = 0; frag < skb_shinfo(skb)->nr_frags; frag++) {
const skb_frag_t *this_frag = &skb_shinfo(skb)->frags[frag];
u32 len;
- u32 ctrl;
dma_addr_t mapping;
+ entry = NEXT_TX(entry);
+
len = skb_frag_size(this_frag);
mapping = dma_map_single(&cp->pdev->dev,
skb_frag_address(this_frag),
eor = (entry == (CP_TX_RING_SIZE - 1)) ? RingEnd : 0;
- ctrl = eor | len | DescOwn;
-
- if (mss)
- ctrl |= LargeSend |
- ((mss & MSSMask) << MSSShift);
- else if (skb->ip_summed == CHECKSUM_PARTIAL) {
- if (ip->protocol == IPPROTO_TCP)
- ctrl |= IPCS | TCPCS;
- else if (ip->protocol == IPPROTO_UDP)
- ctrl |= IPCS | UDPCS;
- else
- BUG();
- }
+ ctrl = opts1 | eor | len;
if (frag == skb_shinfo(skb)->nr_frags - 1)
ctrl |= LastFrag;
txd->opts1 = cpu_to_le32(ctrl);
wmb();
+ cp->tx_opts[entry] = ctrl;
cp->tx_skb[entry] = skb;
- entry = NEXT_TX(entry);
}
txd = &cp->tx_ring[first_entry];
txd->addr = cpu_to_le64(first_mapping);
wmb();
- if (skb->ip_summed == CHECKSUM_PARTIAL) {
- if (ip->protocol == IPPROTO_TCP)
- txd->opts1 = cpu_to_le32(first_eor | first_len |
- FirstFrag | DescOwn |
- IPCS | TCPCS);
- else if (ip->protocol == IPPROTO_UDP)
- txd->opts1 = cpu_to_le32(first_eor | first_len |
- FirstFrag | DescOwn |
- IPCS | UDPCS);
- else
- BUG();
- } else
- txd->opts1 = cpu_to_le32(first_eor | first_len |
- FirstFrag | DescOwn);
+ ctrl = opts1 | first_eor | first_len | FirstFrag;
+ txd->opts1 = cpu_to_le32(ctrl);
wmb();
+
+ cp->tx_opts[first_entry] = ctrl;
+ netif_dbg(cp, tx_queued, cp->dev, "tx queued, slots %d-%d, skblen %d\n",
+ first_entry, entry, skb->len);
}
- cp->tx_head = entry;
+ cp->tx_head = NEXT_TX(entry);
netdev_sent_queue(dev, skb->len);
- netif_dbg(cp, tx_queued, cp->dev, "tx queued, slot %d, skblen %d\n",
- entry, skb->len);
if (TX_BUFFS_AVAIL(cp) <= (MAX_SKB_FRAGS + 1))
netif_stop_queue(dev);
{
memset(cp->tx_ring, 0, sizeof(struct cp_desc) * CP_TX_RING_SIZE);
cp->tx_ring[CP_TX_RING_SIZE - 1].opts1 = cpu_to_le32(RingEnd);
+ memset(cp->tx_opts, 0, sizeof(cp->tx_opts));
cp_init_rings_index(cp);
desc = cp->rx_ring + i;
dma_unmap_single(&cp->pdev->dev,le64_to_cpu(desc->addr),
cp->rx_buf_sz, PCI_DMA_FROMDEVICE);
- dev_kfree_skb(cp->rx_skb[i]);
+ dev_kfree_skb_any(cp->rx_skb[i]);
}
}
le32_to_cpu(desc->opts1) & 0xffff,
PCI_DMA_TODEVICE);
if (le32_to_cpu(desc->opts1) & LastFrag)
- dev_kfree_skb(skb);
+ dev_kfree_skb_any(skb);
cp->dev->stats.tx_dropped++;
}
}
memset(cp->rx_ring, 0, sizeof(struct cp_desc) * CP_RX_RING_SIZE);
memset(cp->tx_ring, 0, sizeof(struct cp_desc) * CP_TX_RING_SIZE);
+ memset(cp->tx_opts, 0, sizeof(cp->tx_opts));
memset(cp->rx_skb, 0, sizeof(struct sk_buff *) * CP_RX_RING_SIZE);
memset(cp->tx_skb, 0, sizeof(struct sk_buff *) * CP_TX_RING_SIZE);
{
struct cp_private *cp = netdev_priv(dev);
unsigned long flags;
- int rc;
+ int rc, i;
netdev_warn(dev, "Transmit timeout, status %2x %4x %4x %4x\n",
cpr8(Cmd), cpr16(CpCmd),
spin_lock_irqsave(&cp->lock, flags);
+ netif_dbg(cp, tx_err, cp->dev, "TX ring head %d tail %d desc %x\n",
+ cp->tx_head, cp->tx_tail, cpr16(TxDmaOkLowDesc));
+ for (i = 0; i < CP_TX_RING_SIZE; i++) {
+ netif_dbg(cp, tx_err, cp->dev,
+ "TX slot %d @%p: %08x (%08x) %08x %llx %p\n",
+ i, &cp->tx_ring[i], le32_to_cpu(cp->tx_ring[i].opts1),
+ cp->tx_opts[i], le32_to_cpu(cp->tx_ring[i].opts2),
+ le64_to_cpu(cp->tx_ring[i].addr),
+ cp->tx_skb[i]);
+ }
+
cp_stop_hw(cp);
cp_clean_rings(cp);
rc = cp_init_rings(cp);
cp_start_hw(cp);
- cp_enable_irq(cp);
+ __cp_set_rx_mode(dev);
+ cpw16_f(IntrMask, cp_norx_intr_mask);
netif_wake_queue(dev);
+ napi_schedule_irqoff(&cp->napi);
spin_unlock_irqrestore(&cp->lock, flags);
}
{
void __iomem *ioaddr = tp->mmio_addr;
struct pci_dev *pdev = tp->pci_dev;
- u16 rg_saw_cnt;
+ int rg_saw_cnt;
u32 data;
static const struct ephy_info e_info_8168h_1[] = {
{ 0x1e, 0x0800, 0x0001 },
if (!gpio_request(reset_gpio, "mdio-reset")) {
gpio_direction_output(reset_gpio, active_low ? 1 : 0);
- udelay(data->delays[0]);
+ if (data->delays[0])
+ msleep(DIV_ROUND_UP(data->delays[0], 1000));
+
gpio_set_value(reset_gpio, active_low ? 0 : 1);
- udelay(data->delays[1]);
+ if (data->delays[1])
+ msleep(DIV_ROUND_UP(data->delays[1], 1000));
+
gpio_set_value(reset_gpio, active_low ? 1 : 0);
- udelay(data->delays[2]);
+ if (data->delays[2])
+ msleep(DIV_ROUND_UP(data->delays[2], 1000));
}
}
#endif
#endif
};
-static struct vnet *vnet_new(const u64 *local_mac)
+static struct vnet *vnet_new(const u64 *local_mac,
+ struct vio_dev *vdev)
{
struct net_device *dev;
struct vnet *vp;
NETIF_F_HW_CSUM | NETIF_F_SG;
dev->features = dev->hw_features;
+ SET_NETDEV_DEV(dev, &vdev->dev);
+
err = register_netdev(dev);
if (err) {
pr_err("Cannot register net device, aborting\n");
return ERR_PTR(err);
}
-static struct vnet *vnet_find_or_create(const u64 *local_mac)
+static struct vnet *vnet_find_or_create(const u64 *local_mac,
+ struct vio_dev *vdev)
{
struct vnet *iter, *vp;
}
}
if (!vp)
- vp = vnet_new(local_mac);
+ vp = vnet_new(local_mac, vdev);
mutex_unlock(&vnet_list_mutex);
return vp;
static const char *local_mac_prop = "local-mac-address";
static struct vnet *vnet_find_parent(struct mdesc_handle *hp,
- u64 port_node)
+ u64 port_node,
+ struct vio_dev *vdev)
{
const u64 *local_mac = NULL;
u64 a;
if (!local_mac)
return ERR_PTR(-ENODEV);
- return vnet_find_or_create(local_mac);
+ return vnet_find_or_create(local_mac, vdev);
}
static struct ldc_channel_config vnet_ldc_cfg = {
hp = mdesc_grab();
- vp = vnet_find_parent(hp, vdev->mp);
+ vp = vnet_find_parent(hp, vdev->mp, vdev);
if (IS_ERR(vp)) {
pr_err("Cannot find port parent vnet\n");
err = PTR_ERR(vp);
interface_list) {
struct netcp_intf_modpriv *intf_modpriv;
- /* If interface not registered then register now */
- if (!netcp_intf->netdev_registered)
- ret = netcp_register_interface(netcp_intf);
-
- if (ret)
- return -ENODEV;
-
intf_modpriv = devm_kzalloc(dev, sizeof(*intf_modpriv),
GFP_KERNEL);
if (!intf_modpriv)
interface = of_parse_phandle(netcp_intf->node_interface,
module->name, 0);
+ if (!interface) {
+ devm_kfree(dev, intf_modpriv);
+ continue;
+ }
+
intf_modpriv->netcp_priv = netcp_intf;
intf_modpriv->netcp_module = module;
list_add_tail(&intf_modpriv->intf_list,
continue;
}
}
+
+ /* Now register the interface with netdev */
+ list_for_each_entry(netcp_intf,
+ &netcp_device->interface_head,
+ interface_list) {
+ /* If interface not registered then register now */
+ if (!netcp_intf->netdev_registered) {
+ ret = netcp_register_interface(netcp_intf);
+ if (ret)
+ return -ENODEV;
+ }
+ }
return 0;
}
if (ret < 0)
goto fail;
}
-
mutex_unlock(&netcp_modules_lock);
return 0;
netcp->rx_pool = NULL;
}
-static void netcp_allocate_rx_buf(struct netcp_intf *netcp, int fdq)
+static int netcp_allocate_rx_buf(struct netcp_intf *netcp, int fdq)
{
struct knav_dma_desc *hwdesc;
unsigned int buf_len, dma_sz;
hwdesc = knav_pool_desc_get(netcp->rx_pool);
if (IS_ERR_OR_NULL(hwdesc)) {
dev_dbg(netcp->ndev_dev, "out of rx pool desc\n");
- return;
+ return -ENOMEM;
}
if (likely(fdq == 0)) {
knav_pool_desc_map(netcp->rx_pool, hwdesc, sizeof(*hwdesc), &dma,
&dma_sz);
knav_queue_push(netcp->rx_fdq[fdq], dma, sizeof(*hwdesc), 0);
- return;
+ return 0;
fail:
knav_pool_desc_put(netcp->rx_pool, hwdesc);
+ return -ENOMEM;
}
/* Refill Rx FDQ with descriptors & attached buffers */
static void netcp_rxpool_refill(struct netcp_intf *netcp)
{
u32 fdq_deficit[KNAV_DMA_FDQ_PER_CHAN] = {0};
- int i;
+ int i, ret = 0;
/* Calculate the FDQ deficit and refill */
for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && netcp->rx_fdq[i]; i++) {
fdq_deficit[i] = netcp->rx_queue_depths[i] -
knav_queue_get_count(netcp->rx_fdq[i]);
- while (fdq_deficit[i]--)
- netcp_allocate_rx_buf(netcp, i);
+ while (fdq_deficit[i]-- && !ret)
+ ret = netcp_allocate_rx_buf(netcp, i);
} /* end for fdqs */
}
packets = netcp_process_rx_packets(netcp, budget);
+ netcp_rxpool_refill(netcp);
if (packets < budget) {
napi_complete(&netcp->rx_napi);
knav_queue_enable_notify(netcp->rx_queue);
}
- netcp_rxpool_refill(netcp);
return packets;
}
continue;
dev_dbg(netcp->ndev_dev, "deleting address %pM, type %x\n",
naddr->addr, naddr->type);
- mutex_lock(&netcp_modules_lock);
for_each_module(netcp, priv) {
module = priv->netcp_module;
if (!module->del_addr)
naddr);
WARN_ON(error);
}
- mutex_unlock(&netcp_modules_lock);
netcp_addr_del(netcp, naddr);
}
}
continue;
dev_dbg(netcp->ndev_dev, "adding address %pM, type %x\n",
naddr->addr, naddr->type);
- mutex_lock(&netcp_modules_lock);
+
for_each_module(netcp, priv) {
module = priv->netcp_module;
if (!module->add_addr)
error = module->add_addr(priv->module_priv, naddr);
WARN_ON(error);
}
- mutex_unlock(&netcp_modules_lock);
}
}
ndev->flags & IFF_ALLMULTI ||
netdev_mc_count(ndev) > NETCP_MAX_MCAST_ADDR);
+ spin_lock(&netcp->lock);
/* first clear all marks */
netcp_addr_clear_mark(netcp);
/* finally sweep and callout into modules */
netcp_addr_sweep_del(netcp);
netcp_addr_sweep_add(netcp);
+ spin_unlock(&netcp->lock);
}
static void netcp_free_navigator_resources(struct netcp_intf *netcp)
goto fail;
}
- mutex_lock(&netcp_modules_lock);
for_each_module(netcp, intf_modpriv) {
module = intf_modpriv->netcp_module;
if (module->open) {
}
}
}
- mutex_unlock(&netcp_modules_lock);
napi_enable(&netcp->rx_napi);
napi_enable(&netcp->tx_napi);
if (module->close)
module->close(intf_modpriv->module_priv, ndev);
}
- mutex_unlock(&netcp_modules_lock);
fail:
netcp_free_navigator_resources(netcp);
napi_disable(&netcp->rx_napi);
napi_disable(&netcp->tx_napi);
- mutex_lock(&netcp_modules_lock);
for_each_module(netcp, intf_modpriv) {
module = intf_modpriv->netcp_module;
if (module->close) {
dev_err(netcp->ndev_dev, "Close failed\n");
}
}
- mutex_unlock(&netcp_modules_lock);
/* Recycle Rx descriptors from completion queue */
netcp_empty_rx_queue(netcp);
if (!netif_running(ndev))
return -EINVAL;
- mutex_lock(&netcp_modules_lock);
for_each_module(netcp, intf_modpriv) {
module = intf_modpriv->netcp_module;
if (!module->ioctl)
}
out:
- mutex_unlock(&netcp_modules_lock);
return (ret == 0) ? 0 : err;
}
struct netcp_intf *netcp = netdev_priv(ndev);
struct netcp_intf_modpriv *intf_modpriv;
struct netcp_module *module;
+ unsigned long flags;
int err = 0;
dev_dbg(netcp->ndev_dev, "adding rx vlan id: %d\n", vid);
- mutex_lock(&netcp_modules_lock);
+ spin_lock_irqsave(&netcp->lock, flags);
for_each_module(netcp, intf_modpriv) {
module = intf_modpriv->netcp_module;
if ((module->add_vid) && (vid != 0)) {
}
}
}
- mutex_unlock(&netcp_modules_lock);
+ spin_unlock_irqrestore(&netcp->lock, flags);
+
return err;
}
struct netcp_intf *netcp = netdev_priv(ndev);
struct netcp_intf_modpriv *intf_modpriv;
struct netcp_module *module;
+ unsigned long flags;
int err = 0;
dev_dbg(netcp->ndev_dev, "removing rx vlan id: %d\n", vid);
- mutex_lock(&netcp_modules_lock);
+ spin_lock_irqsave(&netcp->lock, flags);
for_each_module(netcp, intf_modpriv) {
module = intf_modpriv->netcp_module;
if (module->del_vid) {
}
}
}
- mutex_unlock(&netcp_modules_lock);
+ spin_unlock_irqrestore(&netcp->lock, flags);
return err;
}
struct device_node *child, *interfaces;
struct netcp_device *netcp_device;
struct device *dev = &pdev->dev;
- struct netcp_module *module;
int ret;
if (!node) {
/* Add the device instance to the list */
list_add_tail(&netcp_device->device_list, &netcp_devices);
- /* Probe & attach any modules already registered */
- mutex_lock(&netcp_modules_lock);
- for_each_netcp_module(module) {
- ret = netcp_module_probe(netcp_device, module);
- if (ret < 0)
- dev_err(dev, "module(%s) probe failed\n", module->name);
- }
- mutex_unlock(&netcp_modules_lock);
return 0;
probe_quit_interface:
#define GBENU_ALE_OFFSET 0x1e000
#define GBENU_HOST_PORT_NUM 0
#define GBENU_NUM_ALE_ENTRIES 1024
+#define GBENU_SGMII_MODULE_SIZE 0x100
/* 10G Ethernet SS defines */
#define XGBE_MODULE_NAME "netcp-xgbe"
#define XGBE_STATS2_MODULE 2
/* s: 0-based slave_port */
-#define SGMII_BASE(s) \
- (((s) < 2) ? gbe_dev->sgmii_port_regs : gbe_dev->sgmii_port34_regs)
+#define SGMII_BASE(d, s) \
+ (((s) < 2) ? (d)->sgmii_port_regs : (d)->sgmii_port34_regs)
#define GBE_TX_QUEUE 648
#define GBE_TXHOOK_ORDER 0
return;
if (!SLAVE_LINK_IS_XGMII(slave)) {
- if (gbe_dev->ss_version == GBE_SS_VERSION_14)
- sgmii_link_state =
- netcp_sgmii_get_port_link(SGMII_BASE(sp), sp);
- else
- sgmii_link_state =
- netcp_sgmii_get_port_link(
- gbe_dev->sgmii_port_regs, sp);
+ sgmii_link_state =
+ netcp_sgmii_get_port_link(SGMII_BASE(gbe_dev, sp), sp);
}
phy_link_state = gbe_phy_link_status(slave);
static void gbe_sgmii_rtreset(struct gbe_priv *priv,
struct gbe_slave *slave, bool set)
{
- void __iomem *sgmii_port_regs;
-
if (SLAVE_LINK_IS_XGMII(slave))
return;
- if ((priv->ss_version == GBE_SS_VERSION_14) && (slave->slave_num >= 2))
- sgmii_port_regs = priv->sgmii_port34_regs;
- else
- sgmii_port_regs = priv->sgmii_port_regs;
-
- netcp_sgmii_rtreset(sgmii_port_regs, slave->slave_num, set);
+ netcp_sgmii_rtreset(SGMII_BASE(priv, slave->slave_num),
+ slave->slave_num, set);
}
static void gbe_slave_stop(struct gbe_intf *intf)
static void gbe_sgmii_config(struct gbe_priv *priv, struct gbe_slave *slave)
{
- void __iomem *sgmii_port_regs;
-
- sgmii_port_regs = priv->sgmii_port_regs;
- if ((priv->ss_version == GBE_SS_VERSION_14) && (slave->slave_num >= 2))
- sgmii_port_regs = priv->sgmii_port34_regs;
+ if (SLAVE_LINK_IS_XGMII(slave))
+ return;
- if (!SLAVE_LINK_IS_XGMII(slave)) {
- netcp_sgmii_reset(sgmii_port_regs, slave->slave_num);
- netcp_sgmii_config(sgmii_port_regs, slave->slave_num,
- slave->link_interface);
- }
+ netcp_sgmii_reset(SGMII_BASE(priv, slave->slave_num), slave->slave_num);
+ netcp_sgmii_config(SGMII_BASE(priv, slave->slave_num), slave->slave_num,
+ slave->link_interface);
}
static int gbe_slave_open(struct gbe_intf *gbe_intf)
gbe_dev->switch_regs = regs;
gbe_dev->sgmii_port_regs = gbe_dev->ss_regs + GBENU_SGMII_MODULE_OFFSET;
+
+ /* Although sgmii modules are mem mapped to one contiguous
+ * region on GBENU devices, setting sgmii_port34_regs allows
+ * consistent code when accessing sgmii api
+ */
+ gbe_dev->sgmii_port34_regs = gbe_dev->sgmii_port_regs +
+ (2 * GBENU_SGMII_MODULE_SIZE);
+
gbe_dev->host_port_regs = gbe_dev->switch_regs + GBENU_HOST_PORT_OFFSET;
for (i = 0; i < (gbe_dev->max_num_ports); i++)
config VIA_RHINE
tristate "VIA Rhine support"
- depends on (PCI || OF_IRQ)
+ depends on PCI || (OF_IRQ && GENERIC_PCI_IOMAP)
depends on HAS_DMA
select CRC32
select MII
if (!phydev)
dev_info(dev,
"MDIO of the phy is not registered yet\n");
+ else
+ put_device(&phydev->dev);
return 0;
}
set_bit(epidx, &irq_bit);
break;
}
- }
-
- hw->ep_shm_info[epidx].es_status = info[epidx].es_status;
- hw->ep_shm_info[epidx].zone = info[epidx].zone;
+ hw->ep_shm_info[epidx].es_status =
+ info[epidx].es_status;
+ hw->ep_shm_info[epidx].zone = info[epidx].zone;
+ }
break;
}
__be32 addr;
int err;
+ iph = ip_hdr(skb); /* outer IP header... */
+
if (gs->collect_md) {
static u8 zero_vni[3];
addr = 0;
} else {
vni = gnvh->vni;
- iph = ip_hdr(skb); /* Still outer IP header... */
addr = iph->saddr;
}
skb_reset_network_header(skb);
- iph = ip_hdr(skb); /* Now inner IP header... */
err = IP_ECN_decapsulate(iph, skb);
if (unlikely(err)) {
struct geneve_sock *gs = geneve->sock;
struct ip_tunnel_info *info = NULL;
struct rtable *rt = NULL;
+ const struct iphdr *iip; /* interior IP header */
struct flowi4 fl4;
__u8 tos, ttl;
__be16 sport;
sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true);
skb_reset_mac_header(skb);
+ iip = ip_hdr(skb);
+
if (info) {
const struct ip_tunnel_key *key = &info->key;
u8 *opts = NULL;
if (unlikely(err))
goto err;
- tos = key->tos;
+ tos = ip_tunnel_ecn_encap(key->tos, iip, skb);
ttl = key->ttl;
df = key->tun_flags & TUNNEL_DONT_FRAGMENT ? htons(IP_DF) : 0;
} else {
- const struct iphdr *iip; /* interior IP header */
-
udp_csum = false;
err = geneve_build_skb(rt, skb, 0, geneve->vni,
0, NULL, udp_csum);
if (unlikely(err))
goto err;
- iip = ip_hdr(skb);
tos = ip_tunnel_ecn_encap(fl4.flowi4_tos, iip, skb);
ttl = geneve->ttl;
if (!ttl && IN_MULTICAST(ntohl(fl4.daddr)))
dev->features |= NETIF_F_RXCSUM;
dev->features |= NETIF_F_GSO_SOFTWARE;
- dev->vlan_features = dev->features;
- dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX;
-
dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
dev->hw_features |= NETIF_F_GSO_SOFTWARE;
- dev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX;
netif_keep_dst(dev);
dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE;
static int geneve_configure(struct net *net, struct net_device *dev,
__be32 rem_addr, __u32 vni, __u8 ttl, __u8 tos,
- __u16 dst_port, bool metadata)
+ __be16 dst_port, bool metadata)
{
struct geneve_net *gn = net_generic(net, geneve_net_id);
struct geneve_dev *t, *geneve = netdev_priv(dev);
geneve->ttl = ttl;
geneve->tos = tos;
- geneve->dst_port = htons(dst_port);
+ geneve->dst_port = dst_port;
geneve->collect_md = metadata;
- t = geneve_find_dev(gn, htons(dst_port), rem_addr, geneve->vni,
+ t = geneve_find_dev(gn, dst_port, rem_addr, geneve->vni,
&tun_on_same_port, &tun_collect_md);
if (t)
return -EBUSY;
static int geneve_newlink(struct net *net, struct net_device *dev,
struct nlattr *tb[], struct nlattr *data[])
{
- __u16 dst_port = GENEVE_UDP_PORT;
+ __be16 dst_port = htons(GENEVE_UDP_PORT);
__u8 ttl = 0, tos = 0;
bool metadata = false;
__be32 rem_addr;
tos = nla_get_u8(data[IFLA_GENEVE_TOS]);
if (data[IFLA_GENEVE_PORT])
- dst_port = nla_get_u16(data[IFLA_GENEVE_PORT]);
+ dst_port = nla_get_be16(data[IFLA_GENEVE_PORT]);
if (data[IFLA_GENEVE_COLLECT_METADATA])
metadata = true;
nla_total_size(sizeof(struct in_addr)) + /* IFLA_GENEVE_REMOTE */
nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_TTL */
nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_TOS */
- nla_total_size(sizeof(__u16)) + /* IFLA_GENEVE_PORT */
+ nla_total_size(sizeof(__be16)) + /* IFLA_GENEVE_PORT */
nla_total_size(0) + /* IFLA_GENEVE_COLLECT_METADATA */
0;
}
nla_put_u8(skb, IFLA_GENEVE_TOS, geneve->tos))
goto nla_put_failure;
- if (nla_put_u16(skb, IFLA_GENEVE_PORT, ntohs(geneve->dst_port)))
+ if (nla_put_be16(skb, IFLA_GENEVE_PORT, geneve->dst_port))
goto nla_put_failure;
if (geneve->collect_md) {
if (IS_ERR(dev))
return dev;
- err = geneve_configure(net, dev, 0, 0, 0, 0, dst_port, true);
+ err = geneve_configure(net, dev, 0, 0, 0, 0, htons(dst_port), true);
if (err) {
free_netdev(dev);
return ERR_PTR(err);
static void ali_ircc_sir_change_speed(struct ali_ircc_cb *priv, __u32 speed)
{
struct ali_ircc_cb *self = priv;
- unsigned long flags;
int iobase;
int fcr; /* FIFO control reg */
int lcr; /* Line control reg */
/* Update accounting for new speed */
self->io.speed = speed;
- spin_lock_irqsave(&self->lock, flags);
-
divisor = 115200/speed;
fcr = UART_FCR_ENABLE_FIFO;
/* without this, the connection will be broken after come back from FIR speed,
but with this, the SIR connection is harder to established */
outb((UART_MCR_DTR | UART_MCR_RTS | UART_MCR_OUT2), iobase+UART_MCR);
-
- spin_unlock_irqrestore(&self->lock, flags);
-
}
static void ali_ircc_change_dongle_speed(struct ali_ircc_cb *priv, int speed)
return 0;
case TUNSETSNDBUF:
- if (get_user(u, up))
+ if (get_user(s, sp))
return -EFAULT;
- q->sk.sk_sndbuf = u;
+ q->sk.sk_sndbuf = s;
return 0;
case TUNGETVNETHDRSZ:
struct fixed_mdio_bus *fmb = &platform_fmb;
struct fixed_phy *fp;
- if (!phydev || !phydev->bus)
+ if (!phydev || phydev->bus != fmb->mii_bus)
return -EINVAL;
list_for_each_entry(fp, &fmb->phys, node) {
int adv;
int err;
int lpa;
+ int lpagb;
int status = 0;
/* Update the link, but return if there
if (lpa < 0)
return lpa;
+ lpagb = phy_read(phydev, MII_STAT1000);
+ if (lpagb < 0)
+ return lpagb;
+
adv = phy_read(phydev, MII_ADVERTISE);
if (adv < 0)
return adv;
+ phydev->lp_advertising = mii_stat1000_to_ethtool_lpa_t(lpagb) |
+ mii_lpa_to_ethtool_lpa_t(lpa);
+
lpa &= adv;
if (status & MII_M1011_PHY_STATUS_FULLDUPLEX)
phydev->speed = SPEED_10;
phydev->pause = phydev->asym_pause = 0;
+ phydev->lp_advertising = 0;
}
return 0;
{ .compatible = "brcm,unimac-mdio", },
{ /* sentinel */ },
};
+MODULE_DEVICE_TABLE(of, unimac_mdio_ids);
static struct platform_driver unimac_mdio_driver = {
.driver = {
{ .compatible = "virtual,mdio-gpio", },
{ /* sentinel */ }
};
+MODULE_DEVICE_TABLE(of, mdio_gpio_of_match);
static struct platform_driver mdio_gpio_driver = {
.probe = mdio_gpio_probe,
if (!parent_bus_node)
return -ENODEV;
- parent_bus = of_mdio_find_bus(parent_bus_node);
- if (parent_bus == NULL) {
- ret_val = -EPROBE_DEFER;
- goto err_parent_bus;
- }
-
pb = devm_kzalloc(dev, sizeof(*pb), GFP_KERNEL);
if (pb == NULL) {
ret_val = -ENOMEM;
goto err_parent_bus;
}
+ parent_bus = of_mdio_find_bus(parent_bus_node);
+ if (parent_bus == NULL) {
+ ret_val = -EPROBE_DEFER;
+ goto err_parent_bus;
+ }
+
pb->switch_data = data;
pb->switch_fn = switch_fn;
pb->current_child = -1;
dev_info(dev, "Version " DRV_VERSION "\n");
return 0;
}
+
+ /* balance the reference of_mdio_find_bus() took */
+ put_device(&pb->mii_bus->dev);
+
err_parent_bus:
of_node_put(parent_bus_node);
return ret_val;
mdiobus_free(cb->mii_bus);
cb = cb->next;
}
+
+ /* balance the reference of_mdio_find_bus() in mdio_mux_init() took */
+ put_device(&pb->mii_bus->dev);
}
EXPORT_SYMBOL_GPL(mdio_mux_uninit);
* of_mdio_find_bus - Given an mii_bus node, find the mii_bus.
* @mdio_bus_np: Pointer to the mii_bus.
*
- * Returns a pointer to the mii_bus, or NULL if none found.
+ * Returns a reference to the mii_bus, or NULL if none found. The
+ * embedded struct device will have its reference count incremented,
+ * and this must be put once the bus is finished with.
*
* Because the association of a device_node and mii_bus is made via
* of_mdiobus_register(), the mii_bus cannot be found before it is
#endif
/**
- * mdiobus_register - bring up all the PHYs on a given bus and attach them to bus
+ * __mdiobus_register - bring up all the PHYs on a given bus and attach them to bus
* @bus: target mii_bus
+ * @owner: module containing bus accessor functions
*
* Description: Called by a bus driver to bring up all the PHYs
- * on a given bus, and attach them to the bus.
+ * on a given bus, and attach them to the bus. Drivers should use
+ * mdiobus_register() rather than __mdiobus_register() unless they
+ * need to pass a specific owner module.
*
* Returns 0 on success or < 0 on error.
*/
-int mdiobus_register(struct mii_bus *bus)
+int __mdiobus_register(struct mii_bus *bus, struct module *owner)
{
int i, err;
BUG_ON(bus->state != MDIOBUS_ALLOCATED &&
bus->state != MDIOBUS_UNREGISTERED);
+ bus->owner = owner;
bus->dev.parent = bus->parent;
bus->dev.class = &mdio_bus_class;
bus->dev.groups = NULL;
error:
while (--i >= 0) {
- if (bus->phy_map[i])
- device_unregister(&bus->phy_map[i]->dev);
+ struct phy_device *phydev = bus->phy_map[i];
+ if (phydev) {
+ phy_device_remove(phydev);
+ phy_device_free(phydev);
+ }
}
device_del(&bus->dev);
return err;
}
-EXPORT_SYMBOL(mdiobus_register);
+EXPORT_SYMBOL(__mdiobus_register);
void mdiobus_unregister(struct mii_bus *bus)
{
bus->state = MDIOBUS_UNREGISTERED;
for (i = 0; i < PHY_MAX_ADDR; i++) {
- if (bus->phy_map[i])
- device_unregister(&bus->phy_map[i]->dev);
- bus->phy_map[i] = NULL;
+ struct phy_device *phydev = bus->phy_map[i];
+ if (phydev) {
+ phy_device_remove(phydev);
+ phy_device_free(phydev);
+ }
}
device_del(&bus->dev);
}
}
EXPORT_SYMBOL(phy_device_register);
+/**
+ * phy_device_remove - Remove a previously registered phy device from the MDIO bus
+ * @phydev: phy_device structure to remove
+ *
+ * This doesn't free the phy_device itself, it merely reverses the effects
+ * of phy_device_register(). Use phy_device_free() to free the device
+ * after calling this function.
+ */
+void phy_device_remove(struct phy_device *phydev)
+{
+ struct mii_bus *bus = phydev->bus;
+ int addr = phydev->addr;
+
+ device_del(&phydev->dev);
+ bus->phy_map[addr] = NULL;
+}
+EXPORT_SYMBOL(phy_device_remove);
+
/**
* phy_find_first - finds the first PHY device on the bus
* @bus: the target MII bus
* generic driver is used. The phy_device is given a ptr to
* the attaching device, and given a callback for link status
* change. The phy_device is returned to the attaching driver.
+ * This function takes a reference on the phy device.
*/
int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
u32 flags, phy_interface_t interface)
{
+ struct mii_bus *bus = phydev->bus;
struct device *d = &phydev->dev;
- struct module *bus_module;
int err;
+ if (!try_module_get(bus->owner)) {
+ dev_err(&dev->dev, "failed to get the bus module\n");
+ return -EIO;
+ }
+
+ get_device(d);
+
/* Assume that if there is no driver, that it doesn't
* exist, and we should use the genphy driver.
*/
err = device_bind_driver(d);
if (err)
- return err;
+ goto error;
}
if (phydev->attached_dev) {
dev_err(&dev->dev, "PHY already attached\n");
- return -EBUSY;
- }
-
- /* Increment the bus module reference count */
- bus_module = phydev->bus->dev.driver ?
- phydev->bus->dev.driver->owner : NULL;
- if (!try_module_get(bus_module)) {
- dev_err(&dev->dev, "failed to get the bus module\n");
- return -EIO;
+ err = -EBUSY;
+ goto error;
}
phydev->attached_dev = dev;
phy_resume(phydev);
return err;
+
+error:
+ put_device(d);
+ module_put(bus->owner);
+ return err;
}
EXPORT_SYMBOL(phy_attach_direct);
/**
* phy_detach - detach a PHY device from its network device
* @phydev: target phy_device struct
+ *
+ * This detaches the phy device from its network device and the phy
+ * driver, and drops the reference count taken in phy_attach_direct().
*/
void phy_detach(struct phy_device *phydev)
{
+ struct mii_bus *bus;
int i;
- if (phydev->bus->dev.driver)
- module_put(phydev->bus->dev.driver->owner);
-
phydev->attached_dev->phydev = NULL;
phydev->attached_dev = NULL;
phy_suspend(phydev);
break;
}
}
+
+ /*
+ * The phydev might go away on the put_device() below, so avoid
+ * a use-after-free bug by reading the underlying bus first.
+ */
+ bus = phydev->bus;
+
+ put_device(&phydev->dev);
+ module_put(bus->owner);
}
EXPORT_SYMBOL(phy_detach);
#define PHY_ID_VSC8244 0x000fc6c0
#define PHY_ID_VSC8514 0x00070670
#define PHY_ID_VSC8574 0x000704a0
-#define PHY_ID_VSC8641 0x00070431
#define PHY_ID_VSC8662 0x00070660
#define PHY_ID_VSC8221 0x000fc550
#define PHY_ID_VSC8211 0x000fc4b0
.ack_interrupt = &vsc824x_ack_interrupt,
.config_intr = &vsc82xx_config_intr,
.driver = { .owner = THIS_MODULE,},
-}, {
- .phy_id = PHY_ID_VSC8641,
- .name = "Vitesse VSC8641",
- .phy_id_mask = 0x000ffff0,
- .features = PHY_GBIT_FEATURES,
- .flags = PHY_HAS_INTERRUPT,
- .config_init = &vsc824x_config_init,
- .config_aneg = &vsc82x4_config_aneg,
- .read_status = &genphy_read_status,
- .ack_interrupt = &vsc824x_ack_interrupt,
- .config_intr = &vsc82xx_config_intr,
- .driver = { .owner = THIS_MODULE,},
}, {
.phy_id = PHY_ID_VSC8662,
.name = "Vitesse VSC8662",
{ PHY_ID_VSC8244, 0x000fffc0 },
{ PHY_ID_VSC8514, 0x000ffff0 },
{ PHY_ID_VSC8574, 0x000ffff0 },
- { PHY_ID_VSC8641, 0x000ffff0 },
{ PHY_ID_VSC8662, 0x000ffff0 },
{ PHY_ID_VSC8221, 0x000ffff0 },
{ PHY_ID_VSC8211, 0x000ffff0 },
*/
dev_net_set(dev, net);
+ rtnl_lock();
mutex_lock(&pn->all_ppp_mutex);
if (unit < 0) {
ppp->file.index = unit;
sprintf(dev->name, "ppp%d", unit);
- ret = register_netdev(dev);
+ ret = register_netdevice(dev);
if (ret != 0) {
unit_put(&pn->units_idr, unit);
netdev_err(ppp->dev, "PPP: couldn't register device %s (%d)\n",
atomic_inc(&ppp_unit_count);
mutex_unlock(&pn->all_ppp_mutex);
+ rtnl_unlock();
*retp = 0;
return ppp;
http://ubuntuforums.org/showpost.php?p=10589647&postcount=17
+config USB_NET_CH9200
+ tristate "QingHeng CH9200 USB ethernet support"
+ depends on USB_USBNET
+ select MII
+ help
+ Choose this option if you have a USB ethernet adapter with a QinHeng
+ CH9200 chipset.
+
+ To compile this driver as a module, choose M here: the
+ module will be called ch9200.
+
endif # USB_NET_DRIVERS
obj-$(CONFIG_USB_VL600) += lg-vl600.o
obj-$(CONFIG_USB_NET_QMI_WWAN) += qmi_wwan.o
obj-$(CONFIG_USB_NET_CDC_MBIM) += cdc_mbim.o
-
+obj-$(CONFIG_USB_NET_CH9200) += ch9200.o
--- /dev/null
+/*
+ * USB 10M/100M ethernet adapter
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2. This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/stddef.h>
+#include <linux/init.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/mii.h>
+#include <linux/usb.h>
+#include <linux/crc32.h>
+#include <linux/usb/usbnet.h>
+#include <linux/slab.h>
+
+#define CH9200_VID 0x1A86
+#define CH9200_PID_E092 0xE092
+
+#define CTRL_TIMEOUT_MS 1000
+
+#define CONTROL_TIMEOUT_MS 1000
+
+#define REQUEST_READ 0x0E
+#define REQUEST_WRITE 0x0F
+
+/* Address space:
+ * 00-63 : MII
+ * 64-128: MAC
+ *
+ * Note: all accesses must be 16-bit
+ */
+
+#define MAC_REG_CTRL 64
+#define MAC_REG_STATUS 66
+#define MAC_REG_INTERRUPT_MASK 68
+#define MAC_REG_PHY_COMMAND 70
+#define MAC_REG_PHY_DATA 72
+#define MAC_REG_STATION_L 74
+#define MAC_REG_STATION_M 76
+#define MAC_REG_STATION_H 78
+#define MAC_REG_HASH_L 80
+#define MAC_REG_HASH_M1 82
+#define MAC_REG_HASH_M2 84
+#define MAC_REG_HASH_H 86
+#define MAC_REG_THRESHOLD 88
+#define MAC_REG_FIFO_DEPTH 90
+#define MAC_REG_PAUSE 92
+#define MAC_REG_FLOW_CONTROL 94
+
+/* Control register bits
+ *
+ * Note: bits 13 and 15 are reserved
+ */
+#define LOOPBACK (0x01 << 14)
+#define BASE100X (0x01 << 12)
+#define MBPS_10 (0x01 << 11)
+#define DUPLEX_MODE (0x01 << 10)
+#define PAUSE_FRAME (0x01 << 9)
+#define PROMISCUOUS (0x01 << 8)
+#define MULTICAST (0x01 << 7)
+#define BROADCAST (0x01 << 6)
+#define HASH (0x01 << 5)
+#define APPEND_PAD (0x01 << 4)
+#define APPEND_CRC (0x01 << 3)
+#define TRANSMITTER_ACTION (0x01 << 2)
+#define RECEIVER_ACTION (0x01 << 1)
+#define DMA_ACTION (0x01 << 0)
+
+/* Status register bits
+ *
+ * Note: bits 7-15 are reserved
+ */
+#define ALIGNMENT (0x01 << 6)
+#define FIFO_OVER_RUN (0x01 << 5)
+#define FIFO_UNDER_RUN (0x01 << 4)
+#define RX_ERROR (0x01 << 3)
+#define RX_COMPLETE (0x01 << 2)
+#define TX_ERROR (0x01 << 1)
+#define TX_COMPLETE (0x01 << 0)
+
+/* FIFO depth register bits
+ *
+ * Note: bits 6 and 14 are reserved
+ */
+
+#define ETH_TXBD (0x01 << 15)
+#define ETN_TX_FIFO_DEPTH (0x01 << 8)
+#define ETH_RXBD (0x01 << 7)
+#define ETH_RX_FIFO_DEPTH (0x01 << 0)
+
+static int control_read(struct usbnet *dev,
+ unsigned char request, unsigned short value,
+ unsigned short index, void *data, unsigned short size,
+ int timeout)
+{
+ unsigned char *buf = NULL;
+ unsigned char request_type;
+ int err = 0;
+
+ if (request == REQUEST_READ)
+ request_type = (USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_OTHER);
+ else
+ request_type = (USB_DIR_IN | USB_TYPE_VENDOR |
+ USB_RECIP_DEVICE);
+
+ netdev_dbg(dev->net, "Control_read() index=0x%02x size=%d\n",
+ index, size);
+
+ buf = kmalloc(size, GFP_KERNEL);
+ if (!buf) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ err = usb_control_msg(dev->udev,
+ usb_rcvctrlpipe(dev->udev, 0),
+ request, request_type, value, index, buf, size,
+ timeout);
+ if (err == size)
+ memcpy(data, buf, size);
+ else if (err >= 0)
+ err = -EINVAL;
+ kfree(buf);
+
+ return err;
+
+err_out:
+ return err;
+}
+
+static int control_write(struct usbnet *dev, unsigned char request,
+ unsigned short value, unsigned short index,
+ void *data, unsigned short size, int timeout)
+{
+ unsigned char *buf = NULL;
+ unsigned char request_type;
+ int err = 0;
+
+ if (request == REQUEST_WRITE)
+ request_type = (USB_DIR_OUT | USB_TYPE_VENDOR |
+ USB_RECIP_OTHER);
+ else
+ request_type = (USB_DIR_OUT | USB_TYPE_VENDOR |
+ USB_RECIP_DEVICE);
+
+ netdev_dbg(dev->net, "Control_write() index=0x%02x size=%d\n",
+ index, size);
+
+ if (data) {
+ buf = kmalloc(size, GFP_KERNEL);
+ if (!buf) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+ memcpy(buf, data, size);
+ }
+
+ err = usb_control_msg(dev->udev,
+ usb_sndctrlpipe(dev->udev, 0),
+ request, request_type, value, index, buf, size,
+ timeout);
+ if (err >= 0 && err < size)
+ err = -EINVAL;
+ kfree(buf);
+
+ return 0;
+
+err_out:
+ return err;
+}
+
+static int ch9200_mdio_read(struct net_device *netdev, int phy_id, int loc)
+{
+ struct usbnet *dev = netdev_priv(netdev);
+ unsigned char buff[2];
+
+ netdev_dbg(netdev, "ch9200_mdio_read phy_id:%02x loc:%02x\n",
+ phy_id, loc);
+
+ if (phy_id != 0)
+ return -ENODEV;
+
+ control_read(dev, REQUEST_READ, 0, loc * 2, buff, 0x02,
+ CONTROL_TIMEOUT_MS);
+
+ return (buff[0] | buff[1] << 8);
+}
+
+static void ch9200_mdio_write(struct net_device *netdev,
+ int phy_id, int loc, int val)
+{
+ struct usbnet *dev = netdev_priv(netdev);
+ unsigned char buff[2];
+
+ netdev_dbg(netdev, "ch9200_mdio_write() phy_id=%02x loc:%02x\n",
+ phy_id, loc);
+
+ if (phy_id != 0)
+ return;
+
+ buff[0] = (unsigned char)val;
+ buff[1] = (unsigned char)(val >> 8);
+
+ control_write(dev, REQUEST_WRITE, 0, loc * 2, buff, 0x02,
+ CONTROL_TIMEOUT_MS);
+}
+
+static int ch9200_link_reset(struct usbnet *dev)
+{
+ struct ethtool_cmd ecmd;
+
+ mii_check_media(&dev->mii, 1, 1);
+ mii_ethtool_gset(&dev->mii, &ecmd);
+
+ netdev_dbg(dev->net, "link_reset() speed:%d duplex:%d\n",
+ ecmd.speed, ecmd.duplex);
+
+ return 0;
+}
+
+static void ch9200_status(struct usbnet *dev, struct urb *urb)
+{
+ int link;
+ unsigned char *buf;
+
+ if (urb->actual_length < 16)
+ return;
+
+ buf = urb->transfer_buffer;
+ link = !!(buf[0] & 0x01);
+
+ if (link) {
+ netif_carrier_on(dev->net);
+ usbnet_defer_kevent(dev, EVENT_LINK_RESET);
+ } else {
+ netif_carrier_off(dev->net);
+ }
+}
+
+static struct sk_buff *ch9200_tx_fixup(struct usbnet *dev, struct sk_buff *skb,
+ gfp_t flags)
+{
+ int i = 0;
+ int len = 0;
+ int tx_overhead = 0;
+
+ tx_overhead = 0x40;
+
+ len = skb->len;
+ if (skb_headroom(skb) < tx_overhead) {
+ struct sk_buff *skb2;
+
+ skb2 = skb_copy_expand(skb, tx_overhead, 0, flags);
+ dev_kfree_skb_any(skb);
+ skb = skb2;
+ if (!skb)
+ return NULL;
+ }
+
+ __skb_push(skb, tx_overhead);
+ /* usbnet adds padding if length is a multiple of packet size
+ * if so, adjust length value in header
+ */
+ if ((skb->len % dev->maxpacket) == 0)
+ len++;
+
+ skb->data[0] = len;
+ skb->data[1] = len >> 8;
+ skb->data[2] = 0x00;
+ skb->data[3] = 0x80;
+
+ for (i = 4; i < 48; i++)
+ skb->data[i] = 0x00;
+
+ skb->data[48] = len;
+ skb->data[49] = len >> 8;
+ skb->data[50] = 0x00;
+ skb->data[51] = 0x80;
+
+ for (i = 52; i < 64; i++)
+ skb->data[i] = 0x00;
+
+ return skb;
+}
+
+static int ch9200_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
+{
+ int len = 0;
+ int rx_overhead = 0;
+
+ rx_overhead = 64;
+
+ if (unlikely(skb->len < rx_overhead)) {
+ dev_err(&dev->udev->dev, "unexpected tiny rx frame\n");
+ return 0;
+ }
+
+ len = (skb->data[skb->len - 16] | skb->data[skb->len - 15] << 8);
+ skb_trim(skb, len);
+
+ return 1;
+}
+
+static int get_mac_address(struct usbnet *dev, unsigned char *data)
+{
+ int err = 0;
+ unsigned char mac_addr[0x06];
+ int rd_mac_len = 0;
+
+ netdev_dbg(dev->net, "get_mac_address:\n\tusbnet VID:%0x PID:%0x\n",
+ dev->udev->descriptor.idVendor,
+ dev->udev->descriptor.idProduct);
+
+ memset(mac_addr, 0, sizeof(mac_addr));
+ rd_mac_len = control_read(dev, REQUEST_READ, 0,
+ MAC_REG_STATION_L, mac_addr, 0x02,
+ CONTROL_TIMEOUT_MS);
+ rd_mac_len += control_read(dev, REQUEST_READ, 0, MAC_REG_STATION_M,
+ mac_addr + 2, 0x02, CONTROL_TIMEOUT_MS);
+ rd_mac_len += control_read(dev, REQUEST_READ, 0, MAC_REG_STATION_H,
+ mac_addr + 4, 0x02, CONTROL_TIMEOUT_MS);
+ if (rd_mac_len != ETH_ALEN)
+ err = -EINVAL;
+
+ data[0] = mac_addr[5];
+ data[1] = mac_addr[4];
+ data[2] = mac_addr[3];
+ data[3] = mac_addr[2];
+ data[4] = mac_addr[1];
+ data[5] = mac_addr[0];
+
+ return err;
+}
+
+static int ch9200_bind(struct usbnet *dev, struct usb_interface *intf)
+{
+ int retval = 0;
+ unsigned char data[2];
+
+ retval = usbnet_get_endpoints(dev, intf);
+ if (retval)
+ return retval;
+
+ dev->mii.dev = dev->net;
+ dev->mii.mdio_read = ch9200_mdio_read;
+ dev->mii.mdio_write = ch9200_mdio_write;
+ dev->mii.reg_num_mask = 0x1f;
+
+ dev->mii.phy_id_mask = 0x1f;
+
+ dev->hard_mtu = dev->net->mtu + dev->net->hard_header_len;
+ dev->rx_urb_size = 24 * 64 + 16;
+ mii_nway_restart(&dev->mii);
+
+ data[0] = 0x01;
+ data[1] = 0x0F;
+ retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_THRESHOLD, data,
+ 0x02, CONTROL_TIMEOUT_MS);
+
+ data[0] = 0xA0;
+ data[1] = 0x90;
+ retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_FIFO_DEPTH, data,
+ 0x02, CONTROL_TIMEOUT_MS);
+
+ data[0] = 0x30;
+ data[1] = 0x00;
+ retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_PAUSE, data,
+ 0x02, CONTROL_TIMEOUT_MS);
+
+ data[0] = 0x17;
+ data[1] = 0xD8;
+ retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_FLOW_CONTROL,
+ data, 0x02, CONTROL_TIMEOUT_MS);
+
+ /* Undocumented register */
+ data[0] = 0x01;
+ data[1] = 0x00;
+ retval = control_write(dev, REQUEST_WRITE, 0, 254, data, 0x02,
+ CONTROL_TIMEOUT_MS);
+
+ data[0] = 0x5F;
+ data[1] = 0x0D;
+ retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_CTRL, data, 0x02,
+ CONTROL_TIMEOUT_MS);
+
+ retval = get_mac_address(dev, dev->net->dev_addr);
+
+ return retval;
+}
+
+static const struct driver_info ch9200_info = {
+ .description = "CH9200 USB to Network Adaptor",
+ .flags = FLAG_ETHER,
+ .bind = ch9200_bind,
+ .rx_fixup = ch9200_rx_fixup,
+ .tx_fixup = ch9200_tx_fixup,
+ .status = ch9200_status,
+ .link_reset = ch9200_link_reset,
+ .reset = ch9200_link_reset,
+};
+
+static const struct usb_device_id ch9200_products[] = {
+ {
+ USB_DEVICE(0x1A86, 0xE092),
+ .driver_info = (unsigned long)&ch9200_info,
+ },
+ {},
+};
+
+MODULE_DEVICE_TABLE(usb, ch9200_products);
+
+static struct usb_driver ch9200_driver = {
+ .name = "ch9200",
+ .id_table = ch9200_products,
+ .probe = usbnet_probe,
+ .disconnect = usbnet_disconnect,
+ .suspend = usbnet_suspend,
+ .resume = usbnet_resume,
+};
+
+module_usb_driver(ch9200_driver);
+
+MODULE_DESCRIPTION("QinHeng CH9200 USB Network device");
+MODULE_LICENSE("GPL");
.flowi4_oif = vrf_dev->ifindex,
.flowi4_iif = LOOPBACK_IFINDEX,
.flowi4_tos = RT_TOS(ip4h->tos),
- .flowi4_flags = FLOWI_FLAG_ANYSRC | FLOWI_FLAG_VRFSRC,
+ .flowi4_flags = FLOWI_FLAG_ANYSRC | FLOWI_FLAG_VRFSRC |
+ FLOWI_FLAG_SKIP_NH_OIF,
.daddr = ip4h->daddr,
};
eth_hw_addr_random(dev);
ether_setup(dev);
- if (vxlan->default_dst.remote_ip.sa.sa_family == AF_INET6)
- dev->needed_headroom = ETH_HLEN + VXLAN6_HEADROOM;
- else
- dev->needed_headroom = ETH_HLEN + VXLAN_HEADROOM;
dev->netdev_ops = &vxlan_netdev_ops;
dev->destructor = free_netdev;
dst->remote_ip.sa.sa_family = AF_INET;
if (dst->remote_ip.sa.sa_family == AF_INET6 ||
- vxlan->cfg.saddr.sa.sa_family == AF_INET6)
+ vxlan->cfg.saddr.sa.sa_family == AF_INET6) {
+ if (!IS_ENABLED(CONFIG_IPV6))
+ return -EPFNOSUPPORT;
use_ipv6 = true;
+ }
if (conf->remote_ifindex) {
struct net_device *lowerdev
dev->needed_headroom = lowerdev->hard_header_len +
(use_ipv6 ? VXLAN6_HEADROOM : VXLAN_HEADROOM);
- } else if (use_ipv6)
+ } else if (use_ipv6) {
vxlan->flags |= VXLAN_F_IPV6;
+ dev->needed_headroom = ETH_HLEN + VXLAN6_HEADROOM;
+ } else {
+ dev->needed_headroom = ETH_HLEN + VXLAN_HEADROOM;
+ }
memcpy(&vxlan->cfg, conf, sizeof(*conf));
if (!vxlan->cfg.dst_port)
struct nd_btt *nd_btt = to_nd_btt(dev);
ssize_t rc;
- nvdimm_bus_lock(dev);
device_lock(dev);
+ nvdimm_bus_lock(dev);
rc = nd_namespace_store(dev, &nd_btt->ndns, buf, len);
dev_dbg(dev, "%s: result: %zd wrote: %s%s", __func__,
rc, buf, buf[len - 1] == '\n' ? "" : "\n");
- device_unlock(dev);
nvdimm_bus_unlock(dev);
+ device_unlock(dev);
return rc;
}
struct nd_pfn *nd_pfn = to_nd_pfn(dev);
ssize_t rc;
- nvdimm_bus_lock(dev);
device_lock(dev);
+ nvdimm_bus_lock(dev);
rc = nd_namespace_store(dev, &nd_pfn->ndns, buf, len);
dev_dbg(dev, "%s: result: %zd wrote: %s%s", __func__,
rc, buf, buf[len - 1] == '\n' ? "" : "\n");
- device_unlock(dev);
nvdimm_bus_unlock(dev);
+ device_unlock(dev);
return rc;
}
struct pmem_device *pmem = bdev->bd_disk->private_data;
pmem_do_bvec(pmem, page, PAGE_CACHE_SIZE, 0, rw, sector);
+ if (rw & WRITE)
+ wmb_pmem();
page_endio(page, rw & WRITE, 0);
return 0;
int rc;
/* Stop the user from reading */
- if (pos > nvmem->size)
+ if (pos >= nvmem->size)
return 0;
if (pos + count > nvmem->size)
int rc;
/* Stop the user from writing */
- if (pos > nvmem->size)
+ if (pos >= nvmem->size)
return 0;
if (pos + count > nvmem->size)
return rc;
/* shift bits in-place */
- if (cell->bit_offset || cell->bit_offset)
+ if (cell->bit_offset || cell->nbits)
nvmem_shift_read_buffer_in_place(cell, buf);
*len = cell->bytes;
rc = regmap_raw_write(nvmem->regmap, cell->offset, buf, cell->bytes);
/* free the tmp buffer */
- if (cell->bit_offset)
+ if (cell->bit_offset || cell->nbits)
kfree(buf);
if (IS_ERR_VALUE(rc))
struct nvmem_device *nvmem;
struct regmap *regmap;
struct sunxi_sid *sid;
- int i, size;
+ int ret, i, size;
char *randomness;
sid = devm_kzalloc(dev, sizeof(*sid), GFP_KERNEL);
return PTR_ERR(nvmem);
randomness = kzalloc(sizeof(u8) * size, GFP_KERNEL);
+ if (!randomness) {
+ ret = -EINVAL;
+ goto err_unreg_nvmem;
+ }
+
for (i = 0; i < size; i++)
randomness[i] = sunxi_sid_read_byte(sid, i);
platform_set_drvdata(pdev, nvmem);
return 0;
+
+err_unreg_nvmem:
+ nvmem_unregister(nvmem);
+ return ret;
}
static int sunxi_sid_remove(struct platform_device *pdev)
* of_phy_find_device - Give a PHY node, find the phy_device
* @phy_np: Pointer to the phy's device tree node
*
- * Returns a pointer to the phy_device.
+ * If successful, returns a pointer to the phy_device with the embedded
+ * struct device refcount incremented by one, or NULL on failure.
*/
struct phy_device *of_phy_find_device(struct device_node *phy_np)
{
* @hndlr: Link state callback for the network device
* @iface: PHY data interface type
*
- * Returns a pointer to the phy_device if successful. NULL otherwise
+ * If successful, returns a pointer to the phy_device with the embedded
+ * struct device refcount incremented by one, or NULL on failure. The
+ * refcount must be dropped by calling phy_disconnect() or phy_detach().
*/
struct phy_device *of_phy_connect(struct net_device *dev,
struct device_node *phy_np,
phy_interface_t iface)
{
struct phy_device *phy = of_phy_find_device(phy_np);
+ int ret;
if (!phy)
return NULL;
phy->dev_flags = flags;
- return phy_connect_direct(dev, phy, hndlr, iface) ? NULL : phy;
+ ret = phy_connect_direct(dev, phy, hndlr, iface);
+
+ /* refcount is held by phy_connect_direct() on success */
+ put_device(&phy->dev);
+
+ return ret ? NULL : phy;
}
EXPORT_SYMBOL(of_phy_connect);
* @phy_np: Node pointer for the PHY
* @flags: flags to pass to the PHY
* @iface: PHY data interface type
+ *
+ * If successful, returns a pointer to the phy_device with the embedded
+ * struct device refcount incremented by one, or NULL on failure. The
+ * refcount must be dropped by calling phy_disconnect() or phy_detach().
*/
struct phy_device *of_phy_attach(struct net_device *dev,
struct device_node *phy_np, u32 flags,
phy_interface_t iface)
{
struct phy_device *phy = of_phy_find_device(phy_np);
+ int ret;
if (!phy)
return NULL;
- return phy_attach_direct(dev, phy, flags, iface) ? NULL : phy;
+ ret = phy_attach_direct(dev, phy, flags, iface);
+
+ /* refcount is held by phy_attach_direct() on success */
+ put_device(&phy->dev);
+
+ return ret ? NULL : phy;
}
EXPORT_SYMBOL(of_phy_attach);
*/
rc = pci_read_config_byte(pdev, PCI_INTERRUPT_PIN, &pin);
if (rc != 0)
- return rc;
- /* No pin, exit */
+ goto err;
+ /* No pin, exit with no error message. */
if (pin == 0)
return -ENODEV;
ppnode = pci_bus_to_OF_node(pdev->bus);
/* No node for host bridge ? give up */
- if (ppnode == NULL)
- return -EINVAL;
+ if (ppnode == NULL) {
+ rc = -EINVAL;
+ goto err;
+ }
} else {
/* We found a P2P bridge, check if it has a node */
ppnode = pci_device_to_OF_node(ppdev);
out_irq->args[0] = pin;
laddr[0] = cpu_to_be32((pdev->bus->number << 16) | (pdev->devfn << 8));
laddr[1] = laddr[2] = cpu_to_be32(0);
- return of_irq_parse_raw(laddr, out_irq);
+ rc = of_irq_parse_raw(laddr, out_irq);
+ if (rc)
+ goto err;
+ return 0;
+err:
+ dev_err(&pdev->dev, "of_irq_parse_pci() failed with rc=%d\n", rc);
+ return rc;
}
EXPORT_SYMBOL_GPL(of_irq_parse_pci);
int ret;
ret = of_irq_parse_pci(dev, &oirq);
- if (ret) {
- dev_err(&dev->dev, "of_irq_parse_pci() failed with rc=%d\n", ret);
+ if (ret)
return 0; /* Proper return code 0 == NO_IRQ */
- }
return irq_create_of_mapping(&oirq);
}
} else if (bus->parent) {
int i;
+ pci_read_bridge_bases(bus);
+
+
for(i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
if((bus->self->resource[i].flags &
(IORESOURCE_IO | IORESOURCE_MEM)) == 0)
if (bus->parent) {
int i;
/* PCI-PCI Bridge */
+ pci_read_bridge_bases(bus);
for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++)
pci_claim_bridge_resource(bus->self, i);
} else {
static ssize_t pci_vpd_f0_read(struct pci_dev *dev, loff_t pos, size_t count,
void *arg)
{
- struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn));
+ struct pci_dev *tdev = pci_get_slot(dev->bus,
+ PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
ssize_t ret;
if (!tdev)
static ssize_t pci_vpd_f0_write(struct pci_dev *dev, loff_t pos, size_t count,
const void *arg)
{
- struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn));
+ struct pci_dev *tdev = pci_get_slot(dev->bus,
+ PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
ssize_t ret;
if (!tdev)
.release = pci_vpd_pci22_release,
};
-static int pci_vpd_f0_dev_check(struct pci_dev *dev)
-{
- struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn));
- int ret = 0;
-
- if (!tdev)
- return -ENODEV;
- if (!tdev->vpd || !tdev->multifunction ||
- dev->class != tdev->class || dev->vendor != tdev->vendor ||
- dev->device != tdev->device)
- ret = -ENODEV;
-
- pci_dev_put(tdev);
- return ret;
-}
-
int pci_vpd_pci22_init(struct pci_dev *dev)
{
struct pci_vpd_pci22 *vpd;
cap = pci_find_capability(dev, PCI_CAP_ID_VPD);
if (!cap)
return -ENODEV;
- if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) {
- int ret = pci_vpd_f0_dev_check(dev);
- if (ret)
- return ret;
- }
vpd = kzalloc(sizeof(*vpd), GFP_ATOMIC);
if (!vpd)
return -ENOMEM;
res->start = start;
res->end = end;
+ res->flags &= ~IORESOURCE_UNSET;
+ orig_res.flags &= ~IORESOURCE_UNSET;
dev_printk(KERN_DEBUG, &dev->dev, "%pR clipped to %pR\n",
&orig_res, res);
return -EINVAL;
}
-static void ks_pcie_msi_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void ks_pcie_msi_irq_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc);
* Traverse through pending legacy interrupts and invoke handler for each. Also
* takes care of interrupt controller level mask/ack operation.
*/
-static void ks_pcie_legacy_irq_handler(unsigned int __irq,
- struct irq_desc *desc)
+static void ks_pcie_legacy_irq_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc);
static struct of_device_id rcar_pci_of_match[] = {
{ .compatible = "renesas,pci-r8a7790", },
{ .compatible = "renesas,pci-r8a7791", },
+ { .compatible = "renesas,pci-r8a7794", },
{ },
};
return 0;
}
-static void xgene_msi_isr(unsigned int irq, struct irq_desc *desc)
+static void xgene_msi_isr(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct xgene_msi_group *msi_groups;
* Unbound PCI devices are always put in D0, regardless of
* runtime PM status. During probe, the device is set to
* active and the usage count is incremented. If the driver
- * supports runtime PM, it should call pm_runtime_put_noidle()
- * in its probe routine and pm_runtime_get_noresume() in its
- * remove routine.
+ * supports runtime PM, it should call pm_runtime_put_noidle(),
+ * or any other runtime PM helper function decrementing the usage
+ * count, in its probe routine and pm_runtime_get_noresume() in
+ * its remove routine.
*/
pm_runtime_get_sync(dev);
pci_dev->driver = pci_drv;
static void pci_set_bus_msi_domain(struct pci_bus *bus)
{
struct irq_domain *d;
+ struct pci_bus *b;
/*
- * Either bus is the root, and we must obtain it from the
- * firmware, or we inherit it from the bridge device.
+ * The bus can be a root bus, a subordinate bus, or a virtual bus
+ * created by an SR-IOV device. Walk up to the first bridge device
+ * found or derive the domain from the host bridge.
*/
- if (pci_is_root_bus(bus))
- d = pci_host_bridge_msi_domain(bus);
- else
- d = dev_get_msi_domain(&bus->self->dev);
+ for (b = bus, d = NULL; !d && !pci_is_root_bus(b); b = b->parent) {
+ if (b->self)
+ d = dev_get_msi_domain(&b->self->dev);
+ }
+
+ if (!d)
+ d = pci_host_bridge_msi_domain(b);
dev_set_msi_domain(&bus->dev, d);
}
child->bridge_ctl = bctl;
}
- /* Read and initialize bridge resources */
- pci_read_bridge_bases(child);
-
cmax = pci_scan_child_bus(child);
if (cmax > subordinate)
dev_warn(&dev->dev, "bridge has subordinate %02x but max busn %02x\n",
if (!is_cardbus) {
child->bridge_ctl = bctl;
-
- /* Read and initialize bridge resources */
- pci_read_bridge_bases(child);
max = pci_scan_child_bus(child);
} else {
/*
DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_NETMOS, PCI_ANY_ID,
PCI_CLASS_COMMUNICATION_SERIAL, 8, quirk_netmos);
+/*
+ * Quirk non-zero PCI functions to route VPD access through function 0 for
+ * devices that share VPD resources between functions. The functions are
+ * expected to be identical devices.
+ */
static void quirk_f0_vpd_link(struct pci_dev *dev)
{
- if (!dev->multifunction || !PCI_FUNC(dev->devfn))
+ struct pci_dev *f0;
+
+ if (!PCI_FUNC(dev->devfn))
return;
- dev->dev_flags |= PCI_DEV_FLAGS_VPD_REF_F0;
+
+ f0 = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
+ if (!f0)
+ return;
+
+ if (f0->vpd && dev->class == f0->class &&
+ dev->vendor == f0->vendor && dev->device == f0->device)
+ dev->dev_flags |= PCI_DEV_FLAGS_VPD_REF_F0;
+
+ pci_dev_put(f0);
}
DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
PCI_CLASS_NETWORK_ETHERNET, 8, quirk_f0_vpd_link);
{ .compatible = "marvell,berlin2q-sata-phy" },
{ },
};
+MODULE_DEVICE_TABLE(of, phy_berlin_sata_of_match);
static struct platform_driver phy_berlin_sata_driver = {
.probe = phy_berlin_sata_probe,
out:
return ret;
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_enable_ref_clk);
static
int ufs_qcom_phy_disable_vreg(struct phy *phy,
phy->is_ref_clk_enabled = false;
}
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_ref_clk);
#define UFS_REF_CLK_EN (1 << 5)
{
ufs_qcom_phy_dev_ref_clk_ctrl(generic_phy, true);
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_enable_dev_ref_clk);
void ufs_qcom_phy_disable_dev_ref_clk(struct phy *generic_phy)
{
ufs_qcom_phy_dev_ref_clk_ctrl(generic_phy, false);
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_dev_ref_clk);
/* Turn ON M-PHY RMMI interface clocks */
int ufs_qcom_phy_enable_iface_clk(struct phy *generic_phy)
out:
return ret;
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_enable_iface_clk);
/* Turn OFF M-PHY RMMI interface clocks */
void ufs_qcom_phy_disable_iface_clk(struct phy *generic_phy)
phy->is_iface_clk_enabled = false;
}
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_iface_clk);
int ufs_qcom_phy_start_serdes(struct phy *generic_phy)
{
return ret;
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_start_serdes);
int ufs_qcom_phy_set_tx_lane_enable(struct phy *generic_phy, u32 tx_lanes)
{
return ret;
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_set_tx_lane_enable);
void ufs_qcom_phy_save_controller_version(struct phy *generic_phy,
u8 major, u16 minor, u16 step)
ufs_qcom_phy->host_ctrl_rev_minor = minor;
ufs_qcom_phy->host_ctrl_rev_step = step;
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_save_controller_version);
int ufs_qcom_phy_calibrate_phy(struct phy *generic_phy, bool is_rate_B)
{
return ret;
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_calibrate_phy);
int ufs_qcom_phy_remove(struct phy *generic_phy,
struct ufs_qcom_phy *ufs_qcom_phy)
return ufs_qcom_phy->phy_spec_ops->
is_physical_coding_sublayer_ready(ufs_qcom_phy);
}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_is_pcs_ready);
int ufs_qcom_phy_power_on(struct phy *generic_phy)
{
struct device_node *child;
struct regmap *grf;
unsigned int reg_offset;
+ int err;
grf = syscon_regmap_lookup_by_phandle(dev->of_node, "rockchip,grf");
if (IS_ERR(grf)) {
return PTR_ERR(rk_phy->phy);
}
phy_set_drvdata(rk_phy->phy, rk_phy);
+
+ /* only power up usb phy when it use, so disable it when init*/
+ err = rockchip_usb_phy_power(rk_phy, 1);
+ if (err)
+ return err;
}
phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
return !!(readl(chip->base + offset) & BIT(shift));
}
-static void cygnus_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void cygnus_gpio_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct cygnus_gpio *chip = to_cygnus_gpio(gc);
struct pinctrl_gpio_range *range = NULL;
struct gpio_chip *chip = gpio_to_chip(gpio);
+ if (WARN(!chip, "no gpio_chip for gpio%i?", gpio))
+ return false;
+
mutex_lock(&pinctrldev_list_mutex);
/* Loop over the pin controllers */
}
}
-static void byt_gpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void byt_gpio_irq_handler(struct irq_desc *desc)
{
struct irq_data *data = irq_desc_get_irq_data(desc);
struct byt_gpio *vg = to_byt_gpio(irq_desc_get_handler_data(desc));
.flags = IRQCHIP_SKIP_SET_WAKE,
};
-static void chv_gpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void chv_gpio_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
}
}
-static void intel_gpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void intel_gpio_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct intel_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
}
}
-static void mtk_eint_irq_handler(unsigned irq, struct irq_desc *desc)
+static void mtk_eint_irq_handler(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct mtk_pinctrl *pctl = irq_desc_get_handler_data(desc);
chained_irq_exit(host_chip, desc);
}
-static void nmk_gpio_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void nmk_gpio_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *chip = irq_desc_get_handler_data(desc);
struct nmk_gpio_chip *nmk_chip = container_of(chip, struct nmk_gpio_chip, chip);
__nmk_gpio_irq_handler(desc, status);
}
-static void nmk_gpio_latent_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void nmk_gpio_latent_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *chip = irq_desc_get_handler_data(desc);
struct nmk_gpio_chip *nmk_chip = container_of(chip, struct nmk_gpio_chip, chip);
static inline void preflow_handler(struct irq_desc *desc) { }
#endif
-static void adi_gpio_handle_pint_irq(unsigned int inta_irq,
- struct irq_desc *desc)
+static void adi_gpio_handle_pint_irq(struct irq_desc *desc)
{
u32 request;
u32 level_mask, hwirq;
.irq_set_type = amd_gpio_irq_set_type,
};
-static void amd_gpio_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void amd_gpio_irq_handler(struct irq_desc *desc)
{
- unsigned int irq = irq_desc_get_irq(desc);
u32 i;
u32 off;
u32 reg;
u32 pin_reg;
u64 reg64;
int handled = 0;
+ unsigned int irq;
unsigned long flags;
struct irq_chip *chip = irq_desc_get_chip(desc);
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
}
if (handled == 0)
- handle_bad_irq(irq, desc);
+ handle_bad_irq(desc);
spin_lock_irqsave(&gpio_dev->lock, flags);
reg = readl(gpio_dev->base + WAKE_INT_MASTER_REG);
.irq_set_wake = gpio_irq_set_wake,
};
-static void gpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void gpio_irq_handler(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct gpio_chip *gpio_chip = irq_desc_get_handler_data(desc);
.irq_set_type = u300_gpio_irq_type,
};
-static void u300_gpio_irq_handler(unsigned __irq, struct irq_desc *desc)
+static void u300_gpio_irq_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct irq_chip *parent_chip = irq_desc_get_chip(desc);
pmap->dev = &pdev->dev;
pmap->pctl = pinctrl_register(pctl_desc, &pdev->dev, pmap);
- if (!pmap->pctl) {
+ if (IS_ERR(pmap->pctl)) {
dev_err(&pdev->dev, "pinctrl driver registration failed\n");
- return -EINVAL;
+ return PTR_ERR(pmap->pctl);
}
ret = dc_gpiochip_add(pmap, pdev->dev.of_node);
}
if (type & IRQ_TYPE_LEVEL_MASK)
- __irq_set_handler_locked(data->irq, handle_level_irq);
+ irq_set_handler_locked(data, handle_level_irq);
else
- __irq_set_handler_locked(data->irq, handle_edge_irq);
+ irq_set_handler_locked(data, handle_edge_irq);
return 0;
}
-static void pistachio_gpio_irq_handler(unsigned int __irq,
- struct irq_desc *desc)
+static void pistachio_gpio_irq_handler(struct irq_desc *desc)
{
- unsigned int irq = irq_desc_get_irq(desc);
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct pistachio_gpio_bank *bank = gc_to_bank(gc);
- struct irq_chip *chip = irq_get_chip(irq);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
unsigned long pending;
unsigned int pin;
* Interrupt handling
*/
-static void rockchip_irq_demux(unsigned int __irq, struct irq_desc *desc)
+static void rockchip_irq_demux(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct rockchip_pin_bank *bank = irq_desc_get_handler_data(desc);
* Use this if you have a separate interrupt for each
* pinctrl-single instance.
*/
-static void pcs_irq_chain_handler(unsigned int irq, struct irq_desc *desc)
+static void pcs_irq_chain_handler(struct irq_desc *desc)
{
struct pcs_soc_data *pcs_soc = irq_desc_get_handler_data(desc);
struct irq_chip *chip;
}
}
-static void st_gpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void st_gpio_irq_handler(struct irq_desc *desc)
{
/* interrupt dedicated per bank */
struct irq_chip *chip = irq_desc_get_chip(desc);
chained_irq_exit(chip, desc);
}
-static void st_gpio_irqmux_handler(unsigned irq, struct irq_desc *desc)
+static void st_gpio_irqmux_handler(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct st_pinctrl *info = irq_desc_get_handler_data(desc);
/* See if this pctldev has this function */
while (selector < nfuncs) {
- const char *fname = ops->get_function_name(pctldev,
- selector);
+ const char *fname = ops->get_function_name(pctldev, selector);
if (!strcmp(function, fname))
return selector;
.irq_set_wake = msm_gpio_irq_set_wake,
};
-static void msm_gpio_irq_handler(unsigned int __irq, struct irq_desc *desc)
+static void msm_gpio_irq_handler(struct irq_desc *desc)
{
- unsigned int irq = irq_desc_get_irq(desc);
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
const struct msm_pingroup *g;
struct msm_pinctrl *pctrl = to_msm_pinctrl(gc);
/* No interrupts were flagged */
if (handled == 0)
- handle_bad_irq(irq, desc);
+ handle_bad_irq(desc);
chained_irq_exit(chip, desc);
}
#endif
pctrl->pctrl = pinctrl_register(&pctrl->desc, &pdev->dev, pctrl);
- if (!pctrl->pctrl) {
+ if (IS_ERR(pctrl->pctrl)) {
dev_err(&pdev->dev, "couldn't register pm8xxx gpio driver\n");
- return -ENODEV;
+ return PTR_ERR(pctrl->pctrl);
}
pctrl->chip = pm8xxx_gpio_template;
#endif
pctrl->pctrl = pinctrl_register(&pctrl->desc, &pdev->dev, pctrl);
- if (!pctrl->pctrl) {
+ if (IS_ERR(pctrl->pctrl)) {
dev_err(&pdev->dev, "couldn't register pm8xxx mpp driver\n");
- return -ENODEV;
+ return PTR_ERR(pctrl->pctrl);
}
pctrl->chip = pm8xxx_mpp_template;
};
/* interrupt handler for wakeup interrupts 0..15 */
-static void exynos_irq_eint0_15(unsigned int irq, struct irq_desc *desc)
+static void exynos_irq_eint0_15(struct irq_desc *desc)
{
struct exynos_weint_data *eintd = irq_desc_get_handler_data(desc);
struct samsung_pin_bank *bank = eintd->bank;
}
/* interrupt handler for wakeup interrupt 16 */
-static void exynos_irq_demux_eint16_31(unsigned int irq, struct irq_desc *desc)
+static void exynos_irq_demux_eint16_31(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct exynos_muxed_weint_data *eintd = irq_desc_get_handler_data(desc);
.irq_set_type = s3c24xx_eint_type,
};
-static void s3c2410_demux_eint0_3(unsigned int irq, struct irq_desc *desc)
+static void s3c2410_demux_eint0_3(struct irq_desc *desc)
{
struct irq_data *data = irq_desc_get_irq_data(desc);
struct s3c24xx_eint_data *eint_data = irq_desc_get_handler_data(desc);
.irq_set_type = s3c24xx_eint_type,
};
-static void s3c2412_demux_eint0_3(unsigned int irq, struct irq_desc *desc)
+static void s3c2412_demux_eint0_3(struct irq_desc *desc)
{
struct s3c24xx_eint_data *eint_data = irq_desc_get_handler_data(desc);
struct irq_data *data = irq_desc_get_irq_data(desc);
u32 offset, u32 range)
{
struct s3c24xx_eint_data *data = irq_desc_get_handler_data(desc);
- struct irq_chip *chip = irq_desc_get_irq_chip(desc);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
struct samsung_pinctrl_drv_data *d = data->drvdata;
unsigned int pend, mask;
chained_irq_exit(chip, desc);
}
-static void s3c24xx_demux_eint4_7(unsigned int irq, struct irq_desc *desc)
+static void s3c24xx_demux_eint4_7(struct irq_desc *desc)
{
s3c24xx_demux_eint(desc, 0, 0xf0);
}
-static void s3c24xx_demux_eint8_23(unsigned int irq, struct irq_desc *desc)
+static void s3c24xx_demux_eint8_23(struct irq_desc *desc)
{
s3c24xx_demux_eint(desc, 8, 0xffff00);
}
.xlate = irq_domain_xlate_twocell,
};
-static void s3c64xx_eint_gpio_irq(unsigned int irq, struct irq_desc *desc)
+static void s3c64xx_eint_gpio_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct s3c64xx_eint_gpio_data *data = irq_desc_get_handler_data(desc);
chained_irq_exit(chip, desc);
}
-static void s3c64xx_demux_eint0_3(unsigned int irq, struct irq_desc *desc)
+static void s3c64xx_demux_eint0_3(struct irq_desc *desc)
{
s3c64xx_irq_demux_eint(desc, 0xf);
}
-static void s3c64xx_demux_eint4_11(unsigned int irq, struct irq_desc *desc)
+static void s3c64xx_demux_eint4_11(struct irq_desc *desc)
{
s3c64xx_irq_demux_eint(desc, 0xff0);
}
-static void s3c64xx_demux_eint12_19(unsigned int irq, struct irq_desc *desc)
+static void s3c64xx_demux_eint12_19(struct irq_desc *desc)
{
s3c64xx_irq_demux_eint(desc, 0xff000);
}
-static void s3c64xx_demux_eint20_27(unsigned int irq, struct irq_desc *desc)
+static void s3c64xx_demux_eint20_27(struct irq_desc *desc)
{
s3c64xx_irq_demux_eint(desc, 0xff00000);
}
.irq_set_type = atlas7_gpio_irq_type,
};
-static void atlas7_gpio_handle_irq(unsigned int __irq, struct irq_desc *desc)
+static void atlas7_gpio_handle_irq(struct irq_desc *desc)
{
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(gc);
if (!status) {
pr_warn("%s: gpio [%s] status %#x no interrupt is flaged\n",
__func__, gc->label, status);
- handle_bad_irq(irq, desc);
+ handle_bad_irq(desc);
return;
}
.irq_set_type = sirfsoc_gpio_irq_type,
};
-static void sirfsoc_gpio_handle_irq(unsigned int __irq, struct irq_desc *desc)
+static void sirfsoc_gpio_handle_irq(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
printk(KERN_WARNING
"%s: gpio id %d status %#x no interrupt is flagged\n",
__func__, bank->id, status);
- handle_bad_irq(irq, desc);
+ handle_bad_irq(desc);
return;
}
.irq_set_type = plgpio_irq_set_type,
};
-static void plgpio_irq_handler(unsigned irq, struct irq_desc *desc)
+static void plgpio_irq_handler(struct irq_desc *desc)
{
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct plgpio *plgpio = container_of(gc, struct plgpio, chip);
spin_lock_irqsave(&pctl->lock, flags);
if (type & IRQ_TYPE_LEVEL_MASK)
- __irq_set_chip_handler_name_locked(d->irq,
- &sunxi_pinctrl_level_irq_chip,
- handle_fasteoi_irq, NULL);
+ irq_set_chip_handler_name_locked(d, &sunxi_pinctrl_level_irq_chip,
+ handle_fasteoi_irq, NULL);
else
- __irq_set_chip_handler_name_locked(d->irq,
- &sunxi_pinctrl_edge_irq_chip,
- handle_edge_irq, NULL);
+ irq_set_chip_handler_name_locked(d, &sunxi_pinctrl_edge_irq_chip,
+ handle_edge_irq, NULL);
regval = readl(pctl->membase + reg);
regval &= ~(IRQ_CFG_IRQ_MASK << index);
.xlate = sunxi_pinctrl_irq_of_xlate,
};
-static void sunxi_pinctrl_irq_handler(unsigned __irq, struct irq_desc *desc)
+static void sunxi_pinctrl_irq_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
return VGA_SWITCHEROO_DIS;
}
-static struct vga_switcheroo_handler gmux_handler = {
+static const struct vga_switcheroo_handler gmux_handler = {
.switchto = gmux_switchto,
.power_state = gmux_set_power_state,
.get_client_id = gmux_get_client_id,
},
.driver_data = &quirk_asus_wapf4,
},
+ {
+ .callback = dmi_matched,
+ .ident = "ASUSTeK COMPUTER INC. X456UA",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "X456UA"),
+ },
+ .driver_data = &quirk_asus_wapf4,
+ },
+ {
+ .callback = dmi_matched,
+ .ident = "ASUSTeK COMPUTER INC. X456UF",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "X456UF"),
+ },
+ .driver_data = &quirk_asus_wapf4,
+ },
{
.callback = dmi_matched,
.ident = "ASUSTeK COMPUTER INC. X501U",
#define HPWMI_HARDWARE_QUERY 0x4
#define HPWMI_WIRELESS_QUERY 0x5
#define HPWMI_BIOS_QUERY 0x9
+#define HPWMI_FEATURE_QUERY 0xb
#define HPWMI_HOTKEY_QUERY 0xc
-#define HPWMI_FEATURE_QUERY 0xd
+#define HPWMI_FEATURE2_QUERY 0xd
#define HPWMI_WIRELESS2_QUERY 0x1b
#define HPWMI_POSTCODEERROR_QUERY 0x2a
return (state & 0x4) ? 1 : 0;
}
-static int __init hp_wmi_bios_2009_later(void)
+static int __init hp_wmi_bios_2008_later(void)
{
int state = 0;
int ret = hp_wmi_perform_query(HPWMI_FEATURE_QUERY, 0, &state,
sizeof(state), sizeof(state));
- if (ret)
- return ret;
+ if (!ret)
+ return 1;
- return (state & 0x10) ? 1 : 0;
+ return (ret == HPWMI_RET_UNKNOWN_CMDTYPE) ? 0 : -ENXIO;
}
-static int hp_wmi_enable_hotkeys(void)
+static int __init hp_wmi_bios_2009_later(void)
{
- int ret;
- int query = 0x6e;
+ int state = 0;
+ int ret = hp_wmi_perform_query(HPWMI_FEATURE2_QUERY, 0, &state,
+ sizeof(state), sizeof(state));
+ if (!ret)
+ return 1;
- ret = hp_wmi_perform_query(HPWMI_BIOS_QUERY, 1, &query, sizeof(query),
- 0);
+ return (ret == HPWMI_RET_UNKNOWN_CMDTYPE) ? 0 : -ENXIO;
+}
+static int __init hp_wmi_enable_hotkeys(void)
+{
+ int value = 0x6e;
+ int ret = hp_wmi_perform_query(HPWMI_BIOS_QUERY, 1, &value,
+ sizeof(value), 0);
if (ret)
return -EINVAL;
return 0;
hp_wmi_tablet_state());
input_sync(hp_wmi_input_dev);
- if (hp_wmi_bios_2009_later() == 4)
+ if (!hp_wmi_bios_2009_later() && hp_wmi_bios_2008_later())
hp_wmi_enable_hotkeys();
status = wmi_install_notify_handler(HPWMI_EVENT_GUID, hp_wmi_notify, NULL);
else if (result == TOS_NOT_SUPPORTED)
return -ENODEV;
- return result = TOS_SUCCESS ? 0 : -EIO;
+ return result == TOS_SUCCESS ? 0 : -EIO;
}
static int toshiba_usb_sleep_music_set(struct toshiba_acpi_dev *dev, u32 state)
if (error)
return error;
- error = toshiba_hotkey_event_type_get(dev, &events_type);
- if (error) {
- pr_err("Unable to query Hotkey Event Type\n");
- return error;
- }
+ if (toshiba_hotkey_event_type_get(dev, &events_type))
+ pr_notice("Unable to query Hotkey Event Type\n");
+
dev->hotkey_event_type = events_type;
dev->hotkey_dev = input_allocate_device();
return true;
}
-/*
- * Convert a raw GUID to the ACII string representation
- */
-static int wmi_gtoa(const char *in, char *out)
-{
- int i;
-
- for (i = 3; i >= 0; i--)
- out += sprintf(out, "%02X", in[i] & 0xFF);
-
- out += sprintf(out, "-");
- out += sprintf(out, "%02X", in[5] & 0xFF);
- out += sprintf(out, "%02X", in[4] & 0xFF);
- out += sprintf(out, "-");
- out += sprintf(out, "%02X", in[7] & 0xFF);
- out += sprintf(out, "%02X", in[6] & 0xFF);
- out += sprintf(out, "-");
- out += sprintf(out, "%02X", in[8] & 0xFF);
- out += sprintf(out, "%02X", in[9] & 0xFF);
- out += sprintf(out, "-");
-
- for (i = 10; i <= 15; i++)
- out += sprintf(out, "%02X", in[i] & 0xFF);
-
- *out = '\0';
- return 0;
-}
-
static bool find_guid(const char *guid_string, struct wmi_block **out)
{
char tmp[16], guid_input[16];
static void wmi_dump_wdg(const struct guid_block *g)
{
- char guid_string[37];
-
- wmi_gtoa(g->guid, guid_string);
-
- pr_info("%s:\n", guid_string);
+ pr_info("%pUL:\n", g->guid);
pr_info("\tobject_id: %c%c\n", g->object_id[0], g->object_id[1]);
pr_info("\tnotify_id: %02X\n", g->notify_id);
pr_info("\treserved: %02X\n", g->reserved);
static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
- char guid_string[37];
struct wmi_block *wblock;
wblock = dev_get_drvdata(dev);
return strlen(buf);
}
- wmi_gtoa(wblock->gblock.guid, guid_string);
-
- return sprintf(buf, "wmi:%s\n", guid_string);
+ return sprintf(buf, "wmi:%pUL\n", wblock->gblock.guid);
}
static DEVICE_ATTR_RO(modalias);
if (!wblock)
return -ENOMEM;
- wmi_gtoa(wblock->gblock.guid, guid_string);
+ sprintf(guid_string, "%pUL", wblock->gblock.guid);
strcpy(&env->buf[env->buflen - 1], "wmi:");
memcpy(&env->buf[env->buflen - 1 + 4], guid_string, 36);
static int wmi_create_device(const struct guid_block *gblock,
struct wmi_block *wblock, acpi_handle handle)
{
- char guid_string[37];
-
wblock->dev.class = &wmi_class;
- wmi_gtoa(gblock->guid, guid_string);
- dev_set_name(&wblock->dev, "%s", guid_string);
+ dev_set_name(&wblock->dev, "%pUL", gblock->guid);
dev_set_drvdata(&wblock->dev, wblock);
struct guid_block *block;
struct wmi_block *wblock;
struct list_head *p;
- char guid_string[37];
list_for_each(p, &wmi_block_list) {
wblock = list_entry(p, struct wmi_block, list);
if (wblock->handler)
wblock->handler(event, wblock->handler_data);
if (debug_event) {
- wmi_gtoa(wblock->gblock.guid, guid_string);
- pr_info("DEBUG Event GUID: %s\n", guid_string);
+ pr_info("DEBUG Event GUID: %pUL\n",
+ wblock->gblock.guid);
}
acpi_bus_generate_netlink_event(
#define TWL4030_MSTATEC_COMPLETE1 0x0b
#define TWL4030_MSTATEC_COMPLETE4 0x0e
-#if IS_ENABLED(CONFIG_TWL4030_MADC)
+#if IS_REACHABLE(CONFIG_TWL4030_MADC)
/*
* If AC (Accessory Charger) voltage exceeds 4.5V (MADC 11)
* then AC is available.
phynode = of_find_compatible_node(bci->dev->of_node->parent,
NULL, "ti,twl4030-usb");
- if (phynode) {
+ if (phynode)
bci->transceiver = devm_usb_get_phy_by_node(
bci->dev, phynode, &bci->usb_nb);
- if (IS_ERR(bci->transceiver) &&
- PTR_ERR(bci->transceiver) == -EPROBE_DEFER)
- return -EPROBE_DEFER;
- }
}
/* Enable interrupts now. */
{ .compatible = "fsl,anatop-regulator", },
{ /* end */ }
};
+MODULE_DEVICE_TABLE(of, of_anatop_regulator_match_tbl);
static struct platform_driver anatop_regulator_driver = {
.driver = {
AXP_DESC(AXP22X, DCDC3, "dcdc3", "vin3", 600, 1860, 20,
AXP22X_DCDC3_V_OUT, 0x3f, AXP22X_PWR_OUT_CTRL1, BIT(3)),
AXP_DESC(AXP22X, DCDC4, "dcdc4", "vin4", 600, 1540, 20,
- AXP22X_DCDC4_V_OUT, 0x3f, AXP22X_PWR_OUT_CTRL1, BIT(3)),
+ AXP22X_DCDC4_V_OUT, 0x3f, AXP22X_PWR_OUT_CTRL1, BIT(4)),
AXP_DESC(AXP22X, DCDC5, "dcdc5", "vin5", 1000, 2550, 50,
- AXP22X_DCDC5_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL1, BIT(4)),
+ AXP22X_DCDC5_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL1, BIT(5)),
/* secondary switchable output of DCDC1 */
AXP_DESC_SW(AXP22X, DC1SW, "dc1sw", "dcdc1", 1600, 3400, 100,
AXP22X_DCDC1_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL2, BIT(7)),
return 0;
r = regulator_dev_lookup(dev, rdev->supply_name, &ret);
- if (ret == -ENODEV) {
- /*
- * No supply was specified for this regulator and
- * there will never be one.
- */
- return 0;
- }
-
if (!r) {
+ if (ret == -ENODEV) {
+ /*
+ * No supply was specified for this regulator and
+ * there will never be one.
+ */
+ return 0;
+ }
+
+ /* Did the lookup explicitly defer for us? */
+ if (ret == -EPROBE_DEFER)
+ return ret;
+
if (have_full_constraints()) {
r = dummy_regulator_rdev;
} else {
return ret;
/* Cascade always-on state to supply */
- if (_regulator_is_enabled(rdev)) {
+ if (_regulator_is_enabled(rdev) && rdev->supply) {
ret = regulator_enable(rdev->supply);
if (ret < 0) {
- if (rdev->supply)
- _regulator_put(rdev->supply);
+ _regulator_put(rdev->supply);
return ret;
}
}
{ .compatible = "regulator-gpio", },
{},
};
+MODULE_DEVICE_TABLE(of, regulator_gpio_of_match);
#endif
static struct platform_driver gpio_regulator_driver = {
int voltage;
};
+struct pbias_of_data {
+ unsigned int offset;
+};
+
static const unsigned int pbias_volt_table[] = {
1800000,
3000000
};
#define PBIAS_NUM_REGS ARRAY_SIZE(pbias_matches)
+/* Offset from SCM general area (and syscon) base */
+
+static const struct pbias_of_data pbias_of_data_omap2 = {
+ .offset = 0x230,
+};
+
+static const struct pbias_of_data pbias_of_data_omap3 = {
+ .offset = 0x2b0,
+};
+
+static const struct pbias_of_data pbias_of_data_omap4 = {
+ .offset = 0x60,
+};
+
+static const struct pbias_of_data pbias_of_data_omap5 = {
+ .offset = 0x60,
+};
+
+static const struct pbias_of_data pbias_of_data_dra7 = {
+ .offset = 0xe00,
+};
+
static const struct of_device_id pbias_of_match[] = {
{ .compatible = "ti,pbias-omap", },
+ { .compatible = "ti,pbias-omap2", .data = &pbias_of_data_omap2, },
+ { .compatible = "ti,pbias-omap3", .data = &pbias_of_data_omap3, },
+ { .compatible = "ti,pbias-omap4", .data = &pbias_of_data_omap4, },
+ { .compatible = "ti,pbias-omap5", .data = &pbias_of_data_omap5, },
+ { .compatible = "ti,pbias-dra7", .data = &pbias_of_data_dra7, },
{},
};
MODULE_DEVICE_TABLE(of, pbias_of_match);
const struct pbias_reg_info *info;
int ret = 0;
int count, idx, data_idx = 0;
+ const struct of_device_id *match;
+ const struct pbias_of_data *data;
+ unsigned int offset;
count = of_regulator_match(&pdev->dev, np, pbias_matches,
PBIAS_NUM_REGS);
if (IS_ERR(syscon))
return PTR_ERR(syscon);
+ match = of_match_device(of_match_ptr(pbias_of_match), &pdev->dev);
+ if (match && match->data) {
+ data = match->data;
+ offset = data->offset;
+ } else {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res)
+ return -EINVAL;
+
+ offset = res->start;
+ dev_WARN(&pdev->dev,
+ "using legacy dt data for pbias offset\n");
+ }
+
cfg.regmap = syscon;
cfg.dev = &pdev->dev;
if (!info)
return -ENODEV;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!res)
- return -EINVAL;
-
drvdata[data_idx].syscon = syscon;
drvdata[data_idx].info = info;
drvdata[data_idx].desc.name = info->name;
drvdata[data_idx].desc.volt_table = pbias_volt_table;
drvdata[data_idx].desc.n_voltages = 2;
drvdata[data_idx].desc.enable_time = info->enable_time;
- drvdata[data_idx].desc.vsel_reg = res->start;
+ drvdata[data_idx].desc.vsel_reg = offset;
drvdata[data_idx].desc.vsel_mask = info->vmode;
- drvdata[data_idx].desc.enable_reg = res->start;
+ drvdata[data_idx].desc.enable_reg = offset;
drvdata[data_idx].desc.enable_mask = info->enable_mask;
drvdata[data_idx].desc.enable_val = info->enable;
drvdata[data_idx].desc.disable_val = info->disable_val;
};
static struct tps_info tps65218_pmic_regs[] = {
- TPS65218_INFO(DCDC1, "DCDC1", 850000, 167500),
+ TPS65218_INFO(DCDC1, "DCDC1", 850000, 1675000),
TPS65218_INFO(DCDC2, "DCDC2", 850000, 1675000),
TPS65218_INFO(DCDC3, "DCDC3", 900000, 3400000),
TPS65218_INFO(DCDC4, "DCDC4", 1175000, 3400000),
{ .compatible = "arm,vexpress-volt", },
{ }
};
+MODULE_DEVICE_TABLE(of, vexpress_regulator_of_match);
static struct platform_driver vexpress_regulator_driver = {
.probe = vexpress_regulator_probe,
static int virtio_ccw_read_vq_conf(struct virtio_ccw_device *vcdev,
struct ccw1 *ccw, int index)
{
+ int ret;
+
vcdev->config_block->index = index;
ccw->cmd_code = CCW_CMD_READ_VQ_CONF;
ccw->flags = 0;
ccw->count = sizeof(struct vq_config_block);
ccw->cda = (__u32)(unsigned long)(vcdev->config_block);
- ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_READ_VQ_CONF);
+ ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_READ_VQ_CONF);
+ if (ret)
+ return ret;
return vcdev->config_block->num;
}
goto out_err;
}
info->num = virtio_ccw_read_vq_conf(vcdev, ccw, i);
+ if (info->num < 0) {
+ err = info->num;
+ goto out_err;
+ }
size = PAGE_ALIGN(vring_size(info->num, KVM_VIRTIO_CCW_RING_ALIGN));
info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
if (info->queue == NULL) {
.llseek = noop_llseek,
};
+/*
+ * The controllers use an inline buffer instead of a mapped SGL for small,
+ * single entry buffers. Note that we treat a zero-length transfer like
+ * a mapped SGL.
+ */
+static bool twa_command_mapped(struct scsi_cmnd *cmd)
+{
+ return scsi_sg_count(cmd) != 1 ||
+ scsi_bufflen(cmd) >= TW_MIN_SGL_LENGTH;
+}
+
/* This function will complete an aen request from the isr */
static int twa_aen_complete(TW_Device_Extension *tw_dev, int request_id)
{
}
/* Now complete the io */
- scsi_dma_unmap(cmd);
+ if (twa_command_mapped(cmd))
+ scsi_dma_unmap(cmd);
cmd->scsi_done(cmd);
tw_dev->state[request_id] = TW_S_COMPLETED;
twa_free_request_id(tw_dev, request_id);
struct scsi_cmnd *cmd = tw_dev->srb[i];
cmd->result = (DID_RESET << 16);
- scsi_dma_unmap(cmd);
+ if (twa_command_mapped(cmd))
+ scsi_dma_unmap(cmd);
cmd->scsi_done(cmd);
}
}
retval = twa_scsiop_execute_scsi(tw_dev, request_id, NULL, 0, NULL);
switch (retval) {
case SCSI_MLQUEUE_HOST_BUSY:
- scsi_dma_unmap(SCpnt);
+ if (twa_command_mapped(SCpnt))
+ scsi_dma_unmap(SCpnt);
twa_free_request_id(tw_dev, request_id);
break;
case 1:
SCpnt->result = (DID_ERROR << 16);
- scsi_dma_unmap(SCpnt);
+ if (twa_command_mapped(SCpnt))
+ scsi_dma_unmap(SCpnt);
done(SCpnt);
tw_dev->state[request_id] = TW_S_COMPLETED;
twa_free_request_id(tw_dev, request_id);
/* Map sglist from scsi layer to cmd packet */
if (scsi_sg_count(srb)) {
- if ((scsi_sg_count(srb) == 1) &&
- (scsi_bufflen(srb) < TW_MIN_SGL_LENGTH)) {
+ if (!twa_command_mapped(srb)) {
if (srb->sc_data_direction == DMA_TO_DEVICE ||
srb->sc_data_direction == DMA_BIDIRECTIONAL)
scsi_sg_copy_to_buffer(srb,
{
struct scsi_cmnd *cmd = tw_dev->srb[request_id];
- if (scsi_bufflen(cmd) < TW_MIN_SGL_LENGTH &&
+ if (!twa_command_mapped(cmd) &&
(cmd->sc_data_direction == DMA_FROM_DEVICE ||
cmd->sc_data_direction == DMA_BIDIRECTIONAL)) {
if (scsi_sg_count(cmd) == 1) {
wake_up(&conn->ehwait);
}
-static void iscsi_send_nopout(struct iscsi_conn *conn, struct iscsi_nopin *rhdr)
+static int iscsi_send_nopout(struct iscsi_conn *conn, struct iscsi_nopin *rhdr)
{
struct iscsi_nopout hdr;
struct iscsi_task *task;
if (!rhdr && conn->ping_task)
- return;
+ return -EINVAL;
memset(&hdr, 0, sizeof(struct iscsi_nopout));
hdr.opcode = ISCSI_OP_NOOP_OUT | ISCSI_OP_IMMEDIATE;
hdr.ttt = RESERVED_ITT;
task = __iscsi_conn_send_pdu(conn, (struct iscsi_hdr *)&hdr, NULL, 0);
- if (!task)
+ if (!task) {
iscsi_conn_printk(KERN_ERR, conn, "Could not send nopout\n");
- else if (!rhdr) {
+ return -EIO;
+ } else if (!rhdr) {
/* only track our nops */
conn->ping_task = task;
conn->last_ping = jiffies;
}
+
+ return 0;
}
static int iscsi_nop_out_rsp(struct iscsi_task *task,
if (time_before_eq(last_recv + recv_timeout, jiffies)) {
/* send a ping to try to provoke some traffic */
ISCSI_DBG_CONN(conn, "Sending nopout as ping\n");
- iscsi_send_nopout(conn, NULL);
- next_timeout = conn->last_ping + (conn->ping_timeout * HZ);
+ if (iscsi_send_nopout(conn, NULL))
+ next_timeout = jiffies + (1 * HZ);
+ else
+ next_timeout = conn->last_ping + (conn->ping_timeout * HZ);
} else
next_timeout = last_recv + recv_timeout;
dh = __scsi_dh_lookup(name);
if (!dh) {
- request_module(name);
+ request_module("scsi_dh_%s", name);
dh = __scsi_dh_lookup(name);
}
static void scsi_mq_done(struct scsi_cmnd *cmd)
{
trace_scsi_dispatch_cmd_done(cmd);
- blk_mq_complete_request(cmd->request);
+ blk_mq_complete_request(cmd->request, cmd->request->errors);
}
static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
raw_spin_unlock_irqrestore(&intc_big_lock, flags);
}
-static void intc_redirect_irq(unsigned int irq, struct irq_desc *desc)
+static void intc_redirect_irq(struct irq_desc *desc)
{
generic_handle_irq((unsigned int)irq_desc_get_handler_data(desc));
}
*/
static inline void activate_irq(int irq)
{
-#ifdef CONFIG_ARM
- /* ARM requires an extra step to clear IRQ_NOREQUEST, which it
- * sets on behalf of every irq_chip. Also sets IRQ_NOPROBE.
- */
- set_irq_flags(irq, IRQF_VALID);
-#else
- /* same effect on other architectures */
- irq_set_noprobe(irq);
-#endif
+ irq_modify_status(irq, IRQ_NOREQUEST, IRQ_NOPROBE);
}
static inline int intc_handle_int_cmp(const void *a, const void *b)
return 0;
}
-static void intc_virq_handler(unsigned int __irq, struct irq_desc *desc)
+static void intc_virq_handler(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
struct irq_data *data = irq_desc_get_irq_data(desc);
handle = (unsigned long)irq_desc_get_handler_data(vdesc);
addr = INTC_REG(d, _INTC_ADDR_E(handle), 0);
if (intc_reg_fns[_INTC_FN(handle)](addr, handle, 0))
- generic_handle_irq_desc(entry->irq, vdesc);
+ generic_handle_irq_desc(vdesc);
}
}
static int __init sh_pm_runtime_init(void)
{
if (IS_ENABLED(CONFIG_ARCH_SHMOBILE_MULTI)) {
- if (!of_machine_is_compatible("renesas,emev2") &&
- !of_machine_is_compatible("renesas,r7s72100") &&
-#ifndef CONFIG_PM_GENERIC_DOMAINS_OF
- !of_machine_is_compatible("renesas,r8a73a4") &&
- !of_machine_is_compatible("renesas,r8a7740") &&
- !of_machine_is_compatible("renesas,sh73a0") &&
-#endif
- !of_machine_is_compatible("renesas,r8a7778") &&
- !of_machine_is_compatible("renesas,r8a7779") &&
- !of_machine_is_compatible("renesas,r8a7790") &&
- !of_machine_is_compatible("renesas,r8a7791") &&
- !of_machine_is_compatible("renesas,r8a7792") &&
- !of_machine_is_compatible("renesas,r8a7793") &&
- !of_machine_is_compatible("renesas,r8a7794"))
+ if (!of_find_compatible_node(NULL, NULL,
+ "renesas,cpg-mstp-clocks"))
+ return 0;
+ if (IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS_OF) &&
+ of_find_node_with_property(NULL, "#power-domain-cells"))
return 0;
}
}
/* PMU IRQ controller */
-static void pmu_irq_handler(unsigned int irq, struct irq_desc *desc)
+static void pmu_irq_handler(struct irq_desc *desc)
{
- struct pmu_data *pmu = irq_get_handler_data(irq);
+ struct pmu_data *pmu = irq_desc_get_handler_data(desc);
struct irq_chip_generic *gc = pmu->irq_gc;
struct irq_domain *domain = pmu->irq_domain;
void __iomem *base = gc->reg_base;
u32 done = ~0;
if (stat == 0) {
- handle_bad_irq(irq, desc);
+ handle_bad_irq(desc);
return;
}
return clk_prepare_enable(as->clk);
}
+#ifdef CONFIG_PM_SLEEP
static int atmel_spi_suspend(struct device *dev)
{
struct spi_master *master = dev_get_drvdata(dev);
return ret;
}
+#endif
static const struct dev_pm_ops atmel_spi_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(atmel_spi_suspend, atmel_spi_resume)
/* otherwise we only allow transfers within the same page
* to avoid wasting time on dma_mapping when it is not practical
*/
- if (((size_t)tfr->tx_buf & PAGE_MASK) + tfr->len > PAGE_SIZE) {
+ if (((size_t)tfr->tx_buf & (PAGE_SIZE - 1)) + tfr->len > PAGE_SIZE) {
dev_warn_once(&spi->dev,
"Unaligned spi tx-transfer bridging page\n");
return false;
}
- if (((size_t)tfr->rx_buf & PAGE_MASK) + tfr->len > PAGE_SIZE) {
+ if (((size_t)tfr->rx_buf & (PAGE_SIZE - 1)) + tfr->len > PAGE_SIZE) {
dev_warn_once(&spi->dev,
- "Unaligned spi tx-transfer bridging page\n");
+ "Unaligned spi rx-transfer bridging page\n");
return false;
}
goto free_master;
}
- dspi->irq = platform_get_irq(pdev, 0);
- if (dspi->irq <= 0) {
+ ret = platform_get_irq(pdev, 0);
+ if (ret == 0)
ret = -EINVAL;
+ if (ret < 0)
goto free_master;
- }
+ dspi->irq = ret;
ret = devm_request_threaded_irq(&pdev->dev, dspi->irq, davinci_spi_irq,
dummy_thread_fn, 0, dev_name(&pdev->dev), dspi);
{ .compatible = "amlogic,meson6-spifc", },
{ },
};
+MODULE_DEVICE_TABLE(of, meson_spifc_dt_match);
static struct platform_driver meson_spifc_driver = {
.probe = meson_spifc_probe,
void __iomem *base;
u32 state;
u32 pad_sel;
- struct clk *spi_clk, *parent_clk;
+ struct clk *parent_clk, *sel_clk, *spi_clk;
struct spi_transfer *cur_transfer;
u32 xfer_len;
struct scatterlist *tx_sgl, *rx_sgl;
writel(mdata->pad_sel, mdata->base + SPI_PAD_SEL_REG);
}
-static int mtk_spi_prepare_hardware(struct spi_master *master)
-{
- struct spi_transfer *trans;
- struct mtk_spi *mdata = spi_master_get_devdata(master);
- struct spi_message *msg = master->cur_msg;
-
- trans = list_first_entry(&msg->transfers, struct spi_transfer,
- transfer_list);
- if (!trans->cs_change) {
- mdata->state = MTK_SPI_IDLE;
- mtk_spi_reset(mdata);
- }
-
- return 0;
-}
-
static int mtk_spi_prepare_message(struct spi_master *master,
struct spi_message *msg)
{
struct mtk_spi *mdata = spi_master_get_devdata(spi->master);
reg_val = readl(mdata->base + SPI_CMD_REG);
- if (!enable)
+ if (!enable) {
reg_val |= SPI_CMD_PAUSE_EN;
- else
+ writel(reg_val, mdata->base + SPI_CMD_REG);
+ } else {
reg_val &= ~SPI_CMD_PAUSE_EN;
- writel(reg_val, mdata->base + SPI_CMD_REG);
+ writel(reg_val, mdata->base + SPI_CMD_REG);
+ mdata->state = MTK_SPI_IDLE;
+ mtk_spi_reset(mdata);
+ }
}
static void mtk_spi_prepare_transfer(struct spi_master *master,
master->mode_bits = SPI_CPOL | SPI_CPHA;
master->set_cs = mtk_spi_set_cs;
- master->prepare_transfer_hardware = mtk_spi_prepare_hardware;
master->prepare_message = mtk_spi_prepare_message;
master->transfer_one = mtk_spi_transfer_one;
master->can_dma = mtk_spi_can_dma;
goto err_put_master;
}
- mdata->spi_clk = devm_clk_get(&pdev->dev, "spi-clk");
- if (IS_ERR(mdata->spi_clk)) {
- ret = PTR_ERR(mdata->spi_clk);
- dev_err(&pdev->dev, "failed to get spi-clk: %d\n", ret);
- goto err_put_master;
- }
-
mdata->parent_clk = devm_clk_get(&pdev->dev, "parent-clk");
if (IS_ERR(mdata->parent_clk)) {
ret = PTR_ERR(mdata->parent_clk);
goto err_put_master;
}
+ mdata->sel_clk = devm_clk_get(&pdev->dev, "sel-clk");
+ if (IS_ERR(mdata->sel_clk)) {
+ ret = PTR_ERR(mdata->sel_clk);
+ dev_err(&pdev->dev, "failed to get sel-clk: %d\n", ret);
+ goto err_put_master;
+ }
+
+ mdata->spi_clk = devm_clk_get(&pdev->dev, "spi-clk");
+ if (IS_ERR(mdata->spi_clk)) {
+ ret = PTR_ERR(mdata->spi_clk);
+ dev_err(&pdev->dev, "failed to get spi-clk: %d\n", ret);
+ goto err_put_master;
+ }
+
ret = clk_prepare_enable(mdata->spi_clk);
if (ret < 0) {
dev_err(&pdev->dev, "failed to enable spi_clk (%d)\n", ret);
goto err_put_master;
}
- ret = clk_set_parent(mdata->spi_clk, mdata->parent_clk);
+ ret = clk_set_parent(mdata->sel_clk, mdata->parent_clk);
if (ret < 0) {
dev_err(&pdev->dev, "failed to clk_set_parent (%d)\n", ret);
goto err_disable_clk;
pm_runtime_disable(&pdev->dev);
mtk_spi_reset(mdata);
- clk_disable_unprepare(mdata->spi_clk);
spi_master_put(master);
return 0;
if (!(sccr1_reg & SSCR1_TIE))
mask &= ~SSSR_TFS;
+ /* Ignore RX timeout interrupt if it is disabled */
+ if (!(sccr1_reg & SSCR1_TINTE))
+ mask &= ~SSSR_TINT;
+
if (!(status & mask))
return IRQ_NONE;
static inline void xtfpga_spi_write32(const struct xtfpga_spi *spi,
unsigned addr, u32 val)
{
- iowrite32(val, spi->regs + addr);
+ __raw_writel(val, spi->regs + addr);
}
static inline unsigned int xtfpga_spi_read32(const struct xtfpga_spi *spi,
unsigned addr)
{
- return ioread32(spi->regs + addr);
+ return __raw_readl(spi->regs + addr);
}
static inline void xtfpga_spi_wait_busy(struct xtfpga_spi *xspi)
*
* The caller is responsible for assigning the bus number and initializing
* the master's methods before calling spi_register_master(); and (after errors
- * adding the device) calling spi_master_put() and kfree() to prevent a memory
- * leak.
+ * adding the device) calling spi_master_put() to prevent a memory leak.
*/
struct spi_master *spi_alloc_master(struct device *dev, unsigned size)
{
kfree(spidev->rx_buffer);
spidev->rx_buffer = NULL;
- spidev->speed_hz = spidev->spi->max_speed_hz;
+ if (spidev->spi)
+ spidev->speed_hz = spidev->spi->max_speed_hz;
/* ... after we unbound from the underlying device? */
spin_lock_irq(&spidev->spi_lock);
}
}
-static void pmic_arb_chained_irq(unsigned int irq, struct irq_desc *desc)
+static void pmic_arb_chained_irq(struct irq_desc *desc)
{
struct spmi_pmic_arb_dev *pa = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
- add proper arch dependencies as needed
- audit userspace interfaces to make sure they are sane
+
+ion/
+ - Remove ION_IOC_SYNC: Flushing for devices should be purely a kernel internal
+ interface on top of dma-buf. flush_for_device needs to be added to dma-buf
+ first.
+ - Remove ION_IOC_CUSTOM: Atm used for cache flushing for cpu access in some
+ vendor trees. Should be replaced with an ioctl on the dma-buf to expose the
+ begin/end_cpu_access hooks to userspace.
+ - Clarify the tricks ion plays with explicitly managing coherency behind the
+ dma api's back (this is absolutely needed for high-perf gpu drivers): Add an
+ explicit coherency management mode to flush_for_device to be used by drivers
+ which want to manage caches themselves and which indicates whether cpu caches
+ need flushing.
+ - With those removed there's probably no use for ION_IOC_IMPORT anymore either
+ since ion would just be the central allocator for shared buffers.
+ - Add dt-binding to expose cma regions as ion heaps, with the rule that any
+ such cma regions must already be used by some device for dma. I.e. ion only
+ exposes existing cma regions and doesn't reserve unecessarily memory when
+ booting a system which doesn't use ion.
+
Please send patches to Greg Kroah-Hartman <greg@kroah.com> and Cc:
Arve Hjønnevåg <arve@android.com> and Riley Andrews <riandrews@android.com>
mutex_unlock(&client->lock);
goto end;
}
- mutex_unlock(&client->lock);
handle = ion_handle_create(client, buffer);
- if (IS_ERR(handle))
+ if (IS_ERR(handle)) {
+ mutex_unlock(&client->lock);
goto end;
+ }
- mutex_lock(&client->lock);
ret = ion_handle_add(client, handle);
mutex_unlock(&client->lock);
if (ret) {
/* Set CS active high */
par->spi->mode |= SPI_CS_HIGH;
- ret = par->spi->master->setup(par->spi);
+ ret = spi_setup(par->spi);
if (ret) {
dev_err(par->info->device, "Could not set SPI_CS_HIGH\n");
return ret;
/* enable SPI interface by having CS and MOSI low during reset */
save_mode = par->spi->mode;
par->spi->mode |= SPI_CS_HIGH;
- ret = par->spi->master->setup(par->spi); /* set CS inactive low */
+ ret = spi_setup(par->spi); /* set CS inactive low */
if (ret) {
dev_err(par->info->device, "Could not set SPI_CS_HIGH\n");
return ret;
par->fbtftops.reset(par);
mdelay(1000);
par->spi->mode = save_mode;
- ret = par->spi->master->setup(par->spi);
+ ret = spi_setup(par->spi);
if (ret) {
dev_err(par->info->device, "Could not restore SPI mode\n");
return ret;
/* 9-bit SPI setup */
if (par->spi && display->buswidth == 9) {
- par->spi->bits_per_word = 9;
- ret = par->spi->master->setup(par->spi);
- if (ret) {
+ if (par->spi->master->bits_per_word_mask & SPI_BPW_MASK(9)) {
+ par->spi->bits_per_word = 9;
+ } else {
dev_warn(&par->spi->dev,
"9-bit SPI not available, emulating using 8-bit.\n");
- par->spi->bits_per_word = 8;
- ret = par->spi->master->setup(par->spi);
- if (ret)
- goto out_release;
/* allocate buffer with room for dc bits */
par->extra = devm_kzalloc(par->info->device,
par->txbuf.len + (par->txbuf.len / 8) + 8,
}
par->fbtftops.write_register = fbtft_write_reg8_bus9;
par->fbtftops.write_vmem = fbtft_write_vmem16_bus9;
- sdev->bits_per_word = 9;
- ret = sdev->master->setup(sdev);
- if (ret) {
+ if (par->spi->master->bits_per_word_mask
+ & SPI_BPW_MASK(9)) {
+ par->spi->bits_per_word = 9;
+ } else {
dev_warn(dev,
"9-bit SPI not available, emulating using 8-bit.\n");
- sdev->bits_per_word = 8;
- ret = sdev->master->setup(sdev);
- if (ret)
- goto out_release;
/* allocate buffer with room for dc bits */
par->extra = devm_kzalloc(par->info->device,
par->txbuf.len + (par->txbuf.len / 8) + 8,
Lustre has independent Metadata and Data servers that clients can access
in parallel to maximize performance.
-In order to use Lustre client you will need to download lustre client
-tools from
-https://downloads.hpdd.intel.com/public/lustre/latest-feature-release/
-the package name is lustre-client.
+In order to use Lustre client you will need to download the "lustre-client"
+package that contains the userspace tools from http://lustre.org/download/
You will need to install and configure your Lustre servers separately.
More Information
================
-You can get more information at
-OpenSFS website: http://lustre.opensfs.org/about/
-Intel HPDD wiki: https://wiki.hpdd.intel.com
+You can get more information at the Lustre website: http://wiki.lustre.org/
-Out of tree Lustre client and server code is available at:
-http://git.whamcloud.com/fs/lustre-release.git
+Source for the userspace tools and out-of-tree client and server code
+is available at: http://git.hpdd.intel.com/fs/lustre-release.git
Latest binary packages:
-http://lustre.opensfs.org/download-lustre/
+http://lustre.org/download/
menuconfig MOST
tristate "MOST driver"
+ depends on HAS_DMA
select MOSTCORE
default n
---help---
config HDM_DIM2
tristate "DIM2 HDM"
depends on AIM_NETWORK
+ depends on HAS_IOMEM
---help---
Say Y here if you want to connect via MediaLB to network transceiver.
config HDM_USB
tristate "USB HDM"
- depends on USB
+ depends on USB && NET
select AIM_NETWORK
---help---
Say Y here if you want to connect via USB to network tranceiver.
config MOSTCORE
tristate "MOST Core"
+ depends on HAS_DMA
---help---
Say Y here if you want to enable MOST support.
source "drivers/staging/rdma/amso1100/Kconfig"
+source "drivers/staging/rdma/ehca/Kconfig"
+
source "drivers/staging/rdma/hfi1/Kconfig"
source "drivers/staging/rdma/ipath/Kconfig"
# Entries for RDMA_STAGING tree
obj-$(CONFIG_INFINIBAND_AMSO1100) += amso1100/
+obj-$(CONFIG_INFINIBAND_EHCA) += ehca/
obj-$(CONFIG_INFINIBAND_HFI1) += hfi1/
obj-$(CONFIG_INFINIBAND_IPATH) += ipath/
--- /dev/null
+config INFINIBAND_EHCA
+ tristate "eHCA support"
+ depends on IBMEBUS
+ ---help---
+ This driver supports the deprecated IBM pSeries eHCA InfiniBand
+ adapter.
+
+ To compile the driver as a module, choose M here. The module
+ will be called ib_ehca.
+
--- /dev/null
+# Authors: Heiko J Schick <schickhj@de.ibm.com>
+# Christoph Raisch <raisch@de.ibm.com>
+# Joachim Fenkes <fenkes@de.ibm.com>
+#
+# Copyright (c) 2005 IBM Corporation
+#
+# All rights reserved.
+#
+# This source code is distributed under a dual license of GPL v2.0 and OpenIB BSD.
+
+obj-$(CONFIG_INFINIBAND_EHCA) += ib_ehca.o
+
+ib_ehca-objs = ehca_main.o ehca_hca.o ehca_mcast.o ehca_pd.o ehca_av.o ehca_eq.o \
+ ehca_cq.o ehca_qp.o ehca_sqp.o ehca_mrmw.o ehca_reqs.o ehca_irq.o \
+ ehca_uverbs.o ipz_pt_fn.o hcp_if.o hcp_phyp.o
+
--- /dev/null
+9/2015
+
+The ehca driver has been deprecated and moved to drivers/staging/rdma.
+It will be removed in the 4.6 merge window.
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * address vector functions
+ *
+ * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Khadija Souissi <souissik@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+static struct kmem_cache *av_cache;
+
+int ehca_calc_ipd(struct ehca_shca *shca, int port,
+ enum ib_rate path_rate, u32 *ipd)
+{
+ int path = ib_rate_to_mult(path_rate);
+ int link, ret;
+ struct ib_port_attr pa;
+
+ if (path_rate == IB_RATE_PORT_CURRENT) {
+ *ipd = 0;
+ return 0;
+ }
+
+ if (unlikely(path < 0)) {
+ ehca_err(&shca->ib_device, "Invalid static rate! path_rate=%x",
+ path_rate);
+ return -EINVAL;
+ }
+
+ ret = ehca_query_port(&shca->ib_device, port, &pa);
+ if (unlikely(ret < 0)) {
+ ehca_err(&shca->ib_device, "Failed to query port ret=%i", ret);
+ return ret;
+ }
+
+ link = ib_width_enum_to_int(pa.active_width) * pa.active_speed;
+
+ if (path >= link)
+ /* no need to throttle if path faster than link */
+ *ipd = 0;
+ else
+ /* IPD = round((link / path) - 1) */
+ *ipd = ((link + (path >> 1)) / path) - 1;
+
+ return 0;
+}
+
+struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
+{
+ int ret;
+ struct ehca_av *av;
+ struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+ ib_device);
+
+ av = kmem_cache_alloc(av_cache, GFP_KERNEL);
+ if (!av) {
+ ehca_err(pd->device, "Out of memory pd=%p ah_attr=%p",
+ pd, ah_attr);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ av->av.sl = ah_attr->sl;
+ av->av.dlid = ah_attr->dlid;
+ av->av.slid_path_bits = ah_attr->src_path_bits;
+
+ if (ehca_static_rate < 0) {
+ u32 ipd;
+ if (ehca_calc_ipd(shca, ah_attr->port_num,
+ ah_attr->static_rate, &ipd)) {
+ ret = -EINVAL;
+ goto create_ah_exit1;
+ }
+ av->av.ipd = ipd;
+ } else
+ av->av.ipd = ehca_static_rate;
+
+ av->av.lnh = ah_attr->ah_flags;
+ av->av.grh.word_0 = EHCA_BMASK_SET(GRH_IPVERSION_MASK, 6);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_TCLASS_MASK,
+ ah_attr->grh.traffic_class);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
+ ah_attr->grh.flow_label);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
+ ah_attr->grh.hop_limit);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1B);
+ /* set sgid in grh.word_1 */
+ if (ah_attr->ah_flags & IB_AH_GRH) {
+ int rc;
+ struct ib_port_attr port_attr;
+ union ib_gid gid;
+ memset(&port_attr, 0, sizeof(port_attr));
+ rc = ehca_query_port(pd->device, ah_attr->port_num,
+ &port_attr);
+ if (rc) { /* invalid port number */
+ ret = -EINVAL;
+ ehca_err(pd->device, "Invalid port number "
+ "ehca_query_port() returned %x "
+ "pd=%p ah_attr=%p", rc, pd, ah_attr);
+ goto create_ah_exit1;
+ }
+ memset(&gid, 0, sizeof(gid));
+ rc = ehca_query_gid(pd->device,
+ ah_attr->port_num,
+ ah_attr->grh.sgid_index, &gid);
+ if (rc) {
+ ret = -EINVAL;
+ ehca_err(pd->device, "Failed to retrieve sgid "
+ "ehca_query_gid() returned %x "
+ "pd=%p ah_attr=%p", rc, pd, ah_attr);
+ goto create_ah_exit1;
+ }
+ memcpy(&av->av.grh.word_1, &gid, sizeof(gid));
+ }
+ av->av.pmtu = shca->max_mtu;
+
+ /* dgid comes in grh.word_3 */
+ memcpy(&av->av.grh.word_3, &ah_attr->grh.dgid,
+ sizeof(ah_attr->grh.dgid));
+
+ return &av->ib_ah;
+
+create_ah_exit1:
+ kmem_cache_free(av_cache, av);
+
+ return ERR_PTR(ret);
+}
+
+int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
+{
+ struct ehca_av *av;
+ struct ehca_ud_av new_ehca_av;
+ struct ehca_shca *shca = container_of(ah->pd->device, struct ehca_shca,
+ ib_device);
+
+ memset(&new_ehca_av, 0, sizeof(new_ehca_av));
+ new_ehca_av.sl = ah_attr->sl;
+ new_ehca_av.dlid = ah_attr->dlid;
+ new_ehca_av.slid_path_bits = ah_attr->src_path_bits;
+ new_ehca_av.ipd = ah_attr->static_rate;
+ new_ehca_av.lnh = EHCA_BMASK_SET(GRH_FLAG_MASK,
+ (ah_attr->ah_flags & IB_AH_GRH) > 0);
+ new_ehca_av.grh.word_0 = EHCA_BMASK_SET(GRH_TCLASS_MASK,
+ ah_attr->grh.traffic_class);
+ new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
+ ah_attr->grh.flow_label);
+ new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
+ ah_attr->grh.hop_limit);
+ new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1b);
+
+ /* set sgid in grh.word_1 */
+ if (ah_attr->ah_flags & IB_AH_GRH) {
+ int rc;
+ struct ib_port_attr port_attr;
+ union ib_gid gid;
+ memset(&port_attr, 0, sizeof(port_attr));
+ rc = ehca_query_port(ah->device, ah_attr->port_num,
+ &port_attr);
+ if (rc) { /* invalid port number */
+ ehca_err(ah->device, "Invalid port number "
+ "ehca_query_port() returned %x "
+ "ah=%p ah_attr=%p port_num=%x",
+ rc, ah, ah_attr, ah_attr->port_num);
+ return -EINVAL;
+ }
+ memset(&gid, 0, sizeof(gid));
+ rc = ehca_query_gid(ah->device,
+ ah_attr->port_num,
+ ah_attr->grh.sgid_index, &gid);
+ if (rc) {
+ ehca_err(ah->device, "Failed to retrieve sgid "
+ "ehca_query_gid() returned %x "
+ "ah=%p ah_attr=%p port_num=%x "
+ "sgid_index=%x",
+ rc, ah, ah_attr, ah_attr->port_num,
+ ah_attr->grh.sgid_index);
+ return -EINVAL;
+ }
+ memcpy(&new_ehca_av.grh.word_1, &gid, sizeof(gid));
+ }
+
+ new_ehca_av.pmtu = shca->max_mtu;
+
+ memcpy(&new_ehca_av.grh.word_3, &ah_attr->grh.dgid,
+ sizeof(ah_attr->grh.dgid));
+
+ av = container_of(ah, struct ehca_av, ib_ah);
+ av->av = new_ehca_av;
+
+ return 0;
+}
+
+int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
+{
+ struct ehca_av *av = container_of(ah, struct ehca_av, ib_ah);
+
+ memcpy(&ah_attr->grh.dgid, &av->av.grh.word_3,
+ sizeof(ah_attr->grh.dgid));
+ ah_attr->sl = av->av.sl;
+
+ ah_attr->dlid = av->av.dlid;
+
+ ah_attr->src_path_bits = av->av.slid_path_bits;
+ ah_attr->static_rate = av->av.ipd;
+ ah_attr->ah_flags = EHCA_BMASK_GET(GRH_FLAG_MASK, av->av.lnh);
+ ah_attr->grh.traffic_class = EHCA_BMASK_GET(GRH_TCLASS_MASK,
+ av->av.grh.word_0);
+ ah_attr->grh.hop_limit = EHCA_BMASK_GET(GRH_HOPLIMIT_MASK,
+ av->av.grh.word_0);
+ ah_attr->grh.flow_label = EHCA_BMASK_GET(GRH_FLOWLABEL_MASK,
+ av->av.grh.word_0);
+
+ return 0;
+}
+
+int ehca_destroy_ah(struct ib_ah *ah)
+{
+ kmem_cache_free(av_cache, container_of(ah, struct ehca_av, ib_ah));
+
+ return 0;
+}
+
+int ehca_init_av_cache(void)
+{
+ av_cache = kmem_cache_create("ehca_cache_av",
+ sizeof(struct ehca_av), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!av_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_av_cache(void)
+{
+ if (av_cache)
+ kmem_cache_destroy(av_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Struct definition for eHCA internal structures
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_CLASSES_H__
+#define __EHCA_CLASSES_H__
+
+struct ehca_module;
+struct ehca_qp;
+struct ehca_cq;
+struct ehca_eq;
+struct ehca_mr;
+struct ehca_mw;
+struct ehca_pd;
+struct ehca_av;
+
+#include <linux/wait.h>
+#include <linux/mutex.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+
+#ifdef CONFIG_PPC64
+#include "ehca_classes_pSeries.h"
+#endif
+#include "ipz_pt_fn.h"
+#include "ehca_qes.h"
+#include "ehca_irq.h"
+
+#define EHCA_EQE_CACHE_SIZE 20
+#define EHCA_MAX_NUM_QUEUES 0xffff
+
+struct ehca_eqe_cache_entry {
+ struct ehca_eqe *eqe;
+ struct ehca_cq *cq;
+};
+
+struct ehca_eq {
+ u32 length;
+ struct ipz_queue ipz_queue;
+ struct ipz_eq_handle ipz_eq_handle;
+ struct work_struct work;
+ struct h_galpas galpas;
+ int is_initialized;
+ struct ehca_pfeq pf;
+ spinlock_t spinlock;
+ struct tasklet_struct interrupt_task;
+ u32 ist;
+ spinlock_t irq_spinlock;
+ struct ehca_eqe_cache_entry eqe_cache[EHCA_EQE_CACHE_SIZE];
+};
+
+struct ehca_sma_attr {
+ u16 lid, lmc, sm_sl, sm_lid;
+ u16 pkey_tbl_len, pkeys[16];
+};
+
+struct ehca_sport {
+ struct ib_cq *ibcq_aqp1;
+ struct ib_qp *ibqp_sqp[2];
+ /* lock to serialze modify_qp() calls for sqp in normal
+ * and irq path (when event PORT_ACTIVE is received first time)
+ */
+ spinlock_t mod_sqp_lock;
+ enum ib_port_state port_state;
+ struct ehca_sma_attr saved_attr;
+ u32 pma_qp_nr;
+};
+
+#define HCA_CAP_MR_PGSIZE_4K 0x80000000
+#define HCA_CAP_MR_PGSIZE_64K 0x40000000
+#define HCA_CAP_MR_PGSIZE_1M 0x20000000
+#define HCA_CAP_MR_PGSIZE_16M 0x10000000
+
+struct ehca_shca {
+ struct ib_device ib_device;
+ struct platform_device *ofdev;
+ u8 num_ports;
+ int hw_level;
+ struct list_head shca_list;
+ struct ipz_adapter_handle ipz_hca_handle;
+ struct ehca_sport sport[2];
+ struct ehca_eq eq;
+ struct ehca_eq neq;
+ struct ehca_mr *maxmr;
+ struct ehca_pd *pd;
+ struct h_galpas galpas;
+ struct mutex modify_mutex;
+ u64 hca_cap;
+ /* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
+ u32 hca_cap_mr_pgsize;
+ int max_mtu;
+ int max_num_qps;
+ int max_num_cqs;
+ atomic_t num_cqs;
+ atomic_t num_qps;
+};
+
+struct ehca_pd {
+ struct ib_pd ib_pd;
+ struct ipz_pd fw_pd;
+ /* small queue mgmt */
+ struct mutex lock;
+ struct list_head free[2];
+ struct list_head full[2];
+};
+
+enum ehca_ext_qp_type {
+ EQPT_NORMAL = 0,
+ EQPT_LLQP = 1,
+ EQPT_SRQBASE = 2,
+ EQPT_SRQ = 3,
+};
+
+/* struct to cache modify_qp()'s parms for GSI/SMI qp */
+struct ehca_mod_qp_parm {
+ int mask;
+ struct ib_qp_attr attr;
+};
+
+#define EHCA_MOD_QP_PARM_MAX 4
+
+#define QMAP_IDX_MASK 0xFFFFULL
+
+/* struct for tracking if cqes have been reported to the application */
+struct ehca_qmap_entry {
+ u16 app_wr_id;
+ u8 reported;
+ u8 cqe_req;
+};
+
+struct ehca_queue_map {
+ struct ehca_qmap_entry *map;
+ unsigned int entries;
+ unsigned int tail;
+ unsigned int left_to_poll;
+ unsigned int next_wqe_idx; /* Idx to first wqe to be flushed */
+};
+
+/* function to calculate the next index for the qmap */
+static inline unsigned int next_index(unsigned int cur_index, unsigned int limit)
+{
+ unsigned int temp = cur_index + 1;
+ return (temp == limit) ? 0 : temp;
+}
+
+struct ehca_qp {
+ union {
+ struct ib_qp ib_qp;
+ struct ib_srq ib_srq;
+ };
+ u32 qp_type;
+ enum ehca_ext_qp_type ext_type;
+ enum ib_qp_state state;
+ struct ipz_queue ipz_squeue;
+ struct ehca_queue_map sq_map;
+ struct ipz_queue ipz_rqueue;
+ struct ehca_queue_map rq_map;
+ struct h_galpas galpas;
+ u32 qkey;
+ u32 real_qp_num;
+ u32 token;
+ spinlock_t spinlock_s;
+ spinlock_t spinlock_r;
+ u32 sq_max_inline_data_size;
+ struct ipz_qp_handle ipz_qp_handle;
+ struct ehca_pfqp pf;
+ struct ib_qp_init_attr init_attr;
+ struct ehca_cq *send_cq;
+ struct ehca_cq *recv_cq;
+ unsigned int sqerr_purgeflag;
+ struct hlist_node list_entries;
+ /* array to cache modify_qp()'s parms for GSI/SMI qp */
+ struct ehca_mod_qp_parm *mod_qp_parm;
+ int mod_qp_parm_idx;
+ /* mmap counter for resources mapped into user space */
+ u32 mm_count_squeue;
+ u32 mm_count_rqueue;
+ u32 mm_count_galpa;
+ /* unsolicited ack circumvention */
+ int unsol_ack_circ;
+ int mtu_shift;
+ u32 message_count;
+ u32 packet_count;
+ atomic_t nr_events; /* events seen */
+ wait_queue_head_t wait_completion;
+ int mig_armed;
+ struct list_head sq_err_node;
+ struct list_head rq_err_node;
+};
+
+#define IS_SRQ(qp) (qp->ext_type == EQPT_SRQ)
+#define HAS_SQ(qp) (qp->ext_type != EQPT_SRQ)
+#define HAS_RQ(qp) (qp->ext_type != EQPT_SRQBASE)
+
+/* must be power of 2 */
+#define QP_HASHTAB_LEN 8
+
+struct ehca_cq {
+ struct ib_cq ib_cq;
+ struct ipz_queue ipz_queue;
+ struct h_galpas galpas;
+ spinlock_t spinlock;
+ u32 cq_number;
+ u32 token;
+ u32 nr_of_entries;
+ struct ipz_cq_handle ipz_cq_handle;
+ struct ehca_pfcq pf;
+ spinlock_t cb_lock;
+ struct hlist_head qp_hashtab[QP_HASHTAB_LEN];
+ struct list_head entry;
+ u32 nr_callbacks; /* #events assigned to cpu by scaling code */
+ atomic_t nr_events; /* #events seen */
+ wait_queue_head_t wait_completion;
+ spinlock_t task_lock;
+ /* mmap counter for resources mapped into user space */
+ u32 mm_count_queue;
+ u32 mm_count_galpa;
+ struct list_head sqp_err_list;
+ struct list_head rqp_err_list;
+};
+
+enum ehca_mr_flag {
+ EHCA_MR_FLAG_FMR = 0x80000000, /* FMR, created with ehca_alloc_fmr */
+ EHCA_MR_FLAG_MAXMR = 0x40000000, /* max-MR */
+};
+
+struct ehca_mr {
+ union {
+ struct ib_mr ib_mr; /* must always be first in ehca_mr */
+ struct ib_fmr ib_fmr; /* must always be first in ehca_mr */
+ } ib;
+ struct ib_umem *umem;
+ spinlock_t mrlock;
+
+ enum ehca_mr_flag flags;
+ u32 num_kpages; /* number of kernel pages */
+ u32 num_hwpages; /* number of hw pages to form MR */
+ u64 hwpage_size; /* hw page size used for this MR */
+ int acl; /* ACL (stored here for usage in reregister) */
+ u64 *start; /* virtual start address (stored here for */
+ /* usage in reregister) */
+ u64 size; /* size (stored here for usage in reregister) */
+ u32 fmr_page_size; /* page size for FMR */
+ u32 fmr_max_pages; /* max pages for FMR */
+ u32 fmr_max_maps; /* max outstanding maps for FMR */
+ u32 fmr_map_cnt; /* map counter for FMR */
+ /* fw specific data */
+ struct ipz_mrmw_handle ipz_mr_handle; /* MR handle for h-calls */
+ struct h_galpas galpas;
+};
+
+struct ehca_mw {
+ struct ib_mw ib_mw; /* gen2 mw, must always be first in ehca_mw */
+ spinlock_t mwlock;
+
+ u8 never_bound; /* indication MW was never bound */
+ struct ipz_mrmw_handle ipz_mw_handle; /* MW handle for h-calls */
+ struct h_galpas galpas;
+};
+
+enum ehca_mr_pgi_type {
+ EHCA_MR_PGI_PHYS = 1, /* type of ehca_reg_phys_mr,
+ * ehca_rereg_phys_mr,
+ * ehca_reg_internal_maxmr */
+ EHCA_MR_PGI_USER = 2, /* type of ehca_reg_user_mr */
+ EHCA_MR_PGI_FMR = 3 /* type of ehca_map_phys_fmr */
+};
+
+struct ehca_mr_pginfo {
+ enum ehca_mr_pgi_type type;
+ u64 num_kpages;
+ u64 kpage_cnt;
+ u64 hwpage_size; /* hw page size used for this MR */
+ u64 num_hwpages; /* number of hw pages */
+ u64 hwpage_cnt; /* counter for hw pages */
+ u64 next_hwpage; /* next hw page in buffer/chunk/listelem */
+
+ union {
+ struct { /* type EHCA_MR_PGI_PHYS section */
+ int num_phys_buf;
+ struct ib_phys_buf *phys_buf_array;
+ u64 next_buf;
+ } phy;
+ struct { /* type EHCA_MR_PGI_USER section */
+ struct ib_umem *region;
+ struct scatterlist *next_sg;
+ u64 next_nmap;
+ } usr;
+ struct { /* type EHCA_MR_PGI_FMR section */
+ u64 fmr_pgsize;
+ u64 *page_list;
+ u64 next_listelem;
+ } fmr;
+ } u;
+};
+
+/* output parameters for MR/FMR hipz calls */
+struct ehca_mr_hipzout_parms {
+ struct ipz_mrmw_handle handle;
+ u32 lkey;
+ u32 rkey;
+ u64 len;
+ u64 vaddr;
+ u32 acl;
+};
+
+/* output parameters for MW hipz calls */
+struct ehca_mw_hipzout_parms {
+ struct ipz_mrmw_handle handle;
+ u32 rkey;
+};
+
+struct ehca_av {
+ struct ib_ah ib_ah;
+ struct ehca_ud_av av;
+};
+
+struct ehca_ucontext {
+ struct ib_ucontext ib_ucontext;
+};
+
+int ehca_init_pd_cache(void);
+void ehca_cleanup_pd_cache(void);
+int ehca_init_cq_cache(void);
+void ehca_cleanup_cq_cache(void);
+int ehca_init_qp_cache(void);
+void ehca_cleanup_qp_cache(void);
+int ehca_init_av_cache(void);
+void ehca_cleanup_av_cache(void);
+int ehca_init_mrmw_cache(void);
+void ehca_cleanup_mrmw_cache(void);
+int ehca_init_small_qp_cache(void);
+void ehca_cleanup_small_qp_cache(void);
+
+extern rwlock_t ehca_qp_idr_lock;
+extern rwlock_t ehca_cq_idr_lock;
+extern struct idr ehca_qp_idr;
+extern struct idr ehca_cq_idr;
+extern spinlock_t shca_list_lock;
+
+extern int ehca_static_rate;
+extern int ehca_port_act_time;
+extern bool ehca_use_hp_mr;
+extern bool ehca_scaling_code;
+extern int ehca_lock_hcalls;
+extern int ehca_nr_ports;
+extern int ehca_max_cq;
+extern int ehca_max_qp;
+
+struct ipzu_queue_resp {
+ u32 qe_size; /* queue entry size */
+ u32 act_nr_of_sg;
+ u32 queue_length; /* queue length allocated in bytes */
+ u32 pagesize;
+ u32 toggle_state;
+ u32 offset; /* save offset within a page for small_qp */
+};
+
+struct ehca_create_cq_resp {
+ u32 cq_number;
+ u32 token;
+ struct ipzu_queue_resp ipz_queue;
+ u32 fw_handle_ofs;
+ u32 dummy;
+};
+
+struct ehca_create_qp_resp {
+ u32 qp_num;
+ u32 token;
+ u32 qp_type;
+ u32 ext_type;
+ u32 qkey;
+ /* qp_num assigned by ehca: sqp0/1 may have got different numbers */
+ u32 real_qp_num;
+ u32 fw_handle_ofs;
+ u32 dummy;
+ struct ipzu_queue_resp ipz_squeue;
+ struct ipzu_queue_resp ipz_rqueue;
+};
+
+struct ehca_alloc_cq_parms {
+ u32 nr_cqe;
+ u32 act_nr_of_entries;
+ u32 act_pages;
+ struct ipz_eq_handle eq_handle;
+};
+
+enum ehca_service_type {
+ ST_RC = 0,
+ ST_UC = 1,
+ ST_RD = 2,
+ ST_UD = 3,
+};
+
+enum ehca_ll_comp_flags {
+ LLQP_SEND_COMP = 0x20,
+ LLQP_RECV_COMP = 0x40,
+ LLQP_COMP_MASK = 0x60,
+};
+
+struct ehca_alloc_queue_parms {
+ /* input parameters */
+ int max_wr;
+ int max_sge;
+ int page_size;
+ int is_small;
+
+ /* output parameters */
+ u16 act_nr_wqes;
+ u8 act_nr_sges;
+ u32 queue_size; /* bytes for small queues, pages otherwise */
+};
+
+struct ehca_alloc_qp_parms {
+ struct ehca_alloc_queue_parms squeue;
+ struct ehca_alloc_queue_parms rqueue;
+
+ /* input parameters */
+ enum ehca_service_type servicetype;
+ int qp_storage;
+ int sigtype;
+ enum ehca_ext_qp_type ext_type;
+ enum ehca_ll_comp_flags ll_comp_flags;
+ int ud_av_l_key_ctl;
+
+ u32 token;
+ struct ipz_eq_handle eq_handle;
+ struct ipz_pd pd;
+ struct ipz_cq_handle send_cq_handle, recv_cq_handle;
+
+ u32 srq_qpn, srq_token, srq_limit;
+
+ /* output parameters */
+ u32 real_qp_num;
+ struct ipz_qp_handle qp_handle;
+ struct h_galpas galpas;
+};
+
+int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
+int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int qp_num);
+struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int qp_num);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * pSeries interface definitions
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_CLASSES_PSERIES_H__
+#define __EHCA_CLASSES_PSERIES_H__
+
+#include "hcp_phyp.h"
+#include "ipz_pt_fn.h"
+
+
+struct ehca_pfqp {
+ struct ipz_qpt sqpt;
+ struct ipz_qpt rqpt;
+};
+
+struct ehca_pfcq {
+ struct ipz_qpt qpt;
+ u32 cqnr;
+};
+
+struct ehca_pfeq {
+ struct ipz_qpt qpt;
+ struct h_galpa galpa;
+ u32 eqnr;
+};
+
+struct ipz_adapter_handle {
+ u64 handle;
+};
+
+struct ipz_cq_handle {
+ u64 handle;
+};
+
+struct ipz_eq_handle {
+ u64 handle;
+};
+
+struct ipz_qp_handle {
+ u64 handle;
+};
+struct ipz_mrmw_handle {
+ u64 handle;
+};
+
+struct ipz_pd {
+ u32 value;
+};
+
+struct hcp_modify_qp_control_block {
+ u32 qkey; /* 00 */
+ u32 rdd; /* reliable datagram domain */
+ u32 send_psn; /* 02 */
+ u32 receive_psn; /* 03 */
+ u32 prim_phys_port; /* 04 */
+ u32 alt_phys_port; /* 05 */
+ u32 prim_p_key_idx; /* 06 */
+ u32 alt_p_key_idx; /* 07 */
+ u32 rdma_atomic_ctrl; /* 08 */
+ u32 qp_state; /* 09 */
+ u32 reserved_10; /* 10 */
+ u32 rdma_nr_atomic_resp_res; /* 11 */
+ u32 path_migration_state; /* 12 */
+ u32 rdma_atomic_outst_dest_qp; /* 13 */
+ u32 dest_qp_nr; /* 14 */
+ u32 min_rnr_nak_timer_field; /* 15 */
+ u32 service_level; /* 16 */
+ u32 send_grh_flag; /* 17 */
+ u32 retry_count; /* 18 */
+ u32 timeout; /* 19 */
+ u32 path_mtu; /* 20 */
+ u32 max_static_rate; /* 21 */
+ u32 dlid; /* 22 */
+ u32 rnr_retry_count; /* 23 */
+ u32 source_path_bits; /* 24 */
+ u32 traffic_class; /* 25 */
+ u32 hop_limit; /* 26 */
+ u32 source_gid_idx; /* 27 */
+ u32 flow_label; /* 28 */
+ u32 reserved_29; /* 29 */
+ union { /* 30 */
+ u64 dw[2];
+ u8 byte[16];
+ } dest_gid;
+ u32 service_level_al; /* 34 */
+ u32 send_grh_flag_al; /* 35 */
+ u32 retry_count_al; /* 36 */
+ u32 timeout_al; /* 37 */
+ u32 max_static_rate_al; /* 38 */
+ u32 dlid_al; /* 39 */
+ u32 rnr_retry_count_al; /* 40 */
+ u32 source_path_bits_al; /* 41 */
+ u32 traffic_class_al; /* 42 */
+ u32 hop_limit_al; /* 43 */
+ u32 source_gid_idx_al; /* 44 */
+ u32 flow_label_al; /* 45 */
+ u32 reserved_46; /* 46 */
+ u32 reserved_47; /* 47 */
+ union { /* 48 */
+ u64 dw[2];
+ u8 byte[16];
+ } dest_gid_al;
+ u32 max_nr_outst_send_wr; /* 52 */
+ u32 max_nr_outst_recv_wr; /* 53 */
+ u32 disable_ete_credit_check; /* 54 */
+ u32 qp_number; /* 55 */
+ u64 send_queue_handle; /* 56 */
+ u64 recv_queue_handle; /* 58 */
+ u32 actual_nr_sges_in_sq_wqe; /* 60 */
+ u32 actual_nr_sges_in_rq_wqe; /* 61 */
+ u32 qp_enable; /* 62 */
+ u32 curr_srq_limit; /* 63 */
+ u64 qp_aff_asyn_ev_log_reg; /* 64 */
+ u64 shared_rq_hndl; /* 66 */
+ u64 trigg_doorbell_qp_hndl; /* 68 */
+ u32 reserved_70_127[58]; /* 70 */
+};
+
+#define MQPCB_MASK_QKEY EHCA_BMASK_IBM( 0, 0)
+#define MQPCB_MASK_SEND_PSN EHCA_BMASK_IBM( 2, 2)
+#define MQPCB_MASK_RECEIVE_PSN EHCA_BMASK_IBM( 3, 3)
+#define MQPCB_MASK_PRIM_PHYS_PORT EHCA_BMASK_IBM( 4, 4)
+#define MQPCB_PRIM_PHYS_PORT EHCA_BMASK_IBM(24, 31)
+#define MQPCB_MASK_ALT_PHYS_PORT EHCA_BMASK_IBM( 5, 5)
+#define MQPCB_MASK_PRIM_P_KEY_IDX EHCA_BMASK_IBM( 6, 6)
+#define MQPCB_PRIM_P_KEY_IDX EHCA_BMASK_IBM(24, 31)
+#define MQPCB_MASK_ALT_P_KEY_IDX EHCA_BMASK_IBM( 7, 7)
+#define MQPCB_MASK_RDMA_ATOMIC_CTRL EHCA_BMASK_IBM( 8, 8)
+#define MQPCB_MASK_QP_STATE EHCA_BMASK_IBM( 9, 9)
+#define MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES EHCA_BMASK_IBM(11, 11)
+#define MQPCB_MASK_PATH_MIGRATION_STATE EHCA_BMASK_IBM(12, 12)
+#define MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP EHCA_BMASK_IBM(13, 13)
+#define MQPCB_MASK_DEST_QP_NR EHCA_BMASK_IBM(14, 14)
+#define MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD EHCA_BMASK_IBM(15, 15)
+#define MQPCB_MASK_SERVICE_LEVEL EHCA_BMASK_IBM(16, 16)
+#define MQPCB_MASK_SEND_GRH_FLAG EHCA_BMASK_IBM(17, 17)
+#define MQPCB_MASK_RETRY_COUNT EHCA_BMASK_IBM(18, 18)
+#define MQPCB_MASK_TIMEOUT EHCA_BMASK_IBM(19, 19)
+#define MQPCB_MASK_PATH_MTU EHCA_BMASK_IBM(20, 20)
+#define MQPCB_MASK_MAX_STATIC_RATE EHCA_BMASK_IBM(21, 21)
+#define MQPCB_MASK_DLID EHCA_BMASK_IBM(22, 22)
+#define MQPCB_MASK_RNR_RETRY_COUNT EHCA_BMASK_IBM(23, 23)
+#define MQPCB_MASK_SOURCE_PATH_BITS EHCA_BMASK_IBM(24, 24)
+#define MQPCB_MASK_TRAFFIC_CLASS EHCA_BMASK_IBM(25, 25)
+#define MQPCB_MASK_HOP_LIMIT EHCA_BMASK_IBM(26, 26)
+#define MQPCB_MASK_SOURCE_GID_IDX EHCA_BMASK_IBM(27, 27)
+#define MQPCB_MASK_FLOW_LABEL EHCA_BMASK_IBM(28, 28)
+#define MQPCB_MASK_DEST_GID EHCA_BMASK_IBM(30, 30)
+#define MQPCB_MASK_SERVICE_LEVEL_AL EHCA_BMASK_IBM(31, 31)
+#define MQPCB_MASK_SEND_GRH_FLAG_AL EHCA_BMASK_IBM(32, 32)
+#define MQPCB_MASK_RETRY_COUNT_AL EHCA_BMASK_IBM(33, 33)
+#define MQPCB_MASK_TIMEOUT_AL EHCA_BMASK_IBM(34, 34)
+#define MQPCB_MASK_MAX_STATIC_RATE_AL EHCA_BMASK_IBM(35, 35)
+#define MQPCB_MASK_DLID_AL EHCA_BMASK_IBM(36, 36)
+#define MQPCB_MASK_RNR_RETRY_COUNT_AL EHCA_BMASK_IBM(37, 37)
+#define MQPCB_MASK_SOURCE_PATH_BITS_AL EHCA_BMASK_IBM(38, 38)
+#define MQPCB_MASK_TRAFFIC_CLASS_AL EHCA_BMASK_IBM(39, 39)
+#define MQPCB_MASK_HOP_LIMIT_AL EHCA_BMASK_IBM(40, 40)
+#define MQPCB_MASK_SOURCE_GID_IDX_AL EHCA_BMASK_IBM(41, 41)
+#define MQPCB_MASK_FLOW_LABEL_AL EHCA_BMASK_IBM(42, 42)
+#define MQPCB_MASK_DEST_GID_AL EHCA_BMASK_IBM(44, 44)
+#define MQPCB_MASK_MAX_NR_OUTST_SEND_WR EHCA_BMASK_IBM(45, 45)
+#define MQPCB_MASK_MAX_NR_OUTST_RECV_WR EHCA_BMASK_IBM(46, 46)
+#define MQPCB_MASK_DISABLE_ETE_CREDIT_CHECK EHCA_BMASK_IBM(47, 47)
+#define MQPCB_MASK_QP_ENABLE EHCA_BMASK_IBM(48, 48)
+#define MQPCB_MASK_CURR_SRQ_LIMIT EHCA_BMASK_IBM(49, 49)
+#define MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG EHCA_BMASK_IBM(50, 50)
+#define MQPCB_MASK_SHARED_RQ_HNDL EHCA_BMASK_IBM(51, 51)
+
+#endif /* __EHCA_CLASSES_PSERIES_H__ */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Completion queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_iverbs.h"
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "hcp_if.h"
+
+static struct kmem_cache *cq_cache;
+
+int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
+{
+ unsigned int qp_num = qp->real_qp_num;
+ unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->spinlock, flags);
+ hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
+ spin_unlock_irqrestore(&cq->spinlock, flags);
+
+ ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
+ cq->cq_number, qp_num);
+
+ return 0;
+}
+
+int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
+{
+ int ret = -EINVAL;
+ unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
+ struct hlist_node *iter;
+ struct ehca_qp *qp;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->spinlock, flags);
+ hlist_for_each(iter, &cq->qp_hashtab[key]) {
+ qp = hlist_entry(iter, struct ehca_qp, list_entries);
+ if (qp->real_qp_num == real_qp_num) {
+ hlist_del(iter);
+ ehca_dbg(cq->ib_cq.device,
+ "removed qp from cq .cq_num=%x real_qp_num=%x",
+ cq->cq_number, real_qp_num);
+ ret = 0;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&cq->spinlock, flags);
+ if (ret)
+ ehca_err(cq->ib_cq.device,
+ "qp not found cq_num=%x real_qp_num=%x",
+ cq->cq_number, real_qp_num);
+
+ return ret;
+}
+
+struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
+{
+ struct ehca_qp *ret = NULL;
+ unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
+ struct hlist_node *iter;
+ struct ehca_qp *qp;
+ hlist_for_each(iter, &cq->qp_hashtab[key]) {
+ qp = hlist_entry(iter, struct ehca_qp, list_entries);
+ if (qp->real_qp_num == real_qp_num) {
+ ret = qp;
+ break;
+ }
+ }
+ return ret;
+}
+
+struct ib_cq *ehca_create_cq(struct ib_device *device,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *context,
+ struct ib_udata *udata)
+{
+ int cqe = attr->cqe;
+ static const u32 additional_cqe = 20;
+ struct ib_cq *cq;
+ struct ehca_cq *my_cq;
+ struct ehca_shca *shca =
+ container_of(device, struct ehca_shca, ib_device);
+ struct ipz_adapter_handle adapter_handle;
+ struct ehca_alloc_cq_parms param; /* h_call's out parameters */
+ struct h_galpa gal;
+ void *vpage;
+ u32 counter;
+ u64 rpage, cqx_fec, h_ret;
+ int ipz_rc, i;
+ unsigned long flags;
+
+ if (attr->flags)
+ return ERR_PTR(-EINVAL);
+
+ if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
+ return ERR_PTR(-EINVAL);
+
+ if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
+ ehca_err(device, "Unable to create CQ, max number of %i "
+ "CQs reached.", shca->max_num_cqs);
+ ehca_err(device, "To increase the maximum number of CQs "
+ "use the number_of_cqs module parameter.\n");
+ return ERR_PTR(-ENOSPC);
+ }
+
+ my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
+ if (!my_cq) {
+ ehca_err(device, "Out of memory for ehca_cq struct device=%p",
+ device);
+ atomic_dec(&shca->num_cqs);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms));
+
+ spin_lock_init(&my_cq->spinlock);
+ spin_lock_init(&my_cq->cb_lock);
+ spin_lock_init(&my_cq->task_lock);
+ atomic_set(&my_cq->nr_events, 0);
+ init_waitqueue_head(&my_cq->wait_completion);
+
+ cq = &my_cq->ib_cq;
+
+ adapter_handle = shca->ipz_hca_handle;
+ param.eq_handle = shca->eq.ipz_eq_handle;
+
+ idr_preload(GFP_KERNEL);
+ write_lock_irqsave(&ehca_cq_idr_lock, flags);
+ my_cq->token = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT);
+ write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+ idr_preload_end();
+
+ if (my_cq->token < 0) {
+ cq = ERR_PTR(-ENOMEM);
+ ehca_err(device, "Can't allocate new idr entry. device=%p",
+ device);
+ goto create_cq_exit1;
+ }
+
+ /*
+ * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
+ * for receiving errors CQEs.
+ */
+ param.nr_cqe = cqe + additional_cqe;
+ h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m);
+
+ if (h_ret != H_SUCCESS) {
+ ehca_err(device, "hipz_h_alloc_resource_cq() failed "
+ "h_ret=%lli device=%p", h_ret, device);
+ cq = ERR_PTR(ehca2ib_return_code(h_ret));
+ goto create_cq_exit2;
+ }
+
+ ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
+ EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
+ if (!ipz_rc) {
+ ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
+ ipz_rc, device);
+ cq = ERR_PTR(-EINVAL);
+ goto create_cq_exit3;
+ }
+
+ for (counter = 0; counter < param.act_pages; counter++) {
+ vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
+ if (!vpage) {
+ ehca_err(device, "ipz_qpageit_get_inc() "
+ "returns NULL device=%p", device);
+ cq = ERR_PTR(-EAGAIN);
+ goto create_cq_exit4;
+ }
+ rpage = __pa(vpage);
+
+ h_ret = hipz_h_register_rpage_cq(adapter_handle,
+ my_cq->ipz_cq_handle,
+ &my_cq->pf,
+ 0,
+ 0,
+ rpage,
+ 1,
+ my_cq->galpas.
+ kernel);
+
+ if (h_ret < H_SUCCESS) {
+ ehca_err(device, "hipz_h_register_rpage_cq() failed "
+ "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i "
+ "act_pages=%i", my_cq, my_cq->cq_number,
+ h_ret, counter, param.act_pages);
+ cq = ERR_PTR(-EINVAL);
+ goto create_cq_exit4;
+ }
+
+ if (counter == (param.act_pages - 1)) {
+ vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
+ if ((h_ret != H_SUCCESS) || vpage) {
+ ehca_err(device, "Registration of pages not "
+ "complete ehca_cq=%p cq_num=%x "
+ "h_ret=%lli", my_cq, my_cq->cq_number,
+ h_ret);
+ cq = ERR_PTR(-EAGAIN);
+ goto create_cq_exit4;
+ }
+ } else {
+ if (h_ret != H_PAGE_REGISTERED) {
+ ehca_err(device, "Registration of page failed "
+ "ehca_cq=%p cq_num=%x h_ret=%lli "
+ "counter=%i act_pages=%i",
+ my_cq, my_cq->cq_number,
+ h_ret, counter, param.act_pages);
+ cq = ERR_PTR(-ENOMEM);
+ goto create_cq_exit4;
+ }
+ }
+ }
+
+ ipz_qeit_reset(&my_cq->ipz_queue);
+
+ gal = my_cq->galpas.kernel;
+ cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
+ ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx",
+ my_cq, my_cq->cq_number, cqx_fec);
+
+ my_cq->ib_cq.cqe = my_cq->nr_of_entries =
+ param.act_nr_of_entries - additional_cqe;
+ my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
+
+ for (i = 0; i < QP_HASHTAB_LEN; i++)
+ INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
+
+ INIT_LIST_HEAD(&my_cq->sqp_err_list);
+ INIT_LIST_HEAD(&my_cq->rqp_err_list);
+
+ if (context) {
+ struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
+ struct ehca_create_cq_resp resp;
+ memset(&resp, 0, sizeof(resp));
+ resp.cq_number = my_cq->cq_number;
+ resp.token = my_cq->token;
+ resp.ipz_queue.qe_size = ipz_queue->qe_size;
+ resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
+ resp.ipz_queue.queue_length = ipz_queue->queue_length;
+ resp.ipz_queue.pagesize = ipz_queue->pagesize;
+ resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
+ resp.fw_handle_ofs = (u32)
+ (my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
+ if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
+ ehca_err(device, "Copy to udata failed.");
+ cq = ERR_PTR(-EFAULT);
+ goto create_cq_exit4;
+ }
+ }
+
+ return cq;
+
+create_cq_exit4:
+ ipz_queue_dtor(NULL, &my_cq->ipz_queue);
+
+create_cq_exit3:
+ h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
+ if (h_ret != H_SUCCESS)
+ ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
+ "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret);
+
+create_cq_exit2:
+ write_lock_irqsave(&ehca_cq_idr_lock, flags);
+ idr_remove(&ehca_cq_idr, my_cq->token);
+ write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+create_cq_exit1:
+ kmem_cache_free(cq_cache, my_cq);
+
+ atomic_dec(&shca->num_cqs);
+ return cq;
+}
+
+int ehca_destroy_cq(struct ib_cq *cq)
+{
+ u64 h_ret;
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ int cq_num = my_cq->cq_number;
+ struct ib_device *device = cq->device;
+ struct ehca_shca *shca = container_of(device, struct ehca_shca,
+ ib_device);
+ struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+ unsigned long flags;
+
+ if (cq->uobject) {
+ if (my_cq->mm_count_galpa || my_cq->mm_count_queue) {
+ ehca_err(device, "Resources still referenced in "
+ "user space cq_num=%x", my_cq->cq_number);
+ return -EINVAL;
+ }
+ }
+
+ /*
+ * remove the CQ from the idr first to make sure
+ * no more interrupt tasklets will touch this CQ
+ */
+ write_lock_irqsave(&ehca_cq_idr_lock, flags);
+ idr_remove(&ehca_cq_idr, my_cq->token);
+ write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+ /* now wait until all pending events have completed */
+ wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
+
+ /* nobody's using our CQ any longer -- we can destroy it */
+ h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
+ if (h_ret == H_R_STATE) {
+ /* cq in err: read err data and destroy it forcibly */
+ ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err "
+ "state. Try to delete it forcibly.",
+ my_cq, cq_num, my_cq->ipz_cq_handle.handle);
+ ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
+ h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
+ if (h_ret == H_SUCCESS)
+ ehca_dbg(device, "cq_num=%x deleted successfully.",
+ cq_num);
+ }
+ if (h_ret != H_SUCCESS) {
+ ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli "
+ "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
+ return ehca2ib_return_code(h_ret);
+ }
+ ipz_queue_dtor(NULL, &my_cq->ipz_queue);
+ kmem_cache_free(cq_cache, my_cq);
+
+ atomic_dec(&shca->num_cqs);
+ return 0;
+}
+
+int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
+{
+ /* TODO: proper resize needs to be done */
+ ehca_err(cq->device, "not implemented yet");
+
+ return -EFAULT;
+}
+
+int ehca_init_cq_cache(void)
+{
+ cq_cache = kmem_cache_create("ehca_cache_cq",
+ sizeof(struct ehca_cq), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!cq_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_cq_cache(void)
+{
+ if (cq_cache)
+ kmem_cache_destroy(cq_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Event queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "ehca_iverbs.h"
+#include "ehca_qes.h"
+#include "hcp_if.h"
+#include "ipz_pt_fn.h"
+
+int ehca_create_eq(struct ehca_shca *shca,
+ struct ehca_eq *eq,
+ const enum ehca_eq_type type, const u32 length)
+{
+ int ret;
+ u64 h_ret;
+ u32 nr_pages;
+ u32 i;
+ void *vpage;
+ struct ib_device *ib_dev = &shca->ib_device;
+
+ spin_lock_init(&eq->spinlock);
+ spin_lock_init(&eq->irq_spinlock);
+ eq->is_initialized = 0;
+
+ if (type != EHCA_EQ && type != EHCA_NEQ) {
+ ehca_err(ib_dev, "Invalid EQ type %x. eq=%p", type, eq);
+ return -EINVAL;
+ }
+ if (!length) {
+ ehca_err(ib_dev, "EQ length must not be zero. eq=%p", eq);
+ return -EINVAL;
+ }
+
+ h_ret = hipz_h_alloc_resource_eq(shca->ipz_hca_handle,
+ &eq->pf,
+ type,
+ length,
+ &eq->ipz_eq_handle,
+ &eq->length,
+ &nr_pages, &eq->ist);
+
+ if (h_ret != H_SUCCESS) {
+ ehca_err(ib_dev, "Can't allocate EQ/NEQ. eq=%p", eq);
+ return -EINVAL;
+ }
+
+ ret = ipz_queue_ctor(NULL, &eq->ipz_queue, nr_pages,
+ EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0, 0);
+ if (!ret) {
+ ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq);
+ goto create_eq_exit1;
+ }
+
+ for (i = 0; i < nr_pages; i++) {
+ u64 rpage;
+
+ vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
+ if (!vpage)
+ goto create_eq_exit2;
+
+ rpage = __pa(vpage);
+ h_ret = hipz_h_register_rpage_eq(shca->ipz_hca_handle,
+ eq->ipz_eq_handle,
+ &eq->pf,
+ 0, 0, rpage, 1);
+
+ if (i == (nr_pages - 1)) {
+ /* last page */
+ vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
+ if (h_ret != H_SUCCESS || vpage)
+ goto create_eq_exit2;
+ } else {
+ if (h_ret != H_PAGE_REGISTERED)
+ goto create_eq_exit2;
+ }
+ }
+
+ ipz_qeit_reset(&eq->ipz_queue);
+
+ /* register interrupt handlers and initialize work queues */
+ if (type == EHCA_EQ) {
+ tasklet_init(&eq->interrupt_task, ehca_tasklet_eq, (long)shca);
+
+ ret = ibmebus_request_irq(eq->ist, ehca_interrupt_eq,
+ 0, "ehca_eq",
+ (void *)shca);
+ if (ret < 0)
+ ehca_err(ib_dev, "Can't map interrupt handler.");
+ } else if (type == EHCA_NEQ) {
+ tasklet_init(&eq->interrupt_task, ehca_tasklet_neq, (long)shca);
+
+ ret = ibmebus_request_irq(eq->ist, ehca_interrupt_neq,
+ 0, "ehca_neq",
+ (void *)shca);
+ if (ret < 0)
+ ehca_err(ib_dev, "Can't map interrupt handler.");
+ }
+
+ eq->is_initialized = 1;
+
+ return 0;
+
+create_eq_exit2:
+ ipz_queue_dtor(NULL, &eq->ipz_queue);
+
+create_eq_exit1:
+ hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
+
+ return -EINVAL;
+}
+
+void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq)
+{
+ unsigned long flags;
+ void *eqe;
+
+ spin_lock_irqsave(&eq->spinlock, flags);
+ eqe = ipz_eqit_eq_get_inc_valid(&eq->ipz_queue);
+ spin_unlock_irqrestore(&eq->spinlock, flags);
+
+ return eqe;
+}
+
+int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq)
+{
+ unsigned long flags;
+ u64 h_ret;
+
+ ibmebus_free_irq(eq->ist, (void *)shca);
+
+ spin_lock_irqsave(&shca_list_lock, flags);
+ eq->is_initialized = 0;
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+
+ tasklet_kill(&eq->interrupt_task);
+
+ h_ret = hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
+
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't free EQ resources.");
+ return -EINVAL;
+ }
+ ipz_queue_dtor(NULL, &eq->ipz_queue);
+
+ return 0;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * HCA query functions
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/gfp.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+static unsigned int limit_uint(unsigned int value)
+{
+ return min_t(unsigned int, value, INT_MAX);
+}
+
+int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
+ struct ib_udata *uhw)
+{
+ int i, ret = 0;
+ struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+ ib_device);
+ struct hipz_query_hca *rblock;
+
+ static const u32 cap_mapping[] = {
+ IB_DEVICE_RESIZE_MAX_WR, HCA_CAP_WQE_RESIZE,
+ IB_DEVICE_BAD_PKEY_CNTR, HCA_CAP_BAD_P_KEY_CTR,
+ IB_DEVICE_BAD_QKEY_CNTR, HCA_CAP_Q_KEY_VIOL_CTR,
+ IB_DEVICE_RAW_MULTI, HCA_CAP_RAW_PACKET_MCAST,
+ IB_DEVICE_AUTO_PATH_MIG, HCA_CAP_AUTO_PATH_MIG,
+ IB_DEVICE_CHANGE_PHY_PORT, HCA_CAP_SQD_RTS_PORT_CHANGE,
+ IB_DEVICE_UD_AV_PORT_ENFORCE, HCA_CAP_AH_PORT_NR_CHECK,
+ IB_DEVICE_CURR_QP_STATE_MOD, HCA_CAP_CUR_QP_STATE_MOD,
+ IB_DEVICE_SHUTDOWN_PORT, HCA_CAP_SHUTDOWN_PORT,
+ IB_DEVICE_INIT_TYPE, HCA_CAP_INIT_TYPE,
+ IB_DEVICE_PORT_ACTIVE_EVENT, HCA_CAP_PORT_ACTIVE_EVENT,
+ };
+
+ if (uhw->inlen || uhw->outlen)
+ return -EINVAL;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query device properties");
+ ret = -EINVAL;
+ goto query_device1;
+ }
+
+ memset(props, 0, sizeof(struct ib_device_attr));
+ props->page_size_cap = shca->hca_cap_mr_pgsize;
+ props->fw_ver = rblock->hw_ver;
+ props->max_mr_size = rblock->max_mr_size;
+ props->vendor_id = rblock->vendor_id >> 8;
+ props->vendor_part_id = rblock->vendor_part_id >> 16;
+ props->hw_ver = rblock->hw_ver;
+ props->max_qp = limit_uint(rblock->max_qp);
+ props->max_qp_wr = limit_uint(rblock->max_wqes_wq);
+ props->max_sge = limit_uint(rblock->max_sge);
+ props->max_sge_rd = limit_uint(rblock->max_sge_rd);
+ props->max_cq = limit_uint(rblock->max_cq);
+ props->max_cqe = limit_uint(rblock->max_cqe);
+ props->max_mr = limit_uint(rblock->max_mr);
+ props->max_mw = limit_uint(rblock->max_mw);
+ props->max_pd = limit_uint(rblock->max_pd);
+ props->max_ah = limit_uint(rblock->max_ah);
+ props->max_ee = limit_uint(rblock->max_rd_ee_context);
+ props->max_rdd = limit_uint(rblock->max_rd_domain);
+ props->max_fmr = limit_uint(rblock->max_mr);
+ props->max_qp_rd_atom = limit_uint(rblock->max_rr_qp);
+ props->max_ee_rd_atom = limit_uint(rblock->max_rr_ee_context);
+ props->max_res_rd_atom = limit_uint(rblock->max_rr_hca);
+ props->max_qp_init_rd_atom = limit_uint(rblock->max_act_wqs_qp);
+ props->max_ee_init_rd_atom = limit_uint(rblock->max_act_wqs_ee_context);
+
+ if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
+ props->max_srq = limit_uint(props->max_qp);
+ props->max_srq_wr = limit_uint(props->max_qp_wr);
+ props->max_srq_sge = 3;
+ }
+
+ props->max_pkeys = 16;
+ /* Some FW versions say 0 here; insert sensible value in that case */
+ props->local_ca_ack_delay = rblock->local_ca_ack_delay ?
+ min_t(u8, rblock->local_ca_ack_delay, 255) : 12;
+ props->max_raw_ipv6_qp = limit_uint(rblock->max_raw_ipv6_qp);
+ props->max_raw_ethy_qp = limit_uint(rblock->max_raw_ethy_qp);
+ props->max_mcast_grp = limit_uint(rblock->max_mcast_grp);
+ props->max_mcast_qp_attach = limit_uint(rblock->max_mcast_qp_attach);
+ props->max_total_mcast_qp_attach
+ = limit_uint(rblock->max_total_mcast_qp_attach);
+
+ /* translate device capabilities */
+ props->device_cap_flags = IB_DEVICE_SYS_IMAGE_GUID |
+ IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_N_NOTIFY_CQ;
+ for (i = 0; i < ARRAY_SIZE(cap_mapping); i += 2)
+ if (rblock->hca_cap_indicators & cap_mapping[i + 1])
+ props->device_cap_flags |= cap_mapping[i];
+
+query_device1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+static enum ib_mtu map_mtu(struct ehca_shca *shca, u32 fw_mtu)
+{
+ switch (fw_mtu) {
+ case 0x1:
+ return IB_MTU_256;
+ case 0x2:
+ return IB_MTU_512;
+ case 0x3:
+ return IB_MTU_1024;
+ case 0x4:
+ return IB_MTU_2048;
+ case 0x5:
+ return IB_MTU_4096;
+ default:
+ ehca_err(&shca->ib_device, "Unknown MTU size: %x.",
+ fw_mtu);
+ return 0;
+ }
+}
+
+static u8 map_number_of_vls(struct ehca_shca *shca, u32 vl_cap)
+{
+ switch (vl_cap) {
+ case 0x1:
+ return 1;
+ case 0x2:
+ return 2;
+ case 0x3:
+ return 4;
+ case 0x4:
+ return 8;
+ case 0x5:
+ return 15;
+ default:
+ ehca_err(&shca->ib_device, "invalid Vl Capability: %x.",
+ vl_cap);
+ return 0;
+ }
+}
+
+int ehca_query_port(struct ib_device *ibdev,
+ u8 port, struct ib_port_attr *props)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+ ib_device);
+ struct hipz_query_port *rblock;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_port1;
+ }
+
+ memset(props, 0, sizeof(struct ib_port_attr));
+
+ props->active_mtu = props->max_mtu = map_mtu(shca, rblock->max_mtu);
+ props->port_cap_flags = rblock->capability_mask;
+ props->gid_tbl_len = rblock->gid_tbl_len;
+ if (rblock->max_msg_sz)
+ props->max_msg_sz = rblock->max_msg_sz;
+ else
+ props->max_msg_sz = 0x1 << 31;
+ props->bad_pkey_cntr = rblock->bad_pkey_cntr;
+ props->qkey_viol_cntr = rblock->qkey_viol_cntr;
+ props->pkey_tbl_len = rblock->pkey_tbl_len;
+ props->lid = rblock->lid;
+ props->sm_lid = rblock->sm_lid;
+ props->lmc = rblock->lmc;
+ props->sm_sl = rblock->sm_sl;
+ props->subnet_timeout = rblock->subnet_timeout;
+ props->init_type_reply = rblock->init_type_reply;
+ props->max_vl_num = map_number_of_vls(shca, rblock->vl_cap);
+
+ if (rblock->state && rblock->phys_width) {
+ props->phys_state = rblock->phys_pstate;
+ props->state = rblock->phys_state;
+ props->active_width = rblock->phys_width;
+ props->active_speed = rblock->phys_speed;
+ } else {
+ /* old firmware releases don't report physical
+ * port info, so use default values
+ */
+ props->phys_state = 5;
+ props->state = rblock->state;
+ props->active_width = IB_WIDTH_12X;
+ props->active_speed = IB_SPEED_SDR;
+ }
+
+query_port1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+int ehca_query_sma_attr(struct ehca_shca *shca,
+ u8 port, struct ehca_sma_attr *attr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct hipz_query_port *rblock;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_sma_attr1;
+ }
+
+ memset(attr, 0, sizeof(struct ehca_sma_attr));
+
+ attr->lid = rblock->lid;
+ attr->lmc = rblock->lmc;
+ attr->sm_sl = rblock->sm_sl;
+ attr->sm_lid = rblock->sm_lid;
+
+ attr->pkey_tbl_len = rblock->pkey_tbl_len;
+ memcpy(attr->pkeys, rblock->pkey_entries, sizeof(attr->pkeys));
+
+query_sma_attr1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca;
+ struct hipz_query_port *rblock;
+
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+ if (index > 16) {
+ ehca_err(&shca->ib_device, "Invalid index: %x.", index);
+ return -EINVAL;
+ }
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_pkey1;
+ }
+
+ memcpy(pkey, &rblock->pkey_entries + index, sizeof(u16));
+
+query_pkey1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+int ehca_query_gid(struct ib_device *ibdev, u8 port,
+ int index, union ib_gid *gid)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+ ib_device);
+ struct hipz_query_port *rblock;
+
+ if (index < 0 || index > 255) {
+ ehca_err(&shca->ib_device, "Invalid index: %x.", index);
+ return -EINVAL;
+ }
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_gid1;
+ }
+
+ memcpy(&gid->raw[0], &rblock->gid_prefix, sizeof(u64));
+ memcpy(&gid->raw[8], &rblock->guid_entries[index], sizeof(u64));
+
+query_gid1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+static const u32 allowed_port_caps = (
+ IB_PORT_SM | IB_PORT_LED_INFO_SUP | IB_PORT_CM_SUP |
+ IB_PORT_SNMP_TUNNEL_SUP | IB_PORT_DEVICE_MGMT_SUP |
+ IB_PORT_VENDOR_CLASS_SUP);
+
+int ehca_modify_port(struct ib_device *ibdev,
+ u8 port, int port_modify_mask,
+ struct ib_port_modify *props)
+{
+ int ret = 0;
+ struct ehca_shca *shca;
+ struct hipz_query_port *rblock;
+ u32 cap;
+ u64 hret;
+
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+ if ((props->set_port_cap_mask | props->clr_port_cap_mask)
+ & ~allowed_port_caps) {
+ ehca_err(&shca->ib_device, "Non-changeable bits set in masks "
+ "set=%x clr=%x allowed=%x", props->set_port_cap_mask,
+ props->clr_port_cap_mask, allowed_port_caps);
+ return -EINVAL;
+ }
+
+ if (mutex_lock_interruptible(&shca->modify_mutex))
+ return -ERESTARTSYS;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ ret = -ENOMEM;
+ goto modify_port1;
+ }
+
+ hret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (hret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto modify_port2;
+ }
+
+ cap = (rblock->capability_mask | props->set_port_cap_mask)
+ & ~props->clr_port_cap_mask;
+
+ hret = hipz_h_modify_port(shca->ipz_hca_handle, port,
+ cap, props->init_type, port_modify_mask);
+ if (hret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Modify port failed h_ret=%lli",
+ hret);
+ ret = -EINVAL;
+ }
+
+modify_port2:
+ ehca_free_fw_ctrlblock(rblock);
+
+modify_port1:
+ mutex_unlock(&shca->modify_mutex);
+
+ return ret;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Functions for EQs, NEQs and interrupts
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+#include <linux/smpboot.h>
+
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "ehca_iverbs.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+#include "ipz_pt_fn.h"
+
+#define EQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
+#define EQE_CQ_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
+#define EQE_EE_IDENTIFIER EHCA_BMASK_IBM( 2, 7)
+#define EQE_CQ_NUMBER EHCA_BMASK_IBM( 8, 31)
+#define EQE_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
+#define EQE_QP_TOKEN EHCA_BMASK_IBM(32, 63)
+#define EQE_CQ_TOKEN EHCA_BMASK_IBM(32, 63)
+
+#define NEQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
+#define NEQE_EVENT_CODE EHCA_BMASK_IBM( 2, 7)
+#define NEQE_PORT_NUMBER EHCA_BMASK_IBM( 8, 15)
+#define NEQE_PORT_AVAILABILITY EHCA_BMASK_IBM(16, 16)
+#define NEQE_DISRUPTIVE EHCA_BMASK_IBM(16, 16)
+#define NEQE_SPECIFIC_EVENT EHCA_BMASK_IBM(16, 23)
+
+#define ERROR_DATA_LENGTH EHCA_BMASK_IBM(52, 63)
+#define ERROR_DATA_TYPE EHCA_BMASK_IBM( 0, 7)
+
+static void queue_comp_task(struct ehca_cq *__cq);
+
+static struct ehca_comp_pool *pool;
+
+static inline void comp_event_callback(struct ehca_cq *cq)
+{
+ if (!cq->ib_cq.comp_handler)
+ return;
+
+ spin_lock(&cq->cb_lock);
+ cq->ib_cq.comp_handler(&cq->ib_cq, cq->ib_cq.cq_context);
+ spin_unlock(&cq->cb_lock);
+
+ return;
+}
+
+static void print_error_data(struct ehca_shca *shca, void *data,
+ u64 *rblock, int length)
+{
+ u64 type = EHCA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]);
+ u64 resource = rblock[1];
+
+ switch (type) {
+ case 0x1: /* Queue Pair */
+ {
+ struct ehca_qp *qp = (struct ehca_qp *)data;
+
+ /* only print error data if AER is set */
+ if (rblock[6] == 0)
+ return;
+
+ ehca_err(&shca->ib_device,
+ "QP 0x%x (resource=%llx) has errors.",
+ qp->ib_qp.qp_num, resource);
+ break;
+ }
+ case 0x4: /* Completion Queue */
+ {
+ struct ehca_cq *cq = (struct ehca_cq *)data;
+
+ ehca_err(&shca->ib_device,
+ "CQ 0x%x (resource=%llx) has errors.",
+ cq->cq_number, resource);
+ break;
+ }
+ default:
+ ehca_err(&shca->ib_device,
+ "Unknown error type: %llx on %s.",
+ type, shca->ib_device.name);
+ break;
+ }
+
+ ehca_err(&shca->ib_device, "Error data is available: %llx.", resource);
+ ehca_err(&shca->ib_device, "EHCA ----- error data begin "
+ "---------------------------------------------------");
+ ehca_dmp(rblock, length, "resource=%llx", resource);
+ ehca_err(&shca->ib_device, "EHCA ----- error data end "
+ "----------------------------------------------------");
+
+ return;
+}
+
+int ehca_error_data(struct ehca_shca *shca, void *data,
+ u64 resource)
+{
+
+ unsigned long ret;
+ u64 *rblock;
+ unsigned long block_count;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Cannot allocate rblock memory.");
+ ret = -ENOMEM;
+ goto error_data1;
+ }
+
+ /* rblock must be 4K aligned and should be 4K large */
+ ret = hipz_h_error_data(shca->ipz_hca_handle,
+ resource,
+ rblock,
+ &block_count);
+
+ if (ret == H_R_STATE)
+ ehca_err(&shca->ib_device,
+ "No error data is available: %llx.", resource);
+ else if (ret == H_SUCCESS) {
+ int length;
+
+ length = EHCA_BMASK_GET(ERROR_DATA_LENGTH, rblock[0]);
+
+ if (length > EHCA_PAGESIZE)
+ length = EHCA_PAGESIZE;
+
+ print_error_data(shca, data, rblock, length);
+ } else
+ ehca_err(&shca->ib_device,
+ "Error data could not be fetched: %llx", resource);
+
+ ehca_free_fw_ctrlblock(rblock);
+
+error_data1:
+ return ret;
+
+}
+
+static void dispatch_qp_event(struct ehca_shca *shca, struct ehca_qp *qp,
+ enum ib_event_type event_type)
+{
+ struct ib_event event;
+
+ /* PATH_MIG without the QP ever having been armed is false alarm */
+ if (event_type == IB_EVENT_PATH_MIG && !qp->mig_armed)
+ return;
+
+ event.device = &shca->ib_device;
+ event.event = event_type;
+
+ if (qp->ext_type == EQPT_SRQ) {
+ if (!qp->ib_srq.event_handler)
+ return;
+
+ event.element.srq = &qp->ib_srq;
+ qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context);
+ } else {
+ if (!qp->ib_qp.event_handler)
+ return;
+
+ event.element.qp = &qp->ib_qp;
+ qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+ }
+}
+
+static void qp_event_callback(struct ehca_shca *shca, u64 eqe,
+ enum ib_event_type event_type, int fatal)
+{
+ struct ehca_qp *qp;
+ u32 token = EHCA_BMASK_GET(EQE_QP_TOKEN, eqe);
+
+ read_lock(&ehca_qp_idr_lock);
+ qp = idr_find(&ehca_qp_idr, token);
+ if (qp)
+ atomic_inc(&qp->nr_events);
+ read_unlock(&ehca_qp_idr_lock);
+
+ if (!qp)
+ return;
+
+ if (fatal)
+ ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
+
+ dispatch_qp_event(shca, qp, fatal && qp->ext_type == EQPT_SRQ ?
+ IB_EVENT_SRQ_ERR : event_type);
+
+ /*
+ * eHCA only processes one WQE at a time for SRQ base QPs,
+ * so the last WQE has been processed as soon as the QP enters
+ * error state.
+ */
+ if (fatal && qp->ext_type == EQPT_SRQBASE)
+ dispatch_qp_event(shca, qp, IB_EVENT_QP_LAST_WQE_REACHED);
+
+ if (atomic_dec_and_test(&qp->nr_events))
+ wake_up(&qp->wait_completion);
+ return;
+}
+
+static void cq_event_callback(struct ehca_shca *shca,
+ u64 eqe)
+{
+ struct ehca_cq *cq;
+ u32 token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe);
+
+ read_lock(&ehca_cq_idr_lock);
+ cq = idr_find(&ehca_cq_idr, token);
+ if (cq)
+ atomic_inc(&cq->nr_events);
+ read_unlock(&ehca_cq_idr_lock);
+
+ if (!cq)
+ return;
+
+ ehca_error_data(shca, cq, cq->ipz_cq_handle.handle);
+
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+
+ return;
+}
+
+static void parse_identifier(struct ehca_shca *shca, u64 eqe)
+{
+ u8 identifier = EHCA_BMASK_GET(EQE_EE_IDENTIFIER, eqe);
+
+ switch (identifier) {
+ case 0x02: /* path migrated */
+ qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0);
+ break;
+ case 0x03: /* communication established */
+ qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0);
+ break;
+ case 0x04: /* send queue drained */
+ qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0);
+ break;
+ case 0x05: /* QP error */
+ case 0x06: /* QP error */
+ qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1);
+ break;
+ case 0x07: /* CQ error */
+ case 0x08: /* CQ error */
+ cq_event_callback(shca, eqe);
+ break;
+ case 0x09: /* MRMWPTE error */
+ ehca_err(&shca->ib_device, "MRMWPTE error.");
+ break;
+ case 0x0A: /* port event */
+ ehca_err(&shca->ib_device, "Port event.");
+ break;
+ case 0x0B: /* MR access error */
+ ehca_err(&shca->ib_device, "MR access error.");
+ break;
+ case 0x0C: /* EQ error */
+ ehca_err(&shca->ib_device, "EQ error.");
+ break;
+ case 0x0D: /* P/Q_Key mismatch */
+ ehca_err(&shca->ib_device, "P/Q_Key mismatch.");
+ break;
+ case 0x10: /* sampling complete */
+ ehca_err(&shca->ib_device, "Sampling complete.");
+ break;
+ case 0x11: /* unaffiliated access error */
+ ehca_err(&shca->ib_device, "Unaffiliated access error.");
+ break;
+ case 0x12: /* path migrating */
+ ehca_err(&shca->ib_device, "Path migrating.");
+ break;
+ case 0x13: /* interface trace stopped */
+ ehca_err(&shca->ib_device, "Interface trace stopped.");
+ break;
+ case 0x14: /* first error capture info available */
+ ehca_info(&shca->ib_device, "First error capture available");
+ break;
+ case 0x15: /* SRQ limit reached */
+ qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0);
+ break;
+ default:
+ ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.",
+ identifier, shca->ib_device.name);
+ break;
+ }
+
+ return;
+}
+
+static void dispatch_port_event(struct ehca_shca *shca, int port_num,
+ enum ib_event_type type, const char *msg)
+{
+ struct ib_event event;
+
+ ehca_info(&shca->ib_device, "port %d %s.", port_num, msg);
+ event.device = &shca->ib_device;
+ event.event = type;
+ event.element.port_num = port_num;
+ ib_dispatch_event(&event);
+}
+
+static void notify_port_conf_change(struct ehca_shca *shca, int port_num)
+{
+ struct ehca_sma_attr new_attr;
+ struct ehca_sma_attr *old_attr = &shca->sport[port_num - 1].saved_attr;
+
+ ehca_query_sma_attr(shca, port_num, &new_attr);
+
+ if (new_attr.sm_sl != old_attr->sm_sl ||
+ new_attr.sm_lid != old_attr->sm_lid)
+ dispatch_port_event(shca, port_num, IB_EVENT_SM_CHANGE,
+ "SM changed");
+
+ if (new_attr.lid != old_attr->lid ||
+ new_attr.lmc != old_attr->lmc)
+ dispatch_port_event(shca, port_num, IB_EVENT_LID_CHANGE,
+ "LID changed");
+
+ if (new_attr.pkey_tbl_len != old_attr->pkey_tbl_len ||
+ memcmp(new_attr.pkeys, old_attr->pkeys,
+ sizeof(u16) * new_attr.pkey_tbl_len))
+ dispatch_port_event(shca, port_num, IB_EVENT_PKEY_CHANGE,
+ "P_Key changed");
+
+ *old_attr = new_attr;
+}
+
+/* replay modify_qp for sqps -- return 0 if all is well, 1 if AQP1 destroyed */
+static int replay_modify_qp(struct ehca_sport *sport)
+{
+ int aqp1_destroyed;
+ unsigned long flags;
+
+ spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+
+ aqp1_destroyed = !sport->ibqp_sqp[IB_QPT_GSI];
+
+ if (sport->ibqp_sqp[IB_QPT_SMI])
+ ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_SMI]);
+ if (!aqp1_destroyed)
+ ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_GSI]);
+
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+
+ return aqp1_destroyed;
+}
+
+static void parse_ec(struct ehca_shca *shca, u64 eqe)
+{
+ u8 ec = EHCA_BMASK_GET(NEQE_EVENT_CODE, eqe);
+ u8 port = EHCA_BMASK_GET(NEQE_PORT_NUMBER, eqe);
+ u8 spec_event;
+ struct ehca_sport *sport = &shca->sport[port - 1];
+
+ switch (ec) {
+ case 0x30: /* port availability change */
+ if (EHCA_BMASK_GET(NEQE_PORT_AVAILABILITY, eqe)) {
+ /* only replay modify_qp calls in autodetect mode;
+ * if AQP1 was destroyed, the port is already down
+ * again and we can drop the event.
+ */
+ if (ehca_nr_ports < 0)
+ if (replay_modify_qp(sport))
+ break;
+
+ sport->port_state = IB_PORT_ACTIVE;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
+ "is active");
+ ehca_query_sma_attr(shca, port, &sport->saved_attr);
+ } else {
+ sport->port_state = IB_PORT_DOWN;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
+ "is inactive");
+ }
+ break;
+ case 0x31:
+ /* port configuration change
+ * disruptive change is caused by
+ * LID, PKEY or SM change
+ */
+ if (EHCA_BMASK_GET(NEQE_DISRUPTIVE, eqe)) {
+ ehca_warn(&shca->ib_device, "disruptive port "
+ "%d configuration change", port);
+
+ sport->port_state = IB_PORT_DOWN;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
+ "is inactive");
+
+ sport->port_state = IB_PORT_ACTIVE;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
+ "is active");
+ ehca_query_sma_attr(shca, port,
+ &sport->saved_attr);
+ } else
+ notify_port_conf_change(shca, port);
+ break;
+ case 0x32: /* adapter malfunction */
+ ehca_err(&shca->ib_device, "Adapter malfunction.");
+ break;
+ case 0x33: /* trace stopped */
+ ehca_err(&shca->ib_device, "Traced stopped.");
+ break;
+ case 0x34: /* util async event */
+ spec_event = EHCA_BMASK_GET(NEQE_SPECIFIC_EVENT, eqe);
+ if (spec_event == 0x80) /* client reregister required */
+ dispatch_port_event(shca, port,
+ IB_EVENT_CLIENT_REREGISTER,
+ "client reregister req.");
+ else
+ ehca_warn(&shca->ib_device, "Unknown util async "
+ "event %x on port %x", spec_event, port);
+ break;
+ default:
+ ehca_err(&shca->ib_device, "Unknown event code: %x on %s.",
+ ec, shca->ib_device.name);
+ break;
+ }
+
+ return;
+}
+
+static inline void reset_eq_pending(struct ehca_cq *cq)
+{
+ u64 CQx_EP;
+ struct h_galpa gal = cq->galpas.kernel;
+
+ hipz_galpa_store_cq(gal, cqx_ep, 0x0);
+ CQx_EP = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_ep));
+
+ return;
+}
+
+irqreturn_t ehca_interrupt_neq(int irq, void *dev_id)
+{
+ struct ehca_shca *shca = (struct ehca_shca*)dev_id;
+
+ tasklet_hi_schedule(&shca->neq.interrupt_task);
+
+ return IRQ_HANDLED;
+}
+
+void ehca_tasklet_neq(unsigned long data)
+{
+ struct ehca_shca *shca = (struct ehca_shca*)data;
+ struct ehca_eqe *eqe;
+ u64 ret;
+
+ eqe = ehca_poll_eq(shca, &shca->neq);
+
+ while (eqe) {
+ if (!EHCA_BMASK_GET(NEQE_COMPLETION_EVENT, eqe->entry))
+ parse_ec(shca, eqe->entry);
+
+ eqe = ehca_poll_eq(shca, &shca->neq);
+ }
+
+ ret = hipz_h_reset_event(shca->ipz_hca_handle,
+ shca->neq.ipz_eq_handle, 0xFFFFFFFFFFFFFFFFL);
+
+ if (ret != H_SUCCESS)
+ ehca_err(&shca->ib_device, "Can't clear notification events.");
+
+ return;
+}
+
+irqreturn_t ehca_interrupt_eq(int irq, void *dev_id)
+{
+ struct ehca_shca *shca = (struct ehca_shca*)dev_id;
+
+ tasklet_hi_schedule(&shca->eq.interrupt_task);
+
+ return IRQ_HANDLED;
+}
+
+
+static inline void process_eqe(struct ehca_shca *shca, struct ehca_eqe *eqe)
+{
+ u64 eqe_value;
+ u32 token;
+ struct ehca_cq *cq;
+
+ eqe_value = eqe->entry;
+ ehca_dbg(&shca->ib_device, "eqe_value=%llx", eqe_value);
+ if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
+ ehca_dbg(&shca->ib_device, "Got completion event");
+ token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
+ read_lock(&ehca_cq_idr_lock);
+ cq = idr_find(&ehca_cq_idr, token);
+ if (cq)
+ atomic_inc(&cq->nr_events);
+ read_unlock(&ehca_cq_idr_lock);
+ if (cq == NULL) {
+ ehca_err(&shca->ib_device,
+ "Invalid eqe for non-existing cq token=%x",
+ token);
+ return;
+ }
+ reset_eq_pending(cq);
+ if (ehca_scaling_code)
+ queue_comp_task(cq);
+ else {
+ comp_event_callback(cq);
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+ }
+ } else {
+ ehca_dbg(&shca->ib_device, "Got non completion event");
+ parse_identifier(shca, eqe_value);
+ }
+}
+
+void ehca_process_eq(struct ehca_shca *shca, int is_irq)
+{
+ struct ehca_eq *eq = &shca->eq;
+ struct ehca_eqe_cache_entry *eqe_cache = eq->eqe_cache;
+ u64 eqe_value, ret;
+ int eqe_cnt, i;
+ int eq_empty = 0;
+
+ spin_lock(&eq->irq_spinlock);
+ if (is_irq) {
+ const int max_query_cnt = 100;
+ int query_cnt = 0;
+ int int_state = 1;
+ do {
+ int_state = hipz_h_query_int_state(
+ shca->ipz_hca_handle, eq->ist);
+ query_cnt++;
+ iosync();
+ } while (int_state && query_cnt < max_query_cnt);
+ if (unlikely((query_cnt == max_query_cnt)))
+ ehca_dbg(&shca->ib_device, "int_state=%x query_cnt=%x",
+ int_state, query_cnt);
+ }
+
+ /* read out all eqes */
+ eqe_cnt = 0;
+ do {
+ u32 token;
+ eqe_cache[eqe_cnt].eqe = ehca_poll_eq(shca, eq);
+ if (!eqe_cache[eqe_cnt].eqe)
+ break;
+ eqe_value = eqe_cache[eqe_cnt].eqe->entry;
+ if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
+ token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
+ read_lock(&ehca_cq_idr_lock);
+ eqe_cache[eqe_cnt].cq = idr_find(&ehca_cq_idr, token);
+ if (eqe_cache[eqe_cnt].cq)
+ atomic_inc(&eqe_cache[eqe_cnt].cq->nr_events);
+ read_unlock(&ehca_cq_idr_lock);
+ if (!eqe_cache[eqe_cnt].cq) {
+ ehca_err(&shca->ib_device,
+ "Invalid eqe for non-existing cq "
+ "token=%x", token);
+ continue;
+ }
+ } else
+ eqe_cache[eqe_cnt].cq = NULL;
+ eqe_cnt++;
+ } while (eqe_cnt < EHCA_EQE_CACHE_SIZE);
+ if (!eqe_cnt) {
+ if (is_irq)
+ ehca_dbg(&shca->ib_device,
+ "No eqe found for irq event");
+ goto unlock_irq_spinlock;
+ } else if (!is_irq) {
+ ret = hipz_h_eoi(eq->ist);
+ if (ret != H_SUCCESS)
+ ehca_err(&shca->ib_device,
+ "bad return code EOI -rc = %lld\n", ret);
+ ehca_dbg(&shca->ib_device, "deadman found %x eqe", eqe_cnt);
+ }
+ if (unlikely(eqe_cnt == EHCA_EQE_CACHE_SIZE))
+ ehca_dbg(&shca->ib_device, "too many eqes for one irq event");
+ /* enable irq for new packets */
+ for (i = 0; i < eqe_cnt; i++) {
+ if (eq->eqe_cache[i].cq)
+ reset_eq_pending(eq->eqe_cache[i].cq);
+ }
+ /* check eq */
+ spin_lock(&eq->spinlock);
+ eq_empty = (!ipz_eqit_eq_peek_valid(&shca->eq.ipz_queue));
+ spin_unlock(&eq->spinlock);
+ /* call completion handler for cached eqes */
+ for (i = 0; i < eqe_cnt; i++)
+ if (eq->eqe_cache[i].cq) {
+ if (ehca_scaling_code)
+ queue_comp_task(eq->eqe_cache[i].cq);
+ else {
+ struct ehca_cq *cq = eq->eqe_cache[i].cq;
+ comp_event_callback(cq);
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+ }
+ } else {
+ ehca_dbg(&shca->ib_device, "Got non completion event");
+ parse_identifier(shca, eq->eqe_cache[i].eqe->entry);
+ }
+ /* poll eq if not empty */
+ if (eq_empty)
+ goto unlock_irq_spinlock;
+ do {
+ struct ehca_eqe *eqe;
+ eqe = ehca_poll_eq(shca, &shca->eq);
+ if (!eqe)
+ break;
+ process_eqe(shca, eqe);
+ } while (1);
+
+unlock_irq_spinlock:
+ spin_unlock(&eq->irq_spinlock);
+}
+
+void ehca_tasklet_eq(unsigned long data)
+{
+ ehca_process_eq((struct ehca_shca*)data, 1);
+}
+
+static int find_next_online_cpu(struct ehca_comp_pool *pool)
+{
+ int cpu;
+ unsigned long flags;
+
+ WARN_ON_ONCE(!in_interrupt());
+ if (ehca_debug_level >= 3)
+ ehca_dmp(cpu_online_mask, cpumask_size(), "");
+
+ spin_lock_irqsave(&pool->last_cpu_lock, flags);
+ do {
+ cpu = cpumask_next(pool->last_cpu, cpu_online_mask);
+ if (cpu >= nr_cpu_ids)
+ cpu = cpumask_first(cpu_online_mask);
+ pool->last_cpu = cpu;
+ } while (!per_cpu_ptr(pool->cpu_comp_tasks, cpu)->active);
+ spin_unlock_irqrestore(&pool->last_cpu_lock, flags);
+
+ return cpu;
+}
+
+static void __queue_comp_task(struct ehca_cq *__cq,
+ struct ehca_cpu_comp_task *cct,
+ struct task_struct *thread)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&cct->task_lock, flags);
+ spin_lock(&__cq->task_lock);
+
+ if (__cq->nr_callbacks == 0) {
+ __cq->nr_callbacks++;
+ list_add_tail(&__cq->entry, &cct->cq_list);
+ cct->cq_jobs++;
+ wake_up_process(thread);
+ } else
+ __cq->nr_callbacks++;
+
+ spin_unlock(&__cq->task_lock);
+ spin_unlock_irqrestore(&cct->task_lock, flags);
+}
+
+static void queue_comp_task(struct ehca_cq *__cq)
+{
+ int cpu_id;
+ struct ehca_cpu_comp_task *cct;
+ struct task_struct *thread;
+ int cq_jobs;
+ unsigned long flags;
+
+ cpu_id = find_next_online_cpu(pool);
+ BUG_ON(!cpu_online(cpu_id));
+
+ cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
+ thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
+ BUG_ON(!cct || !thread);
+
+ spin_lock_irqsave(&cct->task_lock, flags);
+ cq_jobs = cct->cq_jobs;
+ spin_unlock_irqrestore(&cct->task_lock, flags);
+ if (cq_jobs > 0) {
+ cpu_id = find_next_online_cpu(pool);
+ cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
+ thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
+ BUG_ON(!cct || !thread);
+ }
+ __queue_comp_task(__cq, cct, thread);
+}
+
+static void run_comp_task(struct ehca_cpu_comp_task *cct)
+{
+ struct ehca_cq *cq;
+
+ while (!list_empty(&cct->cq_list)) {
+ cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
+ spin_unlock_irq(&cct->task_lock);
+
+ comp_event_callback(cq);
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+
+ spin_lock_irq(&cct->task_lock);
+ spin_lock(&cq->task_lock);
+ cq->nr_callbacks--;
+ if (!cq->nr_callbacks) {
+ list_del_init(cct->cq_list.next);
+ cct->cq_jobs--;
+ }
+ spin_unlock(&cq->task_lock);
+ }
+}
+
+static void comp_task_park(unsigned int cpu)
+{
+ struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+ struct ehca_cpu_comp_task *target;
+ struct task_struct *thread;
+ struct ehca_cq *cq, *tmp;
+ LIST_HEAD(list);
+
+ spin_lock_irq(&cct->task_lock);
+ cct->cq_jobs = 0;
+ cct->active = 0;
+ list_splice_init(&cct->cq_list, &list);
+ spin_unlock_irq(&cct->task_lock);
+
+ cpu = find_next_online_cpu(pool);
+ target = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+ thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu);
+ spin_lock_irq(&target->task_lock);
+ list_for_each_entry_safe(cq, tmp, &list, entry) {
+ list_del(&cq->entry);
+ __queue_comp_task(cq, target, thread);
+ }
+ spin_unlock_irq(&target->task_lock);
+}
+
+static void comp_task_stop(unsigned int cpu, bool online)
+{
+ struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+
+ spin_lock_irq(&cct->task_lock);
+ cct->cq_jobs = 0;
+ cct->active = 0;
+ WARN_ON(!list_empty(&cct->cq_list));
+ spin_unlock_irq(&cct->task_lock);
+}
+
+static int comp_task_should_run(unsigned int cpu)
+{
+ struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+
+ return cct->cq_jobs;
+}
+
+static void comp_task(unsigned int cpu)
+{
+ struct ehca_cpu_comp_task *cct = this_cpu_ptr(pool->cpu_comp_tasks);
+ int cql_empty;
+
+ spin_lock_irq(&cct->task_lock);
+ cql_empty = list_empty(&cct->cq_list);
+ if (!cql_empty) {
+ __set_current_state(TASK_RUNNING);
+ run_comp_task(cct);
+ }
+ spin_unlock_irq(&cct->task_lock);
+}
+
+static struct smp_hotplug_thread comp_pool_threads = {
+ .thread_should_run = comp_task_should_run,
+ .thread_fn = comp_task,
+ .thread_comm = "ehca_comp/%u",
+ .cleanup = comp_task_stop,
+ .park = comp_task_park,
+};
+
+int ehca_create_comp_pool(void)
+{
+ int cpu, ret = -ENOMEM;
+
+ if (!ehca_scaling_code)
+ return 0;
+
+ pool = kzalloc(sizeof(struct ehca_comp_pool), GFP_KERNEL);
+ if (pool == NULL)
+ return -ENOMEM;
+
+ spin_lock_init(&pool->last_cpu_lock);
+ pool->last_cpu = cpumask_any(cpu_online_mask);
+
+ pool->cpu_comp_tasks = alloc_percpu(struct ehca_cpu_comp_task);
+ if (!pool->cpu_comp_tasks)
+ goto out_pool;
+
+ pool->cpu_comp_threads = alloc_percpu(struct task_struct *);
+ if (!pool->cpu_comp_threads)
+ goto out_tasks;
+
+ for_each_present_cpu(cpu) {
+ struct ehca_cpu_comp_task *cct;
+
+ cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+ spin_lock_init(&cct->task_lock);
+ INIT_LIST_HEAD(&cct->cq_list);
+ }
+
+ comp_pool_threads.store = pool->cpu_comp_threads;
+ ret = smpboot_register_percpu_thread(&comp_pool_threads);
+ if (ret)
+ goto out_threads;
+
+ pr_info("eHCA scaling code enabled\n");
+ return ret;
+
+out_threads:
+ free_percpu(pool->cpu_comp_threads);
+out_tasks:
+ free_percpu(pool->cpu_comp_tasks);
+out_pool:
+ kfree(pool);
+ return ret;
+}
+
+void ehca_destroy_comp_pool(void)
+{
+ if (!ehca_scaling_code)
+ return;
+
+ smpboot_unregister_percpu_thread(&comp_pool_threads);
+
+ free_percpu(pool->cpu_comp_threads);
+ free_percpu(pool->cpu_comp_tasks);
+ kfree(pool);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Function definitions and structs for EQs, NEQs and interrupts
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_IRQ_H
+#define __EHCA_IRQ_H
+
+
+struct ehca_shca;
+
+#include <linux/interrupt.h>
+#include <linux/types.h>
+
+int ehca_error_data(struct ehca_shca *shca, void *data, u64 resource);
+
+irqreturn_t ehca_interrupt_neq(int irq, void *dev_id);
+void ehca_tasklet_neq(unsigned long data);
+
+irqreturn_t ehca_interrupt_eq(int irq, void *dev_id);
+void ehca_tasklet_eq(unsigned long data);
+void ehca_process_eq(struct ehca_shca *shca, int is_irq);
+
+struct ehca_cpu_comp_task {
+ struct list_head cq_list;
+ spinlock_t task_lock;
+ int cq_jobs;
+ int active;
+};
+
+struct ehca_comp_pool {
+ struct ehca_cpu_comp_task __percpu *cpu_comp_tasks;
+ struct task_struct * __percpu *cpu_comp_threads;
+ int last_cpu;
+ spinlock_t last_cpu_lock;
+};
+
+int ehca_create_comp_pool(void);
+void ehca_destroy_comp_pool(void);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Function definitions for internal functions
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Dietmar Decker <ddecker@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_IVERBS_H__
+#define __EHCA_IVERBS_H__
+
+#include "ehca_classes.h"
+
+int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
+ struct ib_udata *uhw);
+
+int ehca_query_port(struct ib_device *ibdev, u8 port,
+ struct ib_port_attr *props);
+
+enum rdma_protocol_type
+ehca_query_protocol(struct ib_device *device, u8 port_num);
+
+int ehca_query_sma_attr(struct ehca_shca *shca, u8 port,
+ struct ehca_sma_attr *attr);
+
+int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 * pkey);
+
+int ehca_query_gid(struct ib_device *ibdev, u8 port, int index,
+ union ib_gid *gid);
+
+int ehca_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask,
+ struct ib_port_modify *props);
+
+struct ib_pd *ehca_alloc_pd(struct ib_device *device,
+ struct ib_ucontext *context,
+ struct ib_udata *udata);
+
+int ehca_dealloc_pd(struct ib_pd *pd);
+
+struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
+
+int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+int ehca_destroy_ah(struct ib_ah *ah);
+
+struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
+
+struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ int mr_access_flags, u64 *iova_start);
+
+struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+ u64 virt, int mr_access_flags,
+ struct ib_udata *udata);
+
+int ehca_rereg_phys_mr(struct ib_mr *mr,
+ int mr_rereg_mask,
+ struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf, int mr_access_flags, u64 *iova_start);
+
+int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr);
+
+int ehca_dereg_mr(struct ib_mr *mr);
+
+struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type);
+
+int ehca_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
+ struct ib_mw_bind *mw_bind);
+
+int ehca_dealloc_mw(struct ib_mw *mw);
+
+struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
+ int mr_access_flags,
+ struct ib_fmr_attr *fmr_attr);
+
+int ehca_map_phys_fmr(struct ib_fmr *fmr,
+ u64 *page_list, int list_len, u64 iova);
+
+int ehca_unmap_fmr(struct list_head *fmr_list);
+
+int ehca_dealloc_fmr(struct ib_fmr *fmr);
+
+enum ehca_eq_type {
+ EHCA_EQ = 0, /* Event Queue */
+ EHCA_NEQ /* Notification Event Queue */
+};
+
+int ehca_create_eq(struct ehca_shca *shca, struct ehca_eq *eq,
+ enum ehca_eq_type type, const u32 length);
+
+int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq);
+
+void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq);
+
+
+struct ib_cq *ehca_create_cq(struct ib_device *device,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *context,
+ struct ib_udata *udata);
+
+int ehca_destroy_cq(struct ib_cq *cq);
+
+int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
+
+int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
+
+int ehca_peek_cq(struct ib_cq *cq, int wc_cnt);
+
+int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags);
+
+struct ib_qp *ehca_create_qp(struct ib_pd *pd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int ehca_destroy_qp(struct ib_qp *qp);
+
+int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata);
+
+int ehca_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+
+int ehca_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr,
+ struct ib_send_wr **bad_send_wr);
+
+int ehca_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr);
+
+int ehca_post_srq_recv(struct ib_srq *srq,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr);
+
+struct ib_srq *ehca_create_srq(struct ib_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int ehca_modify_srq(struct ib_srq *srq, struct ib_srq_attr *attr,
+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
+
+int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
+
+int ehca_destroy_srq(struct ib_srq *srq);
+
+u64 ehca_define_sqp(struct ehca_shca *shca, struct ehca_qp *ibqp,
+ struct ib_qp_init_attr *qp_init_attr);
+
+int ehca_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
+
+int ehca_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
+
+struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
+ struct ib_udata *udata);
+
+int ehca_dealloc_ucontext(struct ib_ucontext *context);
+
+int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+
+int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad_hdr *in, size_t in_mad_size,
+ struct ib_mad_hdr *out, size_t *out_mad_size,
+ u16 *out_mad_pkey_index);
+
+void ehca_poll_eqs(unsigned long data);
+
+int ehca_calc_ipd(struct ehca_shca *shca, int port,
+ enum ib_rate path_rate, u32 *ipd);
+
+void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq);
+
+#ifdef CONFIG_PPC_64K_PAGES
+void *ehca_alloc_fw_ctrlblock(gfp_t flags);
+void ehca_free_fw_ctrlblock(void *ptr);
+#else
+#define ehca_alloc_fw_ctrlblock(flags) ((void *)get_zeroed_page(flags))
+#define ehca_free_fw_ctrlblock(ptr) free_page((unsigned long)(ptr))
+#endif
+
+void ehca_recover_sqp(struct ib_qp *sqp);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * module start stop, hca detection
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef CONFIG_PPC_64K_PAGES
+#include <linux/slab.h>
+#endif
+
+#include <linux/notifier.h>
+#include <linux/memory.h>
+#include <rdma/ib_mad.h>
+#include "ehca_classes.h"
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+
+#define HCAD_VERSION "0029"
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
+MODULE_DESCRIPTION("IBM eServer HCA InfiniBand Device Driver");
+MODULE_VERSION(HCAD_VERSION);
+
+static bool ehca_open_aqp1 = 0;
+static int ehca_hw_level = 0;
+static bool ehca_poll_all_eqs = 1;
+
+int ehca_debug_level = 0;
+int ehca_nr_ports = -1;
+bool ehca_use_hp_mr = 0;
+int ehca_port_act_time = 30;
+int ehca_static_rate = -1;
+bool ehca_scaling_code = 0;
+int ehca_lock_hcalls = -1;
+int ehca_max_cq = -1;
+int ehca_max_qp = -1;
+
+module_param_named(open_aqp1, ehca_open_aqp1, bool, S_IRUGO);
+module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
+module_param_named(hw_level, ehca_hw_level, int, S_IRUGO);
+module_param_named(nr_ports, ehca_nr_ports, int, S_IRUGO);
+module_param_named(use_hp_mr, ehca_use_hp_mr, bool, S_IRUGO);
+module_param_named(port_act_time, ehca_port_act_time, int, S_IRUGO);
+module_param_named(poll_all_eqs, ehca_poll_all_eqs, bool, S_IRUGO);
+module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
+module_param_named(scaling_code, ehca_scaling_code, bool, S_IRUGO);
+module_param_named(lock_hcalls, ehca_lock_hcalls, bint, S_IRUGO);
+module_param_named(number_of_cqs, ehca_max_cq, int, S_IRUGO);
+module_param_named(number_of_qps, ehca_max_qp, int, S_IRUGO);
+
+MODULE_PARM_DESC(open_aqp1,
+ "Open AQP1 on startup (default: no)");
+MODULE_PARM_DESC(debug_level,
+ "Amount of debug output (0: none (default), 1: traces, "
+ "2: some dumps, 3: lots)");
+MODULE_PARM_DESC(hw_level,
+ "Hardware level (0: autosensing (default), "
+ "0x10..0x14: eHCA, 0x20..0x23: eHCA2)");
+MODULE_PARM_DESC(nr_ports,
+ "number of connected ports (-1: autodetect (default), "
+ "1: port one only, 2: two ports)");
+MODULE_PARM_DESC(use_hp_mr,
+ "Use high performance MRs (default: no)");
+MODULE_PARM_DESC(port_act_time,
+ "Time to wait for port activation (default: 30 sec)");
+MODULE_PARM_DESC(poll_all_eqs,
+ "Poll all event queues periodically (default: yes)");
+MODULE_PARM_DESC(static_rate,
+ "Set permanent static rate (default: no static rate)");
+MODULE_PARM_DESC(scaling_code,
+ "Enable scaling code (default: no)");
+MODULE_PARM_DESC(lock_hcalls,
+ "Serialize all hCalls made by the driver "
+ "(default: autodetect)");
+MODULE_PARM_DESC(number_of_cqs,
+ "Max number of CQs which can be allocated "
+ "(default: autodetect)");
+MODULE_PARM_DESC(number_of_qps,
+ "Max number of QPs which can be allocated "
+ "(default: autodetect)");
+
+DEFINE_RWLOCK(ehca_qp_idr_lock);
+DEFINE_RWLOCK(ehca_cq_idr_lock);
+DEFINE_IDR(ehca_qp_idr);
+DEFINE_IDR(ehca_cq_idr);
+
+static LIST_HEAD(shca_list); /* list of all registered ehcas */
+DEFINE_SPINLOCK(shca_list_lock);
+
+static struct timer_list poll_eqs_timer;
+
+#ifdef CONFIG_PPC_64K_PAGES
+static struct kmem_cache *ctblk_cache;
+
+void *ehca_alloc_fw_ctrlblock(gfp_t flags)
+{
+ void *ret = kmem_cache_zalloc(ctblk_cache, flags);
+ if (!ret)
+ ehca_gen_err("Out of memory for ctblk");
+ return ret;
+}
+
+void ehca_free_fw_ctrlblock(void *ptr)
+{
+ if (ptr)
+ kmem_cache_free(ctblk_cache, ptr);
+
+}
+#endif
+
+int ehca2ib_return_code(u64 ehca_rc)
+{
+ switch (ehca_rc) {
+ case H_SUCCESS:
+ return 0;
+ case H_RESOURCE: /* Resource in use */
+ case H_BUSY:
+ return -EBUSY;
+ case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
+ case H_CONSTRAINED: /* resource constraint */
+ case H_NO_MEM:
+ return -ENOMEM;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int ehca_create_slab_caches(void)
+{
+ int ret;
+
+ ret = ehca_init_pd_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create PD SLAB cache.");
+ return ret;
+ }
+
+ ret = ehca_init_cq_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create CQ SLAB cache.");
+ goto create_slab_caches2;
+ }
+
+ ret = ehca_init_qp_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create QP SLAB cache.");
+ goto create_slab_caches3;
+ }
+
+ ret = ehca_init_av_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create AV SLAB cache.");
+ goto create_slab_caches4;
+ }
+
+ ret = ehca_init_mrmw_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create MR&MW SLAB cache.");
+ goto create_slab_caches5;
+ }
+
+ ret = ehca_init_small_qp_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create small queue SLAB cache.");
+ goto create_slab_caches6;
+ }
+
+#ifdef CONFIG_PPC_64K_PAGES
+ ctblk_cache = kmem_cache_create("ehca_cache_ctblk",
+ EHCA_PAGESIZE, H_CB_ALIGNMENT,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!ctblk_cache) {
+ ehca_gen_err("Cannot create ctblk SLAB cache.");
+ ehca_cleanup_small_qp_cache();
+ ret = -ENOMEM;
+ goto create_slab_caches6;
+ }
+#endif
+ return 0;
+
+create_slab_caches6:
+ ehca_cleanup_mrmw_cache();
+
+create_slab_caches5:
+ ehca_cleanup_av_cache();
+
+create_slab_caches4:
+ ehca_cleanup_qp_cache();
+
+create_slab_caches3:
+ ehca_cleanup_cq_cache();
+
+create_slab_caches2:
+ ehca_cleanup_pd_cache();
+
+ return ret;
+}
+
+static void ehca_destroy_slab_caches(void)
+{
+ ehca_cleanup_small_qp_cache();
+ ehca_cleanup_mrmw_cache();
+ ehca_cleanup_av_cache();
+ ehca_cleanup_qp_cache();
+ ehca_cleanup_cq_cache();
+ ehca_cleanup_pd_cache();
+#ifdef CONFIG_PPC_64K_PAGES
+ if (ctblk_cache)
+ kmem_cache_destroy(ctblk_cache);
+#endif
+}
+
+#define EHCA_HCAAVER EHCA_BMASK_IBM(32, 39)
+#define EHCA_REVID EHCA_BMASK_IBM(40, 63)
+
+static struct cap_descr {
+ u64 mask;
+ char *descr;
+} hca_cap_descr[] = {
+ { HCA_CAP_AH_PORT_NR_CHECK, "HCA_CAP_AH_PORT_NR_CHECK" },
+ { HCA_CAP_ATOMIC, "HCA_CAP_ATOMIC" },
+ { HCA_CAP_AUTO_PATH_MIG, "HCA_CAP_AUTO_PATH_MIG" },
+ { HCA_CAP_BAD_P_KEY_CTR, "HCA_CAP_BAD_P_KEY_CTR" },
+ { HCA_CAP_SQD_RTS_PORT_CHANGE, "HCA_CAP_SQD_RTS_PORT_CHANGE" },
+ { HCA_CAP_CUR_QP_STATE_MOD, "HCA_CAP_CUR_QP_STATE_MOD" },
+ { HCA_CAP_INIT_TYPE, "HCA_CAP_INIT_TYPE" },
+ { HCA_CAP_PORT_ACTIVE_EVENT, "HCA_CAP_PORT_ACTIVE_EVENT" },
+ { HCA_CAP_Q_KEY_VIOL_CTR, "HCA_CAP_Q_KEY_VIOL_CTR" },
+ { HCA_CAP_WQE_RESIZE, "HCA_CAP_WQE_RESIZE" },
+ { HCA_CAP_RAW_PACKET_MCAST, "HCA_CAP_RAW_PACKET_MCAST" },
+ { HCA_CAP_SHUTDOWN_PORT, "HCA_CAP_SHUTDOWN_PORT" },
+ { HCA_CAP_RC_LL_QP, "HCA_CAP_RC_LL_QP" },
+ { HCA_CAP_SRQ, "HCA_CAP_SRQ" },
+ { HCA_CAP_UD_LL_QP, "HCA_CAP_UD_LL_QP" },
+ { HCA_CAP_RESIZE_MR, "HCA_CAP_RESIZE_MR" },
+ { HCA_CAP_MINI_QP, "HCA_CAP_MINI_QP" },
+ { HCA_CAP_H_ALLOC_RES_SYNC, "HCA_CAP_H_ALLOC_RES_SYNC" },
+};
+
+static int ehca_sense_attributes(struct ehca_shca *shca)
+{
+ int i, ret = 0;
+ u64 h_ret;
+ struct hipz_query_hca *rblock;
+ struct hipz_query_port *port;
+ const char *loc_code;
+
+ static const u32 pgsize_map[] = {
+ HCA_CAP_MR_PGSIZE_4K, 0x1000,
+ HCA_CAP_MR_PGSIZE_64K, 0x10000,
+ HCA_CAP_MR_PGSIZE_1M, 0x100000,
+ HCA_CAP_MR_PGSIZE_16M, 0x1000000,
+ };
+
+ ehca_gen_dbg("Probing adapter %s...",
+ shca->ofdev->dev.of_node->full_name);
+ loc_code = of_get_property(shca->ofdev->dev.of_node, "ibm,loc-code",
+ NULL);
+ if (loc_code)
+ ehca_gen_dbg(" ... location lode=%s", loc_code);
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_gen_err("Cannot allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_hca(shca->ipz_hca_handle, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_gen_err("Cannot query device properties. h_ret=%lli",
+ h_ret);
+ ret = -EPERM;
+ goto sense_attributes1;
+ }
+
+ if (ehca_nr_ports == 1)
+ shca->num_ports = 1;
+ else
+ shca->num_ports = (u8)rblock->num_ports;
+
+ ehca_gen_dbg(" ... found %x ports", rblock->num_ports);
+
+ if (ehca_hw_level == 0) {
+ u32 hcaaver;
+ u32 revid;
+
+ hcaaver = EHCA_BMASK_GET(EHCA_HCAAVER, rblock->hw_ver);
+ revid = EHCA_BMASK_GET(EHCA_REVID, rblock->hw_ver);
+
+ ehca_gen_dbg(" ... hardware version=%x:%x", hcaaver, revid);
+
+ if (hcaaver == 1) {
+ if (revid <= 3)
+ shca->hw_level = 0x10 | (revid + 1);
+ else
+ shca->hw_level = 0x14;
+ } else if (hcaaver == 2) {
+ if (revid == 0)
+ shca->hw_level = 0x21;
+ else if (revid == 0x10)
+ shca->hw_level = 0x22;
+ else if (revid == 0x20 || revid == 0x21)
+ shca->hw_level = 0x23;
+ }
+
+ if (!shca->hw_level) {
+ ehca_gen_warn("unknown hardware version"
+ " - assuming default level");
+ shca->hw_level = 0x22;
+ }
+ } else
+ shca->hw_level = ehca_hw_level;
+ ehca_gen_dbg(" ... hardware level=%x", shca->hw_level);
+
+ shca->hca_cap = rblock->hca_cap_indicators;
+ ehca_gen_dbg(" ... HCA capabilities:");
+ for (i = 0; i < ARRAY_SIZE(hca_cap_descr); i++)
+ if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap))
+ ehca_gen_dbg(" %s", hca_cap_descr[i].descr);
+
+ /* Autodetect hCall locking -- the "H_ALLOC_RESOURCE synced" flag is
+ * a firmware property, so it's valid across all adapters
+ */
+ if (ehca_lock_hcalls == -1)
+ ehca_lock_hcalls = !EHCA_BMASK_GET(HCA_CAP_H_ALLOC_RES_SYNC,
+ shca->hca_cap);
+
+ /* translate supported MR page sizes; always support 4K */
+ shca->hca_cap_mr_pgsize = EHCA_PAGESIZE;
+ for (i = 0; i < ARRAY_SIZE(pgsize_map); i += 2)
+ if (rblock->memory_page_size_supported & pgsize_map[i])
+ shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];
+
+ /* Set maximum number of CQs and QPs to calculate EQ size */
+ if (shca->max_num_qps == -1)
+ shca->max_num_qps = min_t(int, rblock->max_qp,
+ EHCA_MAX_NUM_QUEUES);
+ else if (shca->max_num_qps < 1 || shca->max_num_qps > rblock->max_qp) {
+ ehca_gen_warn("The requested number of QPs is out of range "
+ "(1 - %i) specified by HW. Value is set to %i",
+ rblock->max_qp, rblock->max_qp);
+ shca->max_num_qps = rblock->max_qp;
+ }
+
+ if (shca->max_num_cqs == -1)
+ shca->max_num_cqs = min_t(int, rblock->max_cq,
+ EHCA_MAX_NUM_QUEUES);
+ else if (shca->max_num_cqs < 1 || shca->max_num_cqs > rblock->max_cq) {
+ ehca_gen_warn("The requested number of CQs is out of range "
+ "(1 - %i) specified by HW. Value is set to %i",
+ rblock->max_cq, rblock->max_cq);
+ }
+
+ /* query max MTU from first port -- it's the same for all ports */
+ port = (struct hipz_query_port *)rblock;
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
+ if (h_ret != H_SUCCESS) {
+ ehca_gen_err("Cannot query port properties. h_ret=%lli",
+ h_ret);
+ ret = -EPERM;
+ goto sense_attributes1;
+ }
+
+ shca->max_mtu = port->max_mtu;
+
+sense_attributes1:
+ ehca_free_fw_ctrlblock(rblock);
+ return ret;
+}
+
+static int init_node_guid(struct ehca_shca *shca)
+{
+ int ret = 0;
+ struct hipz_query_hca *rblock;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query device properties");
+ ret = -EINVAL;
+ goto init_node_guid1;
+ }
+
+ memcpy(&shca->ib_device.node_guid, &rblock->node_guid, sizeof(u64));
+
+init_node_guid1:
+ ehca_free_fw_ctrlblock(rblock);
+ return ret;
+}
+
+static int ehca_port_immutable(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_immutable *immutable)
+{
+ struct ib_port_attr attr;
+ int err;
+
+ err = ehca_query_port(ibdev, port_num, &attr);
+ if (err)
+ return err;
+
+ immutable->pkey_tbl_len = attr.pkey_tbl_len;
+ immutable->gid_tbl_len = attr.gid_tbl_len;
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+
+ return 0;
+}
+
+static int ehca_init_device(struct ehca_shca *shca)
+{
+ int ret;
+
+ ret = init_node_guid(shca);
+ if (ret)
+ return ret;
+
+ strlcpy(shca->ib_device.name, "ehca%d", IB_DEVICE_NAME_MAX);
+ shca->ib_device.owner = THIS_MODULE;
+
+ shca->ib_device.uverbs_abi_ver = 8;
+ shca->ib_device.uverbs_cmd_mask =
+ (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
+ (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_REG_MR) |
+ (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
+ (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
+ (1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
+
+ shca->ib_device.node_type = RDMA_NODE_IB_CA;
+ shca->ib_device.phys_port_cnt = shca->num_ports;
+ shca->ib_device.num_comp_vectors = 1;
+ shca->ib_device.dma_device = &shca->ofdev->dev;
+ shca->ib_device.query_device = ehca_query_device;
+ shca->ib_device.query_port = ehca_query_port;
+ shca->ib_device.query_gid = ehca_query_gid;
+ shca->ib_device.query_pkey = ehca_query_pkey;
+ /* shca->in_device.modify_device = ehca_modify_device */
+ shca->ib_device.modify_port = ehca_modify_port;
+ shca->ib_device.alloc_ucontext = ehca_alloc_ucontext;
+ shca->ib_device.dealloc_ucontext = ehca_dealloc_ucontext;
+ shca->ib_device.alloc_pd = ehca_alloc_pd;
+ shca->ib_device.dealloc_pd = ehca_dealloc_pd;
+ shca->ib_device.create_ah = ehca_create_ah;
+ /* shca->ib_device.modify_ah = ehca_modify_ah; */
+ shca->ib_device.query_ah = ehca_query_ah;
+ shca->ib_device.destroy_ah = ehca_destroy_ah;
+ shca->ib_device.create_qp = ehca_create_qp;
+ shca->ib_device.modify_qp = ehca_modify_qp;
+ shca->ib_device.query_qp = ehca_query_qp;
+ shca->ib_device.destroy_qp = ehca_destroy_qp;
+ shca->ib_device.post_send = ehca_post_send;
+ shca->ib_device.post_recv = ehca_post_recv;
+ shca->ib_device.create_cq = ehca_create_cq;
+ shca->ib_device.destroy_cq = ehca_destroy_cq;
+ shca->ib_device.resize_cq = ehca_resize_cq;
+ shca->ib_device.poll_cq = ehca_poll_cq;
+ /* shca->ib_device.peek_cq = ehca_peek_cq; */
+ shca->ib_device.req_notify_cq = ehca_req_notify_cq;
+ /* shca->ib_device.req_ncomp_notif = ehca_req_ncomp_notif; */
+ shca->ib_device.get_dma_mr = ehca_get_dma_mr;
+ shca->ib_device.reg_phys_mr = ehca_reg_phys_mr;
+ shca->ib_device.reg_user_mr = ehca_reg_user_mr;
+ shca->ib_device.query_mr = ehca_query_mr;
+ shca->ib_device.dereg_mr = ehca_dereg_mr;
+ shca->ib_device.rereg_phys_mr = ehca_rereg_phys_mr;
+ shca->ib_device.alloc_mw = ehca_alloc_mw;
+ shca->ib_device.bind_mw = ehca_bind_mw;
+ shca->ib_device.dealloc_mw = ehca_dealloc_mw;
+ shca->ib_device.alloc_fmr = ehca_alloc_fmr;
+ shca->ib_device.map_phys_fmr = ehca_map_phys_fmr;
+ shca->ib_device.unmap_fmr = ehca_unmap_fmr;
+ shca->ib_device.dealloc_fmr = ehca_dealloc_fmr;
+ shca->ib_device.attach_mcast = ehca_attach_mcast;
+ shca->ib_device.detach_mcast = ehca_detach_mcast;
+ shca->ib_device.process_mad = ehca_process_mad;
+ shca->ib_device.mmap = ehca_mmap;
+ shca->ib_device.dma_ops = &ehca_dma_mapping_ops;
+ shca->ib_device.get_port_immutable = ehca_port_immutable;
+
+ if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
+ shca->ib_device.uverbs_cmd_mask |=
+ (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
+
+ shca->ib_device.create_srq = ehca_create_srq;
+ shca->ib_device.modify_srq = ehca_modify_srq;
+ shca->ib_device.query_srq = ehca_query_srq;
+ shca->ib_device.destroy_srq = ehca_destroy_srq;
+ shca->ib_device.post_srq_recv = ehca_post_srq_recv;
+ }
+
+ return ret;
+}
+
+static int ehca_create_aqp1(struct ehca_shca *shca, u32 port)
+{
+ struct ehca_sport *sport = &shca->sport[port - 1];
+ struct ib_cq *ibcq;
+ struct ib_qp *ibqp;
+ struct ib_qp_init_attr qp_init_attr;
+ struct ib_cq_init_attr cq_attr = {};
+ int ret;
+
+ if (sport->ibcq_aqp1) {
+ ehca_err(&shca->ib_device, "AQP1 CQ is already created.");
+ return -EPERM;
+ }
+
+ cq_attr.cqe = 10;
+ ibcq = ib_create_cq(&shca->ib_device, NULL, NULL, (void *)(-1),
+ &cq_attr);
+ if (IS_ERR(ibcq)) {
+ ehca_err(&shca->ib_device, "Cannot create AQP1 CQ.");
+ return PTR_ERR(ibcq);
+ }
+ sport->ibcq_aqp1 = ibcq;
+
+ if (sport->ibqp_sqp[IB_QPT_GSI]) {
+ ehca_err(&shca->ib_device, "AQP1 QP is already created.");
+ ret = -EPERM;
+ goto create_aqp1;
+ }
+
+ memset(&qp_init_attr, 0, sizeof(struct ib_qp_init_attr));
+ qp_init_attr.send_cq = ibcq;
+ qp_init_attr.recv_cq = ibcq;
+ qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
+ qp_init_attr.cap.max_send_wr = 100;
+ qp_init_attr.cap.max_recv_wr = 100;
+ qp_init_attr.cap.max_send_sge = 2;
+ qp_init_attr.cap.max_recv_sge = 1;
+ qp_init_attr.qp_type = IB_QPT_GSI;
+ qp_init_attr.port_num = port;
+ qp_init_attr.qp_context = NULL;
+ qp_init_attr.event_handler = NULL;
+ qp_init_attr.srq = NULL;
+
+ ibqp = ib_create_qp(&shca->pd->ib_pd, &qp_init_attr);
+ if (IS_ERR(ibqp)) {
+ ehca_err(&shca->ib_device, "Cannot create AQP1 QP.");
+ ret = PTR_ERR(ibqp);
+ goto create_aqp1;
+ }
+ sport->ibqp_sqp[IB_QPT_GSI] = ibqp;
+
+ return 0;
+
+create_aqp1:
+ ib_destroy_cq(sport->ibcq_aqp1);
+ return ret;
+}
+
+static int ehca_destroy_aqp1(struct ehca_sport *sport)
+{
+ int ret;
+
+ ret = ib_destroy_qp(sport->ibqp_sqp[IB_QPT_GSI]);
+ if (ret) {
+ ehca_gen_err("Cannot destroy AQP1 QP. ret=%i", ret);
+ return ret;
+ }
+
+ ret = ib_destroy_cq(sport->ibcq_aqp1);
+ if (ret)
+ ehca_gen_err("Cannot destroy AQP1 CQ. ret=%i", ret);
+
+ return ret;
+}
+
+static ssize_t ehca_show_debug_level(struct device_driver *ddp, char *buf)
+{
+ return snprintf(buf, PAGE_SIZE, "%d\n", ehca_debug_level);
+}
+
+static ssize_t ehca_store_debug_level(struct device_driver *ddp,
+ const char *buf, size_t count)
+{
+ int value = (*buf) - '0';
+ if (value >= 0 && value <= 9)
+ ehca_debug_level = value;
+ return 1;
+}
+
+static DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
+ ehca_show_debug_level, ehca_store_debug_level);
+
+static struct attribute *ehca_drv_attrs[] = {
+ &driver_attr_debug_level.attr,
+ NULL
+};
+
+static struct attribute_group ehca_drv_attr_grp = {
+ .attrs = ehca_drv_attrs
+};
+
+static const struct attribute_group *ehca_drv_attr_groups[] = {
+ &ehca_drv_attr_grp,
+ NULL,
+};
+
+#define EHCA_RESOURCE_ATTR(name) \
+static ssize_t ehca_show_##name(struct device *dev, \
+ struct device_attribute *attr, \
+ char *buf) \
+{ \
+ struct ehca_shca *shca; \
+ struct hipz_query_hca *rblock; \
+ int data; \
+ \
+ shca = dev_get_drvdata(dev); \
+ \
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL); \
+ if (!rblock) { \
+ dev_err(dev, "Can't allocate rblock memory.\n"); \
+ return 0; \
+ } \
+ \
+ if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) { \
+ dev_err(dev, "Can't query device properties\n"); \
+ ehca_free_fw_ctrlblock(rblock); \
+ return 0; \
+ } \
+ \
+ data = rblock->name; \
+ ehca_free_fw_ctrlblock(rblock); \
+ \
+ if ((strcmp(#name, "num_ports") == 0) && (ehca_nr_ports == 1)) \
+ return snprintf(buf, 256, "1\n"); \
+ else \
+ return snprintf(buf, 256, "%d\n", data); \
+ \
+} \
+static DEVICE_ATTR(name, S_IRUGO, ehca_show_##name, NULL);
+
+EHCA_RESOURCE_ATTR(num_ports);
+EHCA_RESOURCE_ATTR(hw_ver);
+EHCA_RESOURCE_ATTR(max_eq);
+EHCA_RESOURCE_ATTR(cur_eq);
+EHCA_RESOURCE_ATTR(max_cq);
+EHCA_RESOURCE_ATTR(cur_cq);
+EHCA_RESOURCE_ATTR(max_qp);
+EHCA_RESOURCE_ATTR(cur_qp);
+EHCA_RESOURCE_ATTR(max_mr);
+EHCA_RESOURCE_ATTR(cur_mr);
+EHCA_RESOURCE_ATTR(max_mw);
+EHCA_RESOURCE_ATTR(cur_mw);
+EHCA_RESOURCE_ATTR(max_pd);
+EHCA_RESOURCE_ATTR(max_ah);
+
+static ssize_t ehca_show_adapter_handle(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct ehca_shca *shca = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%llx\n", shca->ipz_hca_handle.handle);
+
+}
+static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
+
+static struct attribute *ehca_dev_attrs[] = {
+ &dev_attr_adapter_handle.attr,
+ &dev_attr_num_ports.attr,
+ &dev_attr_hw_ver.attr,
+ &dev_attr_max_eq.attr,
+ &dev_attr_cur_eq.attr,
+ &dev_attr_max_cq.attr,
+ &dev_attr_cur_cq.attr,
+ &dev_attr_max_qp.attr,
+ &dev_attr_cur_qp.attr,
+ &dev_attr_max_mr.attr,
+ &dev_attr_cur_mr.attr,
+ &dev_attr_max_mw.attr,
+ &dev_attr_cur_mw.attr,
+ &dev_attr_max_pd.attr,
+ &dev_attr_max_ah.attr,
+ NULL
+};
+
+static struct attribute_group ehca_dev_attr_grp = {
+ .attrs = ehca_dev_attrs
+};
+
+static int ehca_probe(struct platform_device *dev)
+{
+ struct ehca_shca *shca;
+ const u64 *handle;
+ struct ib_pd *ibpd;
+ int ret, i, eq_size;
+ unsigned long flags;
+
+ handle = of_get_property(dev->dev.of_node, "ibm,hca-handle", NULL);
+ if (!handle) {
+ ehca_gen_err("Cannot get eHCA handle for adapter: %s.",
+ dev->dev.of_node->full_name);
+ return -ENODEV;
+ }
+
+ if (!(*handle)) {
+ ehca_gen_err("Wrong eHCA handle for adapter: %s.",
+ dev->dev.of_node->full_name);
+ return -ENODEV;
+ }
+
+ shca = (struct ehca_shca *)ib_alloc_device(sizeof(*shca));
+ if (!shca) {
+ ehca_gen_err("Cannot allocate shca memory.");
+ return -ENOMEM;
+ }
+
+ mutex_init(&shca->modify_mutex);
+ atomic_set(&shca->num_cqs, 0);
+ atomic_set(&shca->num_qps, 0);
+ shca->max_num_qps = ehca_max_qp;
+ shca->max_num_cqs = ehca_max_cq;
+
+ for (i = 0; i < ARRAY_SIZE(shca->sport); i++)
+ spin_lock_init(&shca->sport[i].mod_sqp_lock);
+
+ shca->ofdev = dev;
+ shca->ipz_hca_handle.handle = *handle;
+ dev_set_drvdata(&dev->dev, shca);
+
+ ret = ehca_sense_attributes(shca);
+ if (ret < 0) {
+ ehca_gen_err("Cannot sense eHCA attributes.");
+ goto probe1;
+ }
+
+ ret = ehca_init_device(shca);
+ if (ret) {
+ ehca_gen_err("Cannot init ehca device struct");
+ goto probe1;
+ }
+
+ eq_size = 2 * shca->max_num_cqs + 4 * shca->max_num_qps;
+ /* create event queues */
+ ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, eq_size);
+ if (ret) {
+ ehca_err(&shca->ib_device, "Cannot create EQ.");
+ goto probe1;
+ }
+
+ ret = ehca_create_eq(shca, &shca->neq, EHCA_NEQ, 513);
+ if (ret) {
+ ehca_err(&shca->ib_device, "Cannot create NEQ.");
+ goto probe3;
+ }
+
+ /* create internal protection domain */
+ ibpd = ehca_alloc_pd(&shca->ib_device, (void *)(-1), NULL);
+ if (IS_ERR(ibpd)) {
+ ehca_err(&shca->ib_device, "Cannot create internal PD.");
+ ret = PTR_ERR(ibpd);
+ goto probe4;
+ }
+
+ shca->pd = container_of(ibpd, struct ehca_pd, ib_pd);
+ shca->pd->ib_pd.device = &shca->ib_device;
+
+ /* create internal max MR */
+ ret = ehca_reg_internal_maxmr(shca, shca->pd, &shca->maxmr);
+
+ if (ret) {
+ ehca_err(&shca->ib_device, "Cannot create internal MR ret=%i",
+ ret);
+ goto probe5;
+ }
+
+ ret = ib_register_device(&shca->ib_device, NULL);
+ if (ret) {
+ ehca_err(&shca->ib_device,
+ "ib_register_device() failed ret=%i", ret);
+ goto probe6;
+ }
+
+ /* create AQP1 for port 1 */
+ if (ehca_open_aqp1 == 1) {
+ shca->sport[0].port_state = IB_PORT_DOWN;
+ ret = ehca_create_aqp1(shca, 1);
+ if (ret) {
+ ehca_err(&shca->ib_device,
+ "Cannot create AQP1 for port 1.");
+ goto probe7;
+ }
+ }
+
+ /* create AQP1 for port 2 */
+ if ((ehca_open_aqp1 == 1) && (shca->num_ports == 2)) {
+ shca->sport[1].port_state = IB_PORT_DOWN;
+ ret = ehca_create_aqp1(shca, 2);
+ if (ret) {
+ ehca_err(&shca->ib_device,
+ "Cannot create AQP1 for port 2.");
+ goto probe8;
+ }
+ }
+
+ ret = sysfs_create_group(&dev->dev.kobj, &ehca_dev_attr_grp);
+ if (ret) /* only complain; we can live without attributes */
+ ehca_err(&shca->ib_device,
+ "Cannot create device attributes ret=%d", ret);
+
+ spin_lock_irqsave(&shca_list_lock, flags);
+ list_add(&shca->shca_list, &shca_list);
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+
+ return 0;
+
+probe8:
+ ret = ehca_destroy_aqp1(&shca->sport[0]);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy AQP1 for port 1. ret=%i", ret);
+
+probe7:
+ ib_unregister_device(&shca->ib_device);
+
+probe6:
+ ret = ehca_dereg_internal_maxmr(shca);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal MR. ret=%x", ret);
+
+probe5:
+ ret = ehca_dealloc_pd(&shca->pd->ib_pd);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal PD. ret=%x", ret);
+
+probe4:
+ ret = ehca_destroy_eq(shca, &shca->neq);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy NEQ. ret=%x", ret);
+
+probe3:
+ ret = ehca_destroy_eq(shca, &shca->eq);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy EQ. ret=%x", ret);
+
+probe1:
+ ib_dealloc_device(&shca->ib_device);
+
+ return -EINVAL;
+}
+
+static int ehca_remove(struct platform_device *dev)
+{
+ struct ehca_shca *shca = dev_get_drvdata(&dev->dev);
+ unsigned long flags;
+ int ret;
+
+ sysfs_remove_group(&dev->dev.kobj, &ehca_dev_attr_grp);
+
+ if (ehca_open_aqp1 == 1) {
+ int i;
+ for (i = 0; i < shca->num_ports; i++) {
+ ret = ehca_destroy_aqp1(&shca->sport[i]);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy AQP1 for port %x "
+ "ret=%i", ret, i);
+ }
+ }
+
+ ib_unregister_device(&shca->ib_device);
+
+ ret = ehca_dereg_internal_maxmr(shca);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal MR. ret=%i", ret);
+
+ ret = ehca_dealloc_pd(&shca->pd->ib_pd);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal PD. ret=%i", ret);
+
+ ret = ehca_destroy_eq(shca, &shca->eq);
+ if (ret)
+ ehca_err(&shca->ib_device, "Cannot destroy EQ. ret=%i", ret);
+
+ ret = ehca_destroy_eq(shca, &shca->neq);
+ if (ret)
+ ehca_err(&shca->ib_device, "Canot destroy NEQ. ret=%i", ret);
+
+ ib_dealloc_device(&shca->ib_device);
+
+ spin_lock_irqsave(&shca_list_lock, flags);
+ list_del(&shca->shca_list);
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+
+ return ret;
+}
+
+static struct of_device_id ehca_device_table[] =
+{
+ {
+ .name = "lhca",
+ .compatible = "IBM,lhca",
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, ehca_device_table);
+
+static struct platform_driver ehca_driver = {
+ .probe = ehca_probe,
+ .remove = ehca_remove,
+ .driver = {
+ .name = "ehca",
+ .owner = THIS_MODULE,
+ .groups = ehca_drv_attr_groups,
+ .of_match_table = ehca_device_table,
+ },
+};
+
+void ehca_poll_eqs(unsigned long data)
+{
+ struct ehca_shca *shca;
+
+ spin_lock(&shca_list_lock);
+ list_for_each_entry(shca, &shca_list, shca_list) {
+ if (shca->eq.is_initialized) {
+ /* call deadman proc only if eq ptr does not change */
+ struct ehca_eq *eq = &shca->eq;
+ int max = 3;
+ volatile u64 q_ofs, q_ofs2;
+ unsigned long flags;
+ spin_lock_irqsave(&eq->spinlock, flags);
+ q_ofs = eq->ipz_queue.current_q_offset;
+ spin_unlock_irqrestore(&eq->spinlock, flags);
+ do {
+ spin_lock_irqsave(&eq->spinlock, flags);
+ q_ofs2 = eq->ipz_queue.current_q_offset;
+ spin_unlock_irqrestore(&eq->spinlock, flags);
+ max--;
+ } while (q_ofs == q_ofs2 && max > 0);
+ if (q_ofs == q_ofs2)
+ ehca_process_eq(shca, 0);
+ }
+ }
+ mod_timer(&poll_eqs_timer, round_jiffies(jiffies + HZ));
+ spin_unlock(&shca_list_lock);
+}
+
+static int ehca_mem_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ static unsigned long ehca_dmem_warn_time;
+ unsigned long flags;
+
+ switch (action) {
+ case MEM_CANCEL_OFFLINE:
+ case MEM_CANCEL_ONLINE:
+ case MEM_ONLINE:
+ case MEM_OFFLINE:
+ return NOTIFY_OK;
+ case MEM_GOING_ONLINE:
+ case MEM_GOING_OFFLINE:
+ /* only ok if no hca is attached to the lpar */
+ spin_lock_irqsave(&shca_list_lock, flags);
+ if (list_empty(&shca_list)) {
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+ return NOTIFY_OK;
+ } else {
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+ if (printk_timed_ratelimit(&ehca_dmem_warn_time,
+ 30 * 1000))
+ ehca_gen_err("DMEM operations are not allowed"
+ "in conjunction with eHCA");
+ return NOTIFY_BAD;
+ }
+ }
+ return NOTIFY_OK;
+}
+
+static struct notifier_block ehca_mem_nb = {
+ .notifier_call = ehca_mem_notifier,
+};
+
+static int __init ehca_module_init(void)
+{
+ int ret;
+
+ printk(KERN_INFO "eHCA Infiniband Device Driver "
+ "(Version " HCAD_VERSION ")\n");
+
+ ret = ehca_create_comp_pool();
+ if (ret) {
+ ehca_gen_err("Cannot create comp pool.");
+ return ret;
+ }
+
+ ret = ehca_create_slab_caches();
+ if (ret) {
+ ehca_gen_err("Cannot create SLAB caches");
+ ret = -ENOMEM;
+ goto module_init1;
+ }
+
+ ret = ehca_create_busmap();
+ if (ret) {
+ ehca_gen_err("Cannot create busmap.");
+ goto module_init2;
+ }
+
+ ret = ibmebus_register_driver(&ehca_driver);
+ if (ret) {
+ ehca_gen_err("Cannot register eHCA device driver");
+ ret = -EINVAL;
+ goto module_init3;
+ }
+
+ ret = register_memory_notifier(&ehca_mem_nb);
+ if (ret) {
+ ehca_gen_err("Failed registering memory add/remove notifier");
+ goto module_init4;
+ }
+
+ if (ehca_poll_all_eqs != 1) {
+ ehca_gen_err("WARNING!!!");
+ ehca_gen_err("It is possible to lose interrupts.");
+ } else {
+ init_timer(&poll_eqs_timer);
+ poll_eqs_timer.function = ehca_poll_eqs;
+ poll_eqs_timer.expires = jiffies + HZ;
+ add_timer(&poll_eqs_timer);
+ }
+
+ return 0;
+
+module_init4:
+ ibmebus_unregister_driver(&ehca_driver);
+
+module_init3:
+ ehca_destroy_busmap();
+
+module_init2:
+ ehca_destroy_slab_caches();
+
+module_init1:
+ ehca_destroy_comp_pool();
+ return ret;
+};
+
+static void __exit ehca_module_exit(void)
+{
+ if (ehca_poll_all_eqs == 1)
+ del_timer_sync(&poll_eqs_timer);
+
+ ibmebus_unregister_driver(&ehca_driver);
+
+ unregister_memory_notifier(&ehca_mem_nb);
+
+ ehca_destroy_busmap();
+
+ ehca_destroy_slab_caches();
+
+ ehca_destroy_comp_pool();
+
+ idr_destroy(&ehca_cq_idr);
+ idr_destroy(&ehca_qp_idr);
+};
+
+module_init(ehca_module_init);
+module_exit(ehca_module_exit);
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * mcast functions
+ *
+ * Authors: Khadija Souissi <souissik@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+#define MAX_MC_LID 0xFFFE
+#define MIN_MC_LID 0xC000 /* Multicast limits */
+#define EHCA_VALID_MULTICAST_GID(gid) ((gid)[0] == 0xFF)
+#define EHCA_VALID_MULTICAST_LID(lid) \
+ (((lid) >= MIN_MC_LID) && ((lid) <= MAX_MC_LID))
+
+int ehca_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
+ ib_device);
+ union ib_gid my_gid;
+ u64 subnet_prefix, interface_id, h_ret;
+
+ if (ibqp->qp_type != IB_QPT_UD) {
+ ehca_err(ibqp->device, "invalid qp_type=%x", ibqp->qp_type);
+ return -EINVAL;
+ }
+
+ if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+ ehca_err(ibqp->device, "invalid mulitcast gid");
+ return -EINVAL;
+ } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+ ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
+ return -EINVAL;
+ }
+
+ memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
+
+ subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
+ interface_id = be64_to_cpu(my_gid.global.interface_id);
+ h_ret = hipz_h_attach_mcqp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ my_qp->galpas.kernel,
+ lid, subnet_prefix, interface_id);
+ if (h_ret != H_SUCCESS)
+ ehca_err(ibqp->device,
+ "ehca_qp=%p qp_num=%x hipz_h_attach_mcqp() failed "
+ "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
+
+ return ehca2ib_return_code(h_ret);
+}
+
+int ehca_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca = container_of(ibqp->pd->device,
+ struct ehca_shca, ib_device);
+ union ib_gid my_gid;
+ u64 subnet_prefix, interface_id, h_ret;
+
+ if (ibqp->qp_type != IB_QPT_UD) {
+ ehca_err(ibqp->device, "invalid qp_type %x", ibqp->qp_type);
+ return -EINVAL;
+ }
+
+ if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+ ehca_err(ibqp->device, "invalid mulitcast gid");
+ return -EINVAL;
+ } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+ ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
+ return -EINVAL;
+ }
+
+ memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
+
+ subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
+ interface_id = be64_to_cpu(my_gid.global.interface_id);
+ h_ret = hipz_h_detach_mcqp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ my_qp->galpas.kernel,
+ lid, subnet_prefix, interface_id);
+ if (h_ret != H_SUCCESS)
+ ehca_err(ibqp->device,
+ "ehca_qp=%p qp_num=%x hipz_h_detach_mcqp() failed "
+ "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
+
+ return ehca2ib_return_code(h_ret);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * MR/MW functions
+ *
+ * Authors: Dietmar Decker <ddecker@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+#include <rdma/ib_umem.h>
+
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "hcp_if.h"
+#include "hipz_hw.h"
+
+#define NUM_CHUNKS(length, chunk_size) \
+ (((length) + (chunk_size - 1)) / (chunk_size))
+
+/* max number of rpages (per hcall register_rpages) */
+#define MAX_RPAGES 512
+
+/* DMEM toleration management */
+#define EHCA_SECTSHIFT SECTION_SIZE_BITS
+#define EHCA_SECTSIZE (1UL << EHCA_SECTSHIFT)
+#define EHCA_HUGEPAGESHIFT 34
+#define EHCA_HUGEPAGE_SIZE (1UL << EHCA_HUGEPAGESHIFT)
+#define EHCA_HUGEPAGE_PFN_MASK ((EHCA_HUGEPAGE_SIZE - 1) >> PAGE_SHIFT)
+#define EHCA_INVAL_ADDR 0xFFFFFFFFFFFFFFFFULL
+#define EHCA_DIR_INDEX_SHIFT 13 /* 8k Entries in 64k block */
+#define EHCA_TOP_INDEX_SHIFT (EHCA_DIR_INDEX_SHIFT * 2)
+#define EHCA_MAP_ENTRIES (1 << EHCA_DIR_INDEX_SHIFT)
+#define EHCA_TOP_MAP_SIZE (0x10000) /* currently fixed map size */
+#define EHCA_DIR_MAP_SIZE (0x10000)
+#define EHCA_ENT_MAP_SIZE (0x10000)
+#define EHCA_INDEX_MASK (EHCA_MAP_ENTRIES - 1)
+
+static unsigned long ehca_mr_len;
+
+/*
+ * Memory map data structures
+ */
+struct ehca_dir_bmap {
+ u64 ent[EHCA_MAP_ENTRIES];
+};
+struct ehca_top_bmap {
+ struct ehca_dir_bmap *dir[EHCA_MAP_ENTRIES];
+};
+struct ehca_bmap {
+ struct ehca_top_bmap *top[EHCA_MAP_ENTRIES];
+};
+
+static struct ehca_bmap *ehca_bmap;
+
+static struct kmem_cache *mr_cache;
+static struct kmem_cache *mw_cache;
+
+enum ehca_mr_pgsize {
+ EHCA_MR_PGSIZE4K = 0x1000L,
+ EHCA_MR_PGSIZE64K = 0x10000L,
+ EHCA_MR_PGSIZE1M = 0x100000L,
+ EHCA_MR_PGSIZE16M = 0x1000000L
+};
+
+#define EHCA_MR_PGSHIFT4K 12
+#define EHCA_MR_PGSHIFT64K 16
+#define EHCA_MR_PGSHIFT1M 20
+#define EHCA_MR_PGSHIFT16M 24
+
+static u64 ehca_map_vaddr(void *caddr);
+
+static u32 ehca_encode_hwpage_size(u32 pgsize)
+{
+ int log = ilog2(pgsize);
+ WARN_ON(log < 12 || log > 24 || log & 3);
+ return (log - 12) / 4;
+}
+
+static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca)
+{
+ return rounddown_pow_of_two(shca->hca_cap_mr_pgsize);
+}
+
+static struct ehca_mr *ehca_mr_new(void)
+{
+ struct ehca_mr *me;
+
+ me = kmem_cache_zalloc(mr_cache, GFP_KERNEL);
+ if (me)
+ spin_lock_init(&me->mrlock);
+ else
+ ehca_gen_err("alloc failed");
+
+ return me;
+}
+
+static void ehca_mr_delete(struct ehca_mr *me)
+{
+ kmem_cache_free(mr_cache, me);
+}
+
+static struct ehca_mw *ehca_mw_new(void)
+{
+ struct ehca_mw *me;
+
+ me = kmem_cache_zalloc(mw_cache, GFP_KERNEL);
+ if (me)
+ spin_lock_init(&me->mwlock);
+ else
+ ehca_gen_err("alloc failed");
+
+ return me;
+}
+
+static void ehca_mw_delete(struct ehca_mw *me)
+{
+ kmem_cache_free(mw_cache, me);
+}
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
+{
+ struct ib_mr *ib_mr;
+ int ret;
+ struct ehca_mr *e_maxmr;
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+
+ if (shca->maxmr) {
+ e_maxmr = ehca_mr_new();
+ if (!e_maxmr) {
+ ehca_err(&shca->ib_device, "out of memory");
+ ib_mr = ERR_PTR(-ENOMEM);
+ goto get_dma_mr_exit0;
+ }
+
+ ret = ehca_reg_maxmr(shca, e_maxmr,
+ (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)),
+ mr_access_flags, e_pd,
+ &e_maxmr->ib.ib_mr.lkey,
+ &e_maxmr->ib.ib_mr.rkey);
+ if (ret) {
+ ehca_mr_delete(e_maxmr);
+ ib_mr = ERR_PTR(ret);
+ goto get_dma_mr_exit0;
+ }
+ ib_mr = &e_maxmr->ib.ib_mr;
+ } else {
+ ehca_err(&shca->ib_device, "no internal max-MR exist!");
+ ib_mr = ERR_PTR(-EINVAL);
+ goto get_dma_mr_exit0;
+ }
+
+get_dma_mr_exit0:
+ if (IS_ERR(ib_mr))
+ ehca_err(&shca->ib_device, "h_ret=%li pd=%p mr_access_flags=%x",
+ PTR_ERR(ib_mr), pd, mr_access_flags);
+ return ib_mr;
+} /* end ehca_get_dma_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ int mr_access_flags,
+ u64 *iova_start)
+{
+ struct ib_mr *ib_mr;
+ int ret;
+ struct ehca_mr *e_mr;
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+
+ u64 size;
+
+ if ((num_phys_buf <= 0) || !phys_buf_array) {
+ ehca_err(pd->device, "bad input values: num_phys_buf=%x "
+ "phys_buf_array=%p", num_phys_buf, phys_buf_array);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_phys_mr_exit0;
+ }
+ if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_phys_mr_exit0;
+ }
+
+ /* check physical buffer list and calculate size */
+ ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
+ iova_start, &size);
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_phys_mr_exit0;
+ }
+ if ((size == 0) ||
+ (((u64)iova_start + size) < (u64)iova_start)) {
+ ehca_err(pd->device, "bad input values: size=%llx iova_start=%p",
+ size, iova_start);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_phys_mr_exit0;
+ }
+
+ e_mr = ehca_mr_new();
+ if (!e_mr) {
+ ehca_err(pd->device, "out of memory");
+ ib_mr = ERR_PTR(-ENOMEM);
+ goto reg_phys_mr_exit0;
+ }
+
+ /* register MR on HCA */
+ if (ehca_mr_is_maxmr(size, iova_start)) {
+ e_mr->flags |= EHCA_MR_FLAG_MAXMR;
+ ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
+ e_pd, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey);
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_phys_mr_exit1;
+ }
+ } else {
+ struct ehca_mr_pginfo pginfo;
+ u32 num_kpages;
+ u32 num_hwpages;
+ u64 hw_pgsize;
+
+ num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
+ PAGE_SIZE);
+ /* for kernel space we try most possible pgsize */
+ hw_pgsize = ehca_get_max_hwpage_size(shca);
+ num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size,
+ hw_pgsize);
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_PHYS;
+ pginfo.num_kpages = num_kpages;
+ pginfo.hwpage_size = hw_pgsize;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.u.phy.num_phys_buf = num_phys_buf;
+ pginfo.u.phy.phys_buf_array = phys_buf_array;
+ pginfo.next_hwpage =
+ ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
+
+ ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
+ e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_phys_mr_exit1;
+ }
+ }
+
+ /* successful registration of all pages */
+ return &e_mr->ib.ib_mr;
+
+reg_phys_mr_exit1:
+ ehca_mr_delete(e_mr);
+reg_phys_mr_exit0:
+ if (IS_ERR(ib_mr))
+ ehca_err(pd->device, "h_ret=%li pd=%p phys_buf_array=%p "
+ "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
+ PTR_ERR(ib_mr), pd, phys_buf_array,
+ num_phys_buf, mr_access_flags, iova_start);
+ return ib_mr;
+} /* end ehca_reg_phys_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+ u64 virt, int mr_access_flags,
+ struct ib_udata *udata)
+{
+ struct ib_mr *ib_mr;
+ struct ehca_mr *e_mr;
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_mr_pginfo pginfo;
+ int ret, page_shift;
+ u32 num_kpages;
+ u32 num_hwpages;
+ u64 hwpage_size;
+
+ if (!pd) {
+ ehca_gen_err("bad pd=%p", pd);
+ return ERR_PTR(-EFAULT);
+ }
+
+ if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_user_mr_exit0;
+ }
+
+ if (length == 0 || virt + length < virt) {
+ ehca_err(pd->device, "bad input values: length=%llx "
+ "virt_base=%llx", length, virt);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_user_mr_exit0;
+ }
+
+ e_mr = ehca_mr_new();
+ if (!e_mr) {
+ ehca_err(pd->device, "out of memory");
+ ib_mr = ERR_PTR(-ENOMEM);
+ goto reg_user_mr_exit0;
+ }
+
+ e_mr->umem = ib_umem_get(pd->uobject->context, start, length,
+ mr_access_flags, 0);
+ if (IS_ERR(e_mr->umem)) {
+ ib_mr = (void *)e_mr->umem;
+ goto reg_user_mr_exit1;
+ }
+
+ if (e_mr->umem->page_size != PAGE_SIZE) {
+ ehca_err(pd->device, "page size not supported, "
+ "e_mr->umem->page_size=%x", e_mr->umem->page_size);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_user_mr_exit2;
+ }
+
+ /* determine number of MR pages */
+ num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
+ /* select proper hw_pgsize */
+ page_shift = PAGE_SHIFT;
+ if (e_mr->umem->hugetlb) {
+ /* determine page_shift, clamp between 4K and 16M */
+ page_shift = (fls64(length - 1) + 3) & ~3;
+ page_shift = min(max(page_shift, EHCA_MR_PGSHIFT4K),
+ EHCA_MR_PGSHIFT16M);
+ }
+ hwpage_size = 1UL << page_shift;
+
+ /* now that we have the desired page size, shift until it's
+ * supported, too. 4K is always supported, so this terminates.
+ */
+ while (!(hwpage_size & shca->hca_cap_mr_pgsize))
+ hwpage_size >>= 4;
+
+reg_user_mr_fallback:
+ num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size);
+ /* register MR on HCA */
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_USER;
+ pginfo.hwpage_size = hwpage_size;
+ pginfo.num_kpages = num_kpages;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.u.usr.region = e_mr->umem;
+ pginfo.next_hwpage = ib_umem_offset(e_mr->umem) / hwpage_size;
+ pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl;
+ ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
+ e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
+ if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) {
+ ehca_warn(pd->device, "failed to register mr "
+ "with hwpage_size=%llx", hwpage_size);
+ ehca_info(pd->device, "try to register mr with "
+ "kpage_size=%lx", PAGE_SIZE);
+ /*
+ * this means kpages are not contiguous for a hw page
+ * try kernel page size as fallback solution
+ */
+ hwpage_size = PAGE_SIZE;
+ goto reg_user_mr_fallback;
+ }
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_user_mr_exit2;
+ }
+
+ /* successful registration of all pages */
+ return &e_mr->ib.ib_mr;
+
+reg_user_mr_exit2:
+ ib_umem_release(e_mr->umem);
+reg_user_mr_exit1:
+ ehca_mr_delete(e_mr);
+reg_user_mr_exit0:
+ if (IS_ERR(ib_mr))
+ ehca_err(pd->device, "rc=%li pd=%p mr_access_flags=%x udata=%p",
+ PTR_ERR(ib_mr), pd, mr_access_flags, udata);
+ return ib_mr;
+} /* end ehca_reg_user_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_rereg_phys_mr(struct ib_mr *mr,
+ int mr_rereg_mask,
+ struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ int mr_access_flags,
+ u64 *iova_start)
+{
+ int ret;
+
+ struct ehca_shca *shca =
+ container_of(mr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+ u64 new_size;
+ u64 *new_start;
+ u32 new_acl;
+ struct ehca_pd *new_pd;
+ u32 tmp_lkey, tmp_rkey;
+ unsigned long sl_flags;
+ u32 num_kpages = 0;
+ u32 num_hwpages = 0;
+ struct ehca_mr_pginfo pginfo;
+
+ if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
+ /* TODO not supported, because PHYP rereg hCall needs pages */
+ ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
+ "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+
+ if (mr_rereg_mask & IB_MR_REREG_PD) {
+ if (!pd) {
+ ehca_err(mr->device, "rereg with bad pd, pd=%p "
+ "mr_rereg_mask=%x", pd, mr_rereg_mask);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ }
+
+ if ((mr_rereg_mask &
+ ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
+ (mr_rereg_mask == 0)) {
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+
+ /* check other parameters */
+ if (e_mr == shca->maxmr) {
+ /* should be impossible, however reject to be sure */
+ ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
+ "shca->maxmr=%p mr->lkey=%x",
+ mr, shca->maxmr, mr->lkey);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
+ if (e_mr->flags & EHCA_MR_FLAG_FMR) {
+ ehca_err(mr->device, "not supported for FMR, mr=%p "
+ "flags=%x", mr, e_mr->flags);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ if (!phys_buf_array || num_phys_buf <= 0) {
+ ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
+ " phys_buf_array=%p num_phys_buf=%x",
+ mr_rereg_mask, phys_buf_array, num_phys_buf);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ }
+ if ((mr_rereg_mask & IB_MR_REREG_ACCESS) && /* change ACL */
+ (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
+ "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+
+ /* set requested values dependent on rereg request */
+ spin_lock_irqsave(&e_mr->mrlock, sl_flags);
+ new_start = e_mr->start;
+ new_size = e_mr->size;
+ new_acl = e_mr->acl;
+ new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
+
+ if (mr_rereg_mask & IB_MR_REREG_TRANS) {
+ u64 hw_pgsize = ehca_get_max_hwpage_size(shca);
+
+ new_start = iova_start; /* change address */
+ /* check physical buffer list and calculate size */
+ ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
+ num_phys_buf, iova_start,
+ &new_size);
+ if (ret)
+ goto rereg_phys_mr_exit1;
+ if ((new_size == 0) ||
+ (((u64)iova_start + new_size) < (u64)iova_start)) {
+ ehca_err(mr->device, "bad input values: new_size=%llx "
+ "iova_start=%p", new_size, iova_start);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit1;
+ }
+ num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
+ new_size, PAGE_SIZE);
+ num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) +
+ new_size, hw_pgsize);
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_PHYS;
+ pginfo.num_kpages = num_kpages;
+ pginfo.hwpage_size = hw_pgsize;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.u.phy.num_phys_buf = num_phys_buf;
+ pginfo.u.phy.phys_buf_array = phys_buf_array;
+ pginfo.next_hwpage =
+ ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
+ }
+ if (mr_rereg_mask & IB_MR_REREG_ACCESS)
+ new_acl = mr_access_flags;
+ if (mr_rereg_mask & IB_MR_REREG_PD)
+ new_pd = container_of(pd, struct ehca_pd, ib_pd);
+
+ ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
+ new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
+ if (ret)
+ goto rereg_phys_mr_exit1;
+
+ /* successful reregistration */
+ if (mr_rereg_mask & IB_MR_REREG_PD)
+ mr->pd = pd;
+ mr->lkey = tmp_lkey;
+ mr->rkey = tmp_rkey;
+
+rereg_phys_mr_exit1:
+ spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
+rereg_phys_mr_exit0:
+ if (ret)
+ ehca_err(mr->device, "ret=%i mr=%p mr_rereg_mask=%x pd=%p "
+ "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
+ "iova_start=%p",
+ ret, mr, mr_rereg_mask, pd, phys_buf_array,
+ num_phys_buf, mr_access_flags, iova_start);
+ return ret;
+} /* end ehca_rereg_phys_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(mr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+ unsigned long sl_flags;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
+ "e_mr->flags=%x", mr, e_mr, e_mr->flags);
+ ret = -EINVAL;
+ goto query_mr_exit0;
+ }
+
+ memset(mr_attr, 0, sizeof(struct ib_mr_attr));
+ spin_lock_irqsave(&e_mr->mrlock, sl_flags);
+
+ h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lli mr=%p "
+ "hca_hndl=%llx mr_hndl=%llx lkey=%x",
+ h_ret, mr, shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle, mr->lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto query_mr_exit1;
+ }
+ mr_attr->pd = mr->pd;
+ mr_attr->device_virt_addr = hipzout.vaddr;
+ mr_attr->size = hipzout.len;
+ mr_attr->lkey = hipzout.lkey;
+ mr_attr->rkey = hipzout.rkey;
+ ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
+
+query_mr_exit1:
+ spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
+query_mr_exit0:
+ if (ret)
+ ehca_err(mr->device, "ret=%i mr=%p mr_attr=%p",
+ ret, mr, mr_attr);
+ return ret;
+} /* end ehca_query_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dereg_mr(struct ib_mr *mr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(mr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+
+ if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
+ "e_mr->flags=%x", mr, e_mr, e_mr->flags);
+ ret = -EINVAL;
+ goto dereg_mr_exit0;
+ } else if (e_mr == shca->maxmr) {
+ /* should be impossible, however reject to be sure */
+ ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
+ "shca->maxmr=%p mr->lkey=%x",
+ mr, shca->maxmr, mr->lkey);
+ ret = -EINVAL;
+ goto dereg_mr_exit0;
+ }
+
+ /* TODO: BUSY: MR still has bound window(s) */
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lli shca=%p "
+ "e_mr=%p hca_hndl=%llx mr_hndl=%llx mr->lkey=%x",
+ h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle, mr->lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto dereg_mr_exit0;
+ }
+
+ if (e_mr->umem)
+ ib_umem_release(e_mr->umem);
+
+ /* successful deregistration */
+ ehca_mr_delete(e_mr);
+
+dereg_mr_exit0:
+ if (ret)
+ ehca_err(mr->device, "ret=%i mr=%p", ret, mr);
+ return ret;
+} /* end ehca_dereg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
+{
+ struct ib_mw *ib_mw;
+ u64 h_ret;
+ struct ehca_mw *e_mw;
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_mw_hipzout_parms hipzout;
+
+ if (type != IB_MW_TYPE_1)
+ return ERR_PTR(-EINVAL);
+
+ e_mw = ehca_mw_new();
+ if (!e_mw) {
+ ib_mw = ERR_PTR(-ENOMEM);
+ goto alloc_mw_exit0;
+ }
+
+ h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
+ e_pd->fw_pd, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lli "
+ "shca=%p hca_hndl=%llx mw=%p",
+ h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
+ ib_mw = ERR_PTR(ehca2ib_return_code(h_ret));
+ goto alloc_mw_exit1;
+ }
+ /* successful MW allocation */
+ e_mw->ipz_mw_handle = hipzout.handle;
+ e_mw->ib_mw.rkey = hipzout.rkey;
+ return &e_mw->ib_mw;
+
+alloc_mw_exit1:
+ ehca_mw_delete(e_mw);
+alloc_mw_exit0:
+ if (IS_ERR(ib_mw))
+ ehca_err(pd->device, "h_ret=%li pd=%p", PTR_ERR(ib_mw), pd);
+ return ib_mw;
+} /* end ehca_alloc_mw() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_bind_mw(struct ib_qp *qp,
+ struct ib_mw *mw,
+ struct ib_mw_bind *mw_bind)
+{
+ /* TODO: not supported up to now */
+ ehca_gen_err("bind MW currently not supported by HCAD");
+
+ return -EPERM;
+} /* end ehca_bind_mw() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dealloc_mw(struct ib_mw *mw)
+{
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(mw->device, struct ehca_shca, ib_device);
+ struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
+
+ h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lli shca=%p "
+ "mw=%p rkey=%x hca_hndl=%llx mw_hndl=%llx",
+ h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
+ e_mw->ipz_mw_handle.handle);
+ return ehca2ib_return_code(h_ret);
+ }
+ /* successful deallocation */
+ ehca_mw_delete(e_mw);
+ return 0;
+} /* end ehca_dealloc_mw() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
+ int mr_access_flags,
+ struct ib_fmr_attr *fmr_attr)
+{
+ struct ib_fmr *ib_fmr;
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_mr *e_fmr;
+ int ret;
+ u32 tmp_lkey, tmp_rkey;
+ struct ehca_mr_pginfo pginfo;
+ u64 hw_pgsize;
+
+ /* check other parameters */
+ if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+ if (mr_access_flags & IB_ACCESS_MW_BIND) {
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+ if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
+ ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
+ "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
+ fmr_attr->max_pages, fmr_attr->max_maps,
+ fmr_attr->page_shift);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+
+ hw_pgsize = 1 << fmr_attr->page_shift;
+ if (!(hw_pgsize & shca->hca_cap_mr_pgsize)) {
+ ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
+ fmr_attr->page_shift);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+
+ e_fmr = ehca_mr_new();
+ if (!e_fmr) {
+ ib_fmr = ERR_PTR(-ENOMEM);
+ goto alloc_fmr_exit0;
+ }
+ e_fmr->flags |= EHCA_MR_FLAG_FMR;
+
+ /* register MR on HCA */
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.hwpage_size = hw_pgsize;
+ /*
+ * pginfo.num_hwpages==0, ie register_rpages() will not be called
+ * but deferred to map_phys_fmr()
+ */
+ ret = ehca_reg_mr(shca, e_fmr, NULL,
+ fmr_attr->max_pages * (1 << fmr_attr->page_shift),
+ mr_access_flags, e_pd, &pginfo,
+ &tmp_lkey, &tmp_rkey, EHCA_REG_MR);
+ if (ret) {
+ ib_fmr = ERR_PTR(ret);
+ goto alloc_fmr_exit1;
+ }
+
+ /* successful */
+ e_fmr->hwpage_size = hw_pgsize;
+ e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
+ e_fmr->fmr_max_pages = fmr_attr->max_pages;
+ e_fmr->fmr_max_maps = fmr_attr->max_maps;
+ e_fmr->fmr_map_cnt = 0;
+ return &e_fmr->ib.ib_fmr;
+
+alloc_fmr_exit1:
+ ehca_mr_delete(e_fmr);
+alloc_fmr_exit0:
+ return ib_fmr;
+} /* end ehca_alloc_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_map_phys_fmr(struct ib_fmr *fmr,
+ u64 *page_list,
+ int list_len,
+ u64 iova)
+{
+ int ret;
+ struct ehca_shca *shca =
+ container_of(fmr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
+ struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
+ struct ehca_mr_pginfo pginfo;
+ u32 tmp_lkey, tmp_rkey;
+
+ if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
+ e_fmr, e_fmr->flags);
+ ret = -EINVAL;
+ goto map_phys_fmr_exit0;
+ }
+ ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
+ if (ret)
+ goto map_phys_fmr_exit0;
+ if (iova % e_fmr->fmr_page_size) {
+ /* only whole-numbered pages */
+ ehca_err(fmr->device, "bad iova, iova=%llx fmr_page_size=%x",
+ iova, e_fmr->fmr_page_size);
+ ret = -EINVAL;
+ goto map_phys_fmr_exit0;
+ }
+ if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
+ /* HCAD does not limit the maps, however trace this anyway */
+ ehca_info(fmr->device, "map limit exceeded, fmr=%p "
+ "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
+ fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
+ }
+
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_FMR;
+ pginfo.num_kpages = list_len;
+ pginfo.hwpage_size = e_fmr->hwpage_size;
+ pginfo.num_hwpages =
+ list_len * e_fmr->fmr_page_size / pginfo.hwpage_size;
+ pginfo.u.fmr.page_list = page_list;
+ pginfo.next_hwpage =
+ (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size;
+ pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
+
+ ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
+ list_len * e_fmr->fmr_page_size,
+ e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
+ if (ret)
+ goto map_phys_fmr_exit0;
+
+ /* successful reregistration */
+ e_fmr->fmr_map_cnt++;
+ e_fmr->ib.ib_fmr.lkey = tmp_lkey;
+ e_fmr->ib.ib_fmr.rkey = tmp_rkey;
+ return 0;
+
+map_phys_fmr_exit0:
+ if (ret)
+ ehca_err(fmr->device, "ret=%i fmr=%p page_list=%p list_len=%x "
+ "iova=%llx", ret, fmr, page_list, list_len, iova);
+ return ret;
+} /* end ehca_map_phys_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_unmap_fmr(struct list_head *fmr_list)
+{
+ int ret = 0;
+ struct ib_fmr *ib_fmr;
+ struct ehca_shca *shca = NULL;
+ struct ehca_shca *prev_shca;
+ struct ehca_mr *e_fmr;
+ u32 num_fmr = 0;
+ u32 unmap_fmr_cnt = 0;
+
+ /* check all FMR belong to same SHCA, and check internal flag */
+ list_for_each_entry(ib_fmr, fmr_list, list) {
+ prev_shca = shca;
+ shca = container_of(ib_fmr->device, struct ehca_shca,
+ ib_device);
+ e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
+ if ((shca != prev_shca) && prev_shca) {
+ ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
+ "prev_shca=%p e_fmr=%p",
+ shca, prev_shca, e_fmr);
+ ret = -EINVAL;
+ goto unmap_fmr_exit0;
+ }
+ if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
+ "e_fmr->flags=%x", e_fmr, e_fmr->flags);
+ ret = -EINVAL;
+ goto unmap_fmr_exit0;
+ }
+ num_fmr++;
+ }
+
+ /* loop over all FMRs to unmap */
+ list_for_each_entry(ib_fmr, fmr_list, list) {
+ unmap_fmr_cnt++;
+ e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
+ shca = container_of(ib_fmr->device, struct ehca_shca,
+ ib_device);
+ ret = ehca_unmap_one_fmr(shca, e_fmr);
+ if (ret) {
+ /* unmap failed, stop unmapping of rest of FMRs */
+ ehca_err(&shca->ib_device, "unmap of one FMR failed, "
+ "stop rest, e_fmr=%p num_fmr=%x "
+ "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
+ unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
+ goto unmap_fmr_exit0;
+ }
+ }
+
+unmap_fmr_exit0:
+ if (ret)
+ ehca_gen_err("ret=%i fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
+ ret, fmr_list, num_fmr, unmap_fmr_cnt);
+ return ret;
+} /* end ehca_unmap_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dealloc_fmr(struct ib_fmr *fmr)
+{
+ int ret;
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(fmr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
+
+ if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
+ e_fmr, e_fmr->flags);
+ ret = -EINVAL;
+ goto free_fmr_exit0;
+ }
+
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lli e_fmr=%p "
+ "hca_hndl=%llx fmr_hndl=%llx fmr->lkey=%x",
+ h_ret, e_fmr, shca->ipz_hca_handle.handle,
+ e_fmr->ipz_mr_handle.handle, fmr->lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto free_fmr_exit0;
+ }
+ /* successful deregistration */
+ ehca_mr_delete(e_fmr);
+ return 0;
+
+free_fmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i fmr=%p", ret, fmr);
+ return ret;
+} /* end ehca_dealloc_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo);
+
+int ehca_reg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey, /*OUT*/
+ u32 *rkey, /*OUT*/
+ enum ehca_reg_type reg_type)
+{
+ int ret;
+ u64 h_ret;
+ u32 hipz_acl;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
+ if (ehca_use_hp_mr == 1)
+ hipz_acl |= 0x00000001;
+
+ h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
+ (u64)iova_start, size, hipz_acl,
+ e_pd->fw_pd, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lli "
+ "hca_hndl=%llx", h_ret, shca->ipz_hca_handle.handle);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_reg_mr_exit0;
+ }
+
+ e_mr->ipz_mr_handle = hipzout.handle;
+
+ if (reg_type == EHCA_REG_BUSMAP_MR)
+ ret = ehca_reg_bmap_mr_rpages(shca, e_mr, pginfo);
+ else if (reg_type == EHCA_REG_MR)
+ ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
+ else
+ ret = -EINVAL;
+
+ if (ret)
+ goto ehca_reg_mr_exit1;
+
+ /* successful registration */
+ e_mr->num_kpages = pginfo->num_kpages;
+ e_mr->num_hwpages = pginfo->num_hwpages;
+ e_mr->hwpage_size = pginfo->hwpage_size;
+ e_mr->start = iova_start;
+ e_mr->size = size;
+ e_mr->acl = acl;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ return 0;
+
+ehca_reg_mr_exit1:
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "h_ret=%lli shca=%p e_mr=%p "
+ "iova_start=%p size=%llx acl=%x e_pd=%p lkey=%x "
+ "pginfo=%p num_kpages=%llx num_hwpages=%llx ret=%i",
+ h_ret, shca, e_mr, iova_start, size, acl, e_pd,
+ hipzout.lkey, pginfo, pginfo->num_kpages,
+ pginfo->num_hwpages, ret);
+ ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
+ "not recoverable");
+ }
+ehca_reg_mr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
+ "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
+ "num_kpages=%llx num_hwpages=%llx",
+ ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
+ pginfo->num_kpages, pginfo->num_hwpages);
+ return ret;
+} /* end ehca_reg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ int ret = 0;
+ u64 h_ret;
+ u32 rnum;
+ u64 rpage;
+ u32 i;
+ u64 *kpage;
+
+ if (!pginfo->num_hwpages) /* in case of fmr */
+ return 0;
+
+ kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!kpage) {
+ ehca_err(&shca->ib_device, "kpage alloc failed");
+ ret = -ENOMEM;
+ goto ehca_reg_mr_rpages_exit0;
+ }
+
+ /* max MAX_RPAGES ehca mr pages per register call */
+ for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
+
+ if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
+ rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */
+ if (rnum == 0)
+ rnum = MAX_RPAGES; /* last shot is full */
+ } else
+ rnum = MAX_RPAGES;
+
+ ret = ehca_set_pagebuf(pginfo, rnum, kpage);
+ if (ret) {
+ ehca_err(&shca->ib_device, "ehca_set_pagebuf "
+ "bad rc, ret=%i rnum=%x kpage=%p",
+ ret, rnum, kpage);
+ goto ehca_reg_mr_rpages_exit1;
+ }
+
+ if (rnum > 1) {
+ rpage = __pa(kpage);
+ if (!rpage) {
+ ehca_err(&shca->ib_device, "kpage=%p i=%x",
+ kpage, i);
+ ret = -EFAULT;
+ goto ehca_reg_mr_rpages_exit1;
+ }
+ } else
+ rpage = *kpage;
+
+ h_ret = hipz_h_register_rpage_mr(
+ shca->ipz_hca_handle, e_mr,
+ ehca_encode_hwpage_size(pginfo->hwpage_size),
+ 0, rpage, rnum);
+
+ if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
+ /*
+ * check for 'registration complete'==H_SUCCESS
+ * and for 'page registered'==H_PAGE_REGISTERED
+ */
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "last "
+ "hipz_reg_rpage_mr failed, h_ret=%lli "
+ "e_mr=%p i=%x hca_hndl=%llx mr_hndl=%llx"
+ " lkey=%x", h_ret, e_mr, i,
+ shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle,
+ e_mr->ib.ib_mr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ break;
+ } else
+ ret = 0;
+ } else if (h_ret != H_PAGE_REGISTERED) {
+ ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
+ "h_ret=%lli e_mr=%p i=%x lkey=%x hca_hndl=%llx "
+ "mr_hndl=%llx", h_ret, e_mr, i,
+ e_mr->ib.ib_mr.lkey,
+ shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle);
+ ret = ehca2ib_return_code(h_ret);
+ break;
+ } else
+ ret = 0;
+ } /* end for(i) */
+
+
+ehca_reg_mr_rpages_exit1:
+ ehca_free_fw_ctrlblock(kpage);
+ehca_reg_mr_rpages_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p pginfo=%p "
+ "num_kpages=%llx num_hwpages=%llx", ret, shca, e_mr,
+ pginfo, pginfo->num_kpages, pginfo->num_hwpages);
+ return ret;
+} /* end ehca_reg_mr_rpages() */
+
+/*----------------------------------------------------------------------*/
+
+inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ u32 acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey, /*OUT*/
+ u32 *rkey) /*OUT*/
+{
+ int ret;
+ u64 h_ret;
+ u32 hipz_acl;
+ u64 *kpage;
+ u64 rpage;
+ struct ehca_mr_pginfo pginfo_save;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
+
+ kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!kpage) {
+ ehca_err(&shca->ib_device, "kpage alloc failed");
+ ret = -ENOMEM;
+ goto ehca_rereg_mr_rereg1_exit0;
+ }
+
+ pginfo_save = *pginfo;
+ ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);
+ if (ret) {
+ ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
+ "pginfo=%p type=%x num_kpages=%llx num_hwpages=%llx "
+ "kpage=%p", e_mr, pginfo, pginfo->type,
+ pginfo->num_kpages, pginfo->num_hwpages, kpage);
+ goto ehca_rereg_mr_rereg1_exit1;
+ }
+ rpage = __pa(kpage);
+ if (!rpage) {
+ ehca_err(&shca->ib_device, "kpage=%p", kpage);
+ ret = -EFAULT;
+ goto ehca_rereg_mr_rereg1_exit1;
+ }
+ h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
+ (u64)iova_start, size, hipz_acl,
+ e_pd->fw_pd, rpage, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ /*
+ * reregistration unsuccessful, try it again with the 3 hCalls,
+ * e.g. this is required in case H_MR_CONDITION
+ * (MW bound or MR is shared)
+ */
+ ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
+ "(Rereg1), h_ret=%lli e_mr=%p", h_ret, e_mr);
+ *pginfo = pginfo_save;
+ ret = -EAGAIN;
+ } else if ((u64 *)hipzout.vaddr != iova_start) {
+ ehca_err(&shca->ib_device, "PHYP changed iova_start in "
+ "rereg_pmr, iova_start=%p iova_start_out=%llx e_mr=%p "
+ "mr_handle=%llx lkey=%x lkey_out=%x", iova_start,
+ hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
+ e_mr->ib.ib_mr.lkey, hipzout.lkey);
+ ret = -EFAULT;
+ } else {
+ /*
+ * successful reregistration
+ * note: start and start_out are identical for eServer HCAs
+ */
+ e_mr->num_kpages = pginfo->num_kpages;
+ e_mr->num_hwpages = pginfo->num_hwpages;
+ e_mr->hwpage_size = pginfo->hwpage_size;
+ e_mr->start = iova_start;
+ e_mr->size = size;
+ e_mr->acl = acl;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ }
+
+ehca_rereg_mr_rereg1_exit1:
+ ehca_free_fw_ctrlblock(kpage);
+ehca_rereg_mr_rereg1_exit0:
+ if ( ret && (ret != -EAGAIN) )
+ ehca_err(&shca->ib_device, "ret=%i lkey=%x rkey=%x "
+ "pginfo=%p num_kpages=%llx num_hwpages=%llx",
+ ret, *lkey, *rkey, pginfo, pginfo->num_kpages,
+ pginfo->num_hwpages);
+ return ret;
+} /* end ehca_rereg_mr_rereg1() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_rereg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey,
+ u32 *rkey)
+{
+ int ret = 0;
+ u64 h_ret;
+ int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
+ int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
+
+ /* first determine reregistration hCall(s) */
+ if ((pginfo->num_hwpages > MAX_RPAGES) ||
+ (e_mr->num_hwpages > MAX_RPAGES) ||
+ (pginfo->num_hwpages > e_mr->num_hwpages)) {
+ ehca_dbg(&shca->ib_device, "Rereg3 case, "
+ "pginfo->num_hwpages=%llx e_mr->num_hwpages=%x",
+ pginfo->num_hwpages, e_mr->num_hwpages);
+ rereg_1_hcall = 0;
+ rereg_3_hcall = 1;
+ }
+
+ if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
+ rereg_1_hcall = 0;
+ rereg_3_hcall = 1;
+ e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
+ ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
+ e_mr);
+ }
+
+ if (rereg_1_hcall) {
+ ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
+ acl, e_pd, pginfo, lkey, rkey);
+ if (ret) {
+ if (ret == -EAGAIN)
+ rereg_3_hcall = 1;
+ else
+ goto ehca_rereg_mr_exit0;
+ }
+ }
+
+ if (rereg_3_hcall) {
+ struct ehca_mr save_mr;
+
+ /* first deregister old MR */
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_free_mr failed, "
+ "h_ret=%lli e_mr=%p hca_hndl=%llx mr_hndl=%llx "
+ "mr->lkey=%x",
+ h_ret, e_mr, shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle,
+ e_mr->ib.ib_mr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_rereg_mr_exit0;
+ }
+ /* clean ehca_mr_t, without changing struct ib_mr and lock */
+ save_mr = *e_mr;
+ ehca_mr_deletenew(e_mr);
+
+ /* set some MR values */
+ e_mr->flags = save_mr.flags;
+ e_mr->hwpage_size = save_mr.hwpage_size;
+ e_mr->fmr_page_size = save_mr.fmr_page_size;
+ e_mr->fmr_max_pages = save_mr.fmr_max_pages;
+ e_mr->fmr_max_maps = save_mr.fmr_max_maps;
+ e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
+
+ ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
+ e_pd, pginfo, lkey, rkey, EHCA_REG_MR);
+ if (ret) {
+ u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
+ memcpy(&e_mr->flags, &(save_mr.flags),
+ sizeof(struct ehca_mr) - offset);
+ goto ehca_rereg_mr_exit0;
+ }
+ }
+
+ehca_rereg_mr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
+ "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
+ "num_kpages=%llx lkey=%x rkey=%x rereg_1_hcall=%x "
+ "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
+ acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,
+ rereg_1_hcall, rereg_3_hcall);
+ return ret;
+} /* end ehca_rereg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_unmap_one_fmr(struct ehca_shca *shca,
+ struct ehca_mr *e_fmr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_pd *e_pd =
+ container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
+ struct ehca_mr save_fmr;
+ u32 tmp_lkey, tmp_rkey;
+ struct ehca_mr_pginfo pginfo;
+ struct ehca_mr_hipzout_parms hipzout;
+ struct ehca_mr save_mr;
+
+ if (e_fmr->fmr_max_pages <= MAX_RPAGES) {
+ /*
+ * note: after using rereg hcall with len=0,
+ * rereg hcall must be used again for registering pages
+ */
+ h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
+ 0, 0, e_pd->fw_pd, 0, &hipzout);
+ if (h_ret == H_SUCCESS) {
+ /* successful reregistration */
+ e_fmr->start = NULL;
+ e_fmr->size = 0;
+ tmp_lkey = hipzout.lkey;
+ tmp_rkey = hipzout.rkey;
+ return 0;
+ }
+ /*
+ * should not happen, because length checked above,
+ * FMRs are not shared and no MW bound to FMRs
+ */
+ ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
+ "(Rereg1), h_ret=%lli e_fmr=%p hca_hndl=%llx "
+ "mr_hndl=%llx lkey=%x lkey_out=%x",
+ h_ret, e_fmr, shca->ipz_hca_handle.handle,
+ e_fmr->ipz_mr_handle.handle,
+ e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
+ /* try free and rereg */
+ }
+
+ /* first free old FMR */
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_free_mr failed, "
+ "h_ret=%lli e_fmr=%p hca_hndl=%llx mr_hndl=%llx "
+ "lkey=%x",
+ h_ret, e_fmr, shca->ipz_hca_handle.handle,
+ e_fmr->ipz_mr_handle.handle,
+ e_fmr->ib.ib_fmr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_unmap_one_fmr_exit0;
+ }
+ /* clean ehca_mr_t, without changing lock */
+ save_fmr = *e_fmr;
+ ehca_mr_deletenew(e_fmr);
+
+ /* set some MR values */
+ e_fmr->flags = save_fmr.flags;
+ e_fmr->hwpage_size = save_fmr.hwpage_size;
+ e_fmr->fmr_page_size = save_fmr.fmr_page_size;
+ e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
+ e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
+ e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
+ e_fmr->acl = save_fmr.acl;
+
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_FMR;
+ ret = ehca_reg_mr(shca, e_fmr, NULL,
+ (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
+ e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
+ &tmp_rkey, EHCA_REG_MR);
+ if (ret) {
+ u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
+ memcpy(&e_fmr->flags, &(save_mr.flags),
+ sizeof(struct ehca_mr) - offset);
+ }
+
+ehca_unmap_one_fmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i tmp_lkey=%x tmp_rkey=%x "
+ "fmr_max_pages=%x",
+ ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages);
+ return ret;
+} /* end ehca_unmap_one_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_smr(struct ehca_shca *shca,
+ struct ehca_mr *e_origmr,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey, /*OUT*/
+ u32 *rkey) /*OUT*/
+{
+ int ret = 0;
+ u64 h_ret;
+ u32 hipz_acl;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
+
+ h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
+ (u64)iova_start, hipz_acl, e_pd->fw_pd,
+ &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
+ "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
+ "e_pd=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
+ h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
+ shca->ipz_hca_handle.handle,
+ e_origmr->ipz_mr_handle.handle,
+ e_origmr->ib.ib_mr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_reg_smr_exit0;
+ }
+ /* successful registration */
+ e_newmr->num_kpages = e_origmr->num_kpages;
+ e_newmr->num_hwpages = e_origmr->num_hwpages;
+ e_newmr->hwpage_size = e_origmr->hwpage_size;
+ e_newmr->start = iova_start;
+ e_newmr->size = e_origmr->size;
+ e_newmr->acl = acl;
+ e_newmr->ipz_mr_handle = hipzout.handle;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ return 0;
+
+ehca_reg_smr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_origmr=%p "
+ "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
+ ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
+ return ret;
+} /* end ehca_reg_smr() */
+
+/*----------------------------------------------------------------------*/
+static inline void *ehca_calc_sectbase(int top, int dir, int idx)
+{
+ unsigned long ret = idx;
+ ret |= dir << EHCA_DIR_INDEX_SHIFT;
+ ret |= top << EHCA_TOP_INDEX_SHIFT;
+ return __va(ret << SECTION_SIZE_BITS);
+}
+
+#define ehca_bmap_valid(entry) \
+ ((u64)entry != (u64)EHCA_INVAL_ADDR)
+
+static u64 ehca_reg_mr_section(int top, int dir, int idx, u64 *kpage,
+ struct ehca_shca *shca, struct ehca_mr *mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ u64 h_ret = 0;
+ unsigned long page = 0;
+ u64 rpage = __pa(kpage);
+ int page_count;
+
+ void *sectbase = ehca_calc_sectbase(top, dir, idx);
+ if ((unsigned long)sectbase & (pginfo->hwpage_size - 1)) {
+ ehca_err(&shca->ib_device, "reg_mr_section will probably fail:"
+ "hwpage_size does not fit to "
+ "section start address");
+ }
+ page_count = EHCA_SECTSIZE / pginfo->hwpage_size;
+
+ while (page < page_count) {
+ u64 rnum;
+ for (rnum = 0; (rnum < MAX_RPAGES) && (page < page_count);
+ rnum++) {
+ void *pg = sectbase + ((page++) * pginfo->hwpage_size);
+ kpage[rnum] = __pa(pg);
+ }
+
+ h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, mr,
+ ehca_encode_hwpage_size(pginfo->hwpage_size),
+ 0, rpage, rnum);
+
+ if ((h_ret != H_SUCCESS) && (h_ret != H_PAGE_REGISTERED)) {
+ ehca_err(&shca->ib_device, "register_rpage_mr failed");
+ return h_ret;
+ }
+ }
+ return h_ret;
+}
+
+static u64 ehca_reg_mr_sections(int top, int dir, u64 *kpage,
+ struct ehca_shca *shca, struct ehca_mr *mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ u64 hret = H_SUCCESS;
+ int idx;
+
+ for (idx = 0; idx < EHCA_MAP_ENTRIES; idx++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]->ent[idx]))
+ continue;
+
+ hret = ehca_reg_mr_section(top, dir, idx, kpage, shca, mr,
+ pginfo);
+ if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
+ return hret;
+ }
+ return hret;
+}
+
+static u64 ehca_reg_mr_dir_sections(int top, u64 *kpage, struct ehca_shca *shca,
+ struct ehca_mr *mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ u64 hret = H_SUCCESS;
+ int dir;
+
+ for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
+ continue;
+
+ hret = ehca_reg_mr_sections(top, dir, kpage, shca, mr, pginfo);
+ if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
+ return hret;
+ }
+ return hret;
+}
+
+/* register internal max-MR to internal SHCA */
+int ehca_reg_internal_maxmr(
+ struct ehca_shca *shca,
+ struct ehca_pd *e_pd,
+ struct ehca_mr **e_maxmr) /*OUT*/
+{
+ int ret;
+ struct ehca_mr *e_mr;
+ u64 *iova_start;
+ u64 size_maxmr;
+ struct ehca_mr_pginfo pginfo;
+ struct ib_phys_buf ib_pbuf;
+ u32 num_kpages;
+ u32 num_hwpages;
+ u64 hw_pgsize;
+
+ if (!ehca_bmap) {
+ ret = -EFAULT;
+ goto ehca_reg_internal_maxmr_exit0;
+ }
+
+ e_mr = ehca_mr_new();
+ if (!e_mr) {
+ ehca_err(&shca->ib_device, "out of memory");
+ ret = -ENOMEM;
+ goto ehca_reg_internal_maxmr_exit0;
+ }
+ e_mr->flags |= EHCA_MR_FLAG_MAXMR;
+
+ /* register internal max-MR on HCA */
+ size_maxmr = ehca_mr_len;
+ iova_start = (u64 *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START));
+ ib_pbuf.addr = 0;
+ ib_pbuf.size = size_maxmr;
+ num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
+ PAGE_SIZE);
+ hw_pgsize = ehca_get_max_hwpage_size(shca);
+ num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr,
+ hw_pgsize);
+
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_PHYS;
+ pginfo.num_kpages = num_kpages;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.hwpage_size = hw_pgsize;
+ pginfo.u.phy.num_phys_buf = 1;
+ pginfo.u.phy.phys_buf_array = &ib_pbuf;
+
+ ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
+ &pginfo, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey, EHCA_REG_BUSMAP_MR);
+ if (ret) {
+ ehca_err(&shca->ib_device, "reg of internal max MR failed, "
+ "e_mr=%p iova_start=%p size_maxmr=%llx num_kpages=%x "
+ "num_hwpages=%x", e_mr, iova_start, size_maxmr,
+ num_kpages, num_hwpages);
+ goto ehca_reg_internal_maxmr_exit1;
+ }
+
+ /* successful registration of all pages */
+ e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
+ e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
+ e_mr->ib.ib_mr.uobject = NULL;
+ atomic_inc(&(e_pd->ib_pd.usecnt));
+ atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
+ *e_maxmr = e_mr;
+ return 0;
+
+ehca_reg_internal_maxmr_exit1:
+ ehca_mr_delete(e_mr);
+ehca_reg_internal_maxmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_pd=%p e_maxmr=%p",
+ ret, shca, e_pd, e_maxmr);
+ return ret;
+} /* end ehca_reg_internal_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_maxmr(struct ehca_shca *shca,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey,
+ u32 *rkey)
+{
+ u64 h_ret;
+ struct ehca_mr *e_origmr = shca->maxmr;
+ u32 hipz_acl;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
+
+ h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
+ (u64)iova_start, hipz_acl, e_pd->fw_pd,
+ &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
+ "e_origmr=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
+ h_ret, e_origmr, shca->ipz_hca_handle.handle,
+ e_origmr->ipz_mr_handle.handle,
+ e_origmr->ib.ib_mr.lkey);
+ return ehca2ib_return_code(h_ret);
+ }
+ /* successful registration */
+ e_newmr->num_kpages = e_origmr->num_kpages;
+ e_newmr->num_hwpages = e_origmr->num_hwpages;
+ e_newmr->hwpage_size = e_origmr->hwpage_size;
+ e_newmr->start = iova_start;
+ e_newmr->size = e_origmr->size;
+ e_newmr->acl = acl;
+ e_newmr->ipz_mr_handle = hipzout.handle;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ return 0;
+} /* end ehca_reg_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
+{
+ int ret;
+ struct ehca_mr *e_maxmr;
+ struct ib_pd *ib_pd;
+
+ if (!shca->maxmr) {
+ ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
+ ret = -EINVAL;
+ goto ehca_dereg_internal_maxmr_exit0;
+ }
+
+ e_maxmr = shca->maxmr;
+ ib_pd = e_maxmr->ib.ib_mr.pd;
+ shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
+
+ ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
+ if (ret) {
+ ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
+ "ret=%i e_maxmr=%p shca=%p lkey=%x",
+ ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
+ shca->maxmr = e_maxmr;
+ goto ehca_dereg_internal_maxmr_exit0;
+ }
+
+ atomic_dec(&ib_pd->usecnt);
+
+ehca_dereg_internal_maxmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p shca->maxmr=%p",
+ ret, shca, shca->maxmr);
+ return ret;
+} /* end ehca_dereg_internal_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * check physical buffer array of MR verbs for validness and
+ * calculates MR size
+ */
+int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ u64 *iova_start,
+ u64 *size)
+{
+ struct ib_phys_buf *pbuf = phys_buf_array;
+ u64 size_count = 0;
+ u32 i;
+
+ if (num_phys_buf == 0) {
+ ehca_gen_err("bad phys buf array len, num_phys_buf=0");
+ return -EINVAL;
+ }
+ /* check first buffer */
+ if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
+ ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
+ "pbuf->addr=%llx pbuf->size=%llx",
+ iova_start, pbuf->addr, pbuf->size);
+ return -EINVAL;
+ }
+ if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
+ (num_phys_buf > 1)) {
+ ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%llx "
+ "pbuf->size=%llx", pbuf->addr, pbuf->size);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < num_phys_buf; i++) {
+ if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
+ ehca_gen_err("bad address, i=%x pbuf->addr=%llx "
+ "pbuf->size=%llx",
+ i, pbuf->addr, pbuf->size);
+ return -EINVAL;
+ }
+ if (((i > 0) && /* not 1st */
+ (i < (num_phys_buf - 1)) && /* not last */
+ (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
+ ehca_gen_err("bad size, i=%x pbuf->size=%llx",
+ i, pbuf->size);
+ return -EINVAL;
+ }
+ size_count += pbuf->size;
+ pbuf++;
+ }
+
+ *size = size_count;
+ return 0;
+} /* end ehca_mr_chk_buf_and_calc_size() */
+
+/*----------------------------------------------------------------------*/
+
+/* check page list of map FMR verb for validness */
+int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
+ u64 *page_list,
+ int list_len)
+{
+ u32 i;
+ u64 *page;
+
+ if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
+ ehca_gen_err("bad list_len, list_len=%x "
+ "e_fmr->fmr_max_pages=%x fmr=%p",
+ list_len, e_fmr->fmr_max_pages, e_fmr);
+ return -EINVAL;
+ }
+
+ /* each page must be aligned */
+ page = page_list;
+ for (i = 0; i < list_len; i++) {
+ if (*page % e_fmr->fmr_page_size) {
+ ehca_gen_err("bad page, i=%x *page=%llx page=%p fmr=%p "
+ "fmr_page_size=%x", i, *page, page, e_fmr,
+ e_fmr->fmr_page_size);
+ return -EINVAL;
+ }
+ page++;
+ }
+
+ return 0;
+} /* end ehca_fmr_check_page_list() */
+
+/*----------------------------------------------------------------------*/
+
+/* PAGE_SIZE >= pginfo->hwpage_size */
+static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage)
+{
+ int ret = 0;
+ u64 pgaddr;
+ u32 j = 0;
+ int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
+ struct scatterlist **sg = &pginfo->u.usr.next_sg;
+
+ while (*sg != NULL) {
+ pgaddr = page_to_pfn(sg_page(*sg))
+ << PAGE_SHIFT;
+ *kpage = pgaddr + (pginfo->next_hwpage *
+ pginfo->hwpage_size);
+ if (!(*kpage)) {
+ ehca_gen_err("pgaddr=%llx "
+ "sg_dma_address=%llx "
+ "entry=%llx next_hwpage=%llx",
+ pgaddr, (u64)sg_dma_address(*sg),
+ pginfo->u.usr.next_nmap,
+ pginfo->next_hwpage);
+ return -EFAULT;
+ }
+ (pginfo->hwpage_cnt)++;
+ (pginfo->next_hwpage)++;
+ kpage++;
+ if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
+ (pginfo->kpage_cnt)++;
+ (pginfo->u.usr.next_nmap)++;
+ pginfo->next_hwpage = 0;
+ *sg = sg_next(*sg);
+ }
+ j++;
+ if (j >= number)
+ break;
+ }
+
+ return ret;
+}
+
+/*
+ * check given pages for contiguous layout
+ * last page addr is returned in prev_pgaddr for further check
+ */
+static int ehca_check_kpages_per_ate(struct scatterlist **sg,
+ int num_pages,
+ u64 *prev_pgaddr)
+{
+ for (; *sg && num_pages > 0; *sg = sg_next(*sg), num_pages--) {
+ u64 pgaddr = page_to_pfn(sg_page(*sg)) << PAGE_SHIFT;
+ if (ehca_debug_level >= 3)
+ ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr,
+ *(u64 *)__va(pgaddr));
+ if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
+ ehca_gen_err("uncontiguous page found pgaddr=%llx "
+ "prev_pgaddr=%llx entries_left_in_hwpage=%x",
+ pgaddr, *prev_pgaddr, num_pages);
+ return -EINVAL;
+ }
+ *prev_pgaddr = pgaddr;
+ }
+ return 0;
+}
+
+/* PAGE_SIZE < pginfo->hwpage_size */
+static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage)
+{
+ int ret = 0;
+ u64 pgaddr, prev_pgaddr;
+ u32 j = 0;
+ int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
+ int nr_kpages = kpages_per_hwpage;
+ struct scatterlist **sg = &pginfo->u.usr.next_sg;
+
+ while (*sg != NULL) {
+
+ if (nr_kpages == kpages_per_hwpage) {
+ pgaddr = (page_to_pfn(sg_page(*sg))
+ << PAGE_SHIFT);
+ *kpage = pgaddr;
+ if (!(*kpage)) {
+ ehca_gen_err("pgaddr=%llx entry=%llx",
+ pgaddr, pginfo->u.usr.next_nmap);
+ ret = -EFAULT;
+ return ret;
+ }
+ /*
+ * The first page in a hwpage must be aligned;
+ * the first MR page is exempt from this rule.
+ */
+ if (pgaddr & (pginfo->hwpage_size - 1)) {
+ if (pginfo->hwpage_cnt) {
+ ehca_gen_err(
+ "invalid alignment "
+ "pgaddr=%llx entry=%llx "
+ "mr_pgsize=%llx",
+ pgaddr, pginfo->u.usr.next_nmap,
+ pginfo->hwpage_size);
+ ret = -EFAULT;
+ return ret;
+ }
+ /* first MR page */
+ pginfo->kpage_cnt =
+ (pgaddr &
+ (pginfo->hwpage_size - 1)) >>
+ PAGE_SHIFT;
+ nr_kpages -= pginfo->kpage_cnt;
+ *kpage = pgaddr &
+ ~(pginfo->hwpage_size - 1);
+ }
+ if (ehca_debug_level >= 3) {
+ u64 val = *(u64 *)__va(pgaddr);
+ ehca_gen_dbg("kpage=%llx page=%llx "
+ "value=%016llx",
+ *kpage, pgaddr, val);
+ }
+ prev_pgaddr = pgaddr;
+ *sg = sg_next(*sg);
+ pginfo->kpage_cnt++;
+ pginfo->u.usr.next_nmap++;
+ nr_kpages--;
+ if (!nr_kpages)
+ goto next_kpage;
+ continue;
+ }
+
+ ret = ehca_check_kpages_per_ate(sg, nr_kpages,
+ &prev_pgaddr);
+ if (ret)
+ return ret;
+ pginfo->kpage_cnt += nr_kpages;
+ pginfo->u.usr.next_nmap += nr_kpages;
+
+next_kpage:
+ nr_kpages = kpages_per_hwpage;
+ (pginfo->hwpage_cnt)++;
+ kpage++;
+ j++;
+ if (j >= number)
+ break;
+ }
+
+ return ret;
+}
+
+static int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
+ u32 number, u64 *kpage)
+{
+ int ret = 0;
+ struct ib_phys_buf *pbuf;
+ u64 num_hw, offs_hw;
+ u32 i = 0;
+
+ /* loop over desired phys_buf_array entries */
+ while (i < number) {
+ pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
+ num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +
+ pbuf->size, pginfo->hwpage_size);
+ offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /
+ pginfo->hwpage_size;
+ while (pginfo->next_hwpage < offs_hw + num_hw) {
+ /* sanity check */
+ if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
+ (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
+ ehca_gen_err("kpage_cnt >= num_kpages, "
+ "kpage_cnt=%llx num_kpages=%llx "
+ "hwpage_cnt=%llx "
+ "num_hwpages=%llx i=%x",
+ pginfo->kpage_cnt,
+ pginfo->num_kpages,
+ pginfo->hwpage_cnt,
+ pginfo->num_hwpages, i);
+ return -EFAULT;
+ }
+ *kpage = (pbuf->addr & ~(pginfo->hwpage_size - 1)) +
+ (pginfo->next_hwpage * pginfo->hwpage_size);
+ if ( !(*kpage) && pbuf->addr ) {
+ ehca_gen_err("pbuf->addr=%llx pbuf->size=%llx "
+ "next_hwpage=%llx", pbuf->addr,
+ pbuf->size, pginfo->next_hwpage);
+ return -EFAULT;
+ }
+ (pginfo->hwpage_cnt)++;
+ (pginfo->next_hwpage)++;
+ if (PAGE_SIZE >= pginfo->hwpage_size) {
+ if (pginfo->next_hwpage %
+ (PAGE_SIZE / pginfo->hwpage_size) == 0)
+ (pginfo->kpage_cnt)++;
+ } else
+ pginfo->kpage_cnt += pginfo->hwpage_size /
+ PAGE_SIZE;
+ kpage++;
+ i++;
+ if (i >= number) break;
+ }
+ if (pginfo->next_hwpage >= offs_hw + num_hw) {
+ (pginfo->u.phy.next_buf)++;
+ pginfo->next_hwpage = 0;
+ }
+ }
+ return ret;
+}
+
+static int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
+ u32 number, u64 *kpage)
+{
+ int ret = 0;
+ u64 *fmrlist;
+ u32 i;
+
+ /* loop over desired page_list entries */
+ fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
+ for (i = 0; i < number; i++) {
+ *kpage = (*fmrlist & ~(pginfo->hwpage_size - 1)) +
+ pginfo->next_hwpage * pginfo->hwpage_size;
+ if ( !(*kpage) ) {
+ ehca_gen_err("*fmrlist=%llx fmrlist=%p "
+ "next_listelem=%llx next_hwpage=%llx",
+ *fmrlist, fmrlist,
+ pginfo->u.fmr.next_listelem,
+ pginfo->next_hwpage);
+ return -EFAULT;
+ }
+ (pginfo->hwpage_cnt)++;
+ if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) {
+ if (pginfo->next_hwpage %
+ (pginfo->u.fmr.fmr_pgsize /
+ pginfo->hwpage_size) == 0) {
+ (pginfo->kpage_cnt)++;
+ (pginfo->u.fmr.next_listelem)++;
+ fmrlist++;
+ pginfo->next_hwpage = 0;
+ } else
+ (pginfo->next_hwpage)++;
+ } else {
+ unsigned int cnt_per_hwpage = pginfo->hwpage_size /
+ pginfo->u.fmr.fmr_pgsize;
+ unsigned int j;
+ u64 prev = *kpage;
+ /* check if adrs are contiguous */
+ for (j = 1; j < cnt_per_hwpage; j++) {
+ u64 p = fmrlist[j] & ~(pginfo->hwpage_size - 1);
+ if (prev + pginfo->u.fmr.fmr_pgsize != p) {
+ ehca_gen_err("uncontiguous fmr pages "
+ "found prev=%llx p=%llx "
+ "idx=%x", prev, p, i + j);
+ return -EINVAL;
+ }
+ prev = p;
+ }
+ pginfo->kpage_cnt += cnt_per_hwpage;
+ pginfo->u.fmr.next_listelem += cnt_per_hwpage;
+ fmrlist += cnt_per_hwpage;
+ }
+ kpage++;
+ }
+ return ret;
+}
+
+/* setup page buffer from page info */
+int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage)
+{
+ int ret;
+
+ switch (pginfo->type) {
+ case EHCA_MR_PGI_PHYS:
+ ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
+ break;
+ case EHCA_MR_PGI_USER:
+ ret = PAGE_SIZE >= pginfo->hwpage_size ?
+ ehca_set_pagebuf_user1(pginfo, number, kpage) :
+ ehca_set_pagebuf_user2(pginfo, number, kpage);
+ break;
+ case EHCA_MR_PGI_FMR:
+ ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
+ break;
+ default:
+ ehca_gen_err("bad pginfo->type=%x", pginfo->type);
+ ret = -EFAULT;
+ break;
+ }
+ return ret;
+} /* end ehca_set_pagebuf() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * check MR if it is a max-MR, i.e. uses whole memory
+ * in case it's a max-MR 1 is returned, else 0
+ */
+int ehca_mr_is_maxmr(u64 size,
+ u64 *iova_start)
+{
+ /* a MR is treated as max-MR only if it fits following: */
+ if ((size == ehca_mr_len) &&
+ (iova_start == (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)))) {
+ ehca_gen_dbg("this is a max-MR");
+ return 1;
+ } else
+ return 0;
+} /* end ehca_mr_is_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+/* map access control for MR/MW. This routine is used for MR and MW. */
+void ehca_mrmw_map_acl(int ib_acl,
+ u32 *hipz_acl)
+{
+ *hipz_acl = 0;
+ if (ib_acl & IB_ACCESS_REMOTE_READ)
+ *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
+ if (ib_acl & IB_ACCESS_REMOTE_WRITE)
+ *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
+ if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
+ *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
+ if (ib_acl & IB_ACCESS_LOCAL_WRITE)
+ *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
+ if (ib_acl & IB_ACCESS_MW_BIND)
+ *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
+} /* end ehca_mrmw_map_acl() */
+
+/*----------------------------------------------------------------------*/
+
+/* sets page size in hipz access control for MR/MW. */
+void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/
+{
+ *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24);
+} /* end ehca_mrmw_set_pgsize_hipz_acl() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * reverse map access control for MR/MW.
+ * This routine is used for MR and MW.
+ */
+void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
+ int *ib_acl) /*OUT*/
+{
+ *ib_acl = 0;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
+ *ib_acl |= IB_ACCESS_REMOTE_READ;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
+ *ib_acl |= IB_ACCESS_REMOTE_WRITE;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
+ *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
+ *ib_acl |= IB_ACCESS_LOCAL_WRITE;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
+ *ib_acl |= IB_ACCESS_MW_BIND;
+} /* end ehca_mrmw_reverse_map_acl() */
+
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * MR destructor and constructor
+ * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
+ * except struct ib_mr and spinlock
+ */
+void ehca_mr_deletenew(struct ehca_mr *mr)
+{
+ mr->flags = 0;
+ mr->num_kpages = 0;
+ mr->num_hwpages = 0;
+ mr->acl = 0;
+ mr->start = NULL;
+ mr->fmr_page_size = 0;
+ mr->fmr_max_pages = 0;
+ mr->fmr_max_maps = 0;
+ mr->fmr_map_cnt = 0;
+ memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
+ memset(&mr->galpas, 0, sizeof(mr->galpas));
+} /* end ehca_mr_deletenew() */
+
+int ehca_init_mrmw_cache(void)
+{
+ mr_cache = kmem_cache_create("ehca_cache_mr",
+ sizeof(struct ehca_mr), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!mr_cache)
+ return -ENOMEM;
+ mw_cache = kmem_cache_create("ehca_cache_mw",
+ sizeof(struct ehca_mw), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!mw_cache) {
+ kmem_cache_destroy(mr_cache);
+ mr_cache = NULL;
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+void ehca_cleanup_mrmw_cache(void)
+{
+ if (mr_cache)
+ kmem_cache_destroy(mr_cache);
+ if (mw_cache)
+ kmem_cache_destroy(mw_cache);
+}
+
+static inline int ehca_init_top_bmap(struct ehca_top_bmap *ehca_top_bmap,
+ int dir)
+{
+ if (!ehca_bmap_valid(ehca_top_bmap->dir[dir])) {
+ ehca_top_bmap->dir[dir] =
+ kmalloc(sizeof(struct ehca_dir_bmap), GFP_KERNEL);
+ if (!ehca_top_bmap->dir[dir])
+ return -ENOMEM;
+ /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
+ memset(ehca_top_bmap->dir[dir], 0xFF, EHCA_ENT_MAP_SIZE);
+ }
+ return 0;
+}
+
+static inline int ehca_init_bmap(struct ehca_bmap *ehca_bmap, int top, int dir)
+{
+ if (!ehca_bmap_valid(ehca_bmap->top[top])) {
+ ehca_bmap->top[top] =
+ kmalloc(sizeof(struct ehca_top_bmap), GFP_KERNEL);
+ if (!ehca_bmap->top[top])
+ return -ENOMEM;
+ /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
+ memset(ehca_bmap->top[top], 0xFF, EHCA_DIR_MAP_SIZE);
+ }
+ return ehca_init_top_bmap(ehca_bmap->top[top], dir);
+}
+
+static inline int ehca_calc_index(unsigned long i, unsigned long s)
+{
+ return (i >> s) & EHCA_INDEX_MASK;
+}
+
+void ehca_destroy_busmap(void)
+{
+ int top, dir;
+
+ if (!ehca_bmap)
+ return;
+
+ for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]))
+ continue;
+ for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
+ continue;
+
+ kfree(ehca_bmap->top[top]->dir[dir]);
+ }
+
+ kfree(ehca_bmap->top[top]);
+ }
+
+ kfree(ehca_bmap);
+ ehca_bmap = NULL;
+}
+
+static int ehca_update_busmap(unsigned long pfn, unsigned long nr_pages)
+{
+ unsigned long i, start_section, end_section;
+ int top, dir, idx;
+
+ if (!nr_pages)
+ return 0;
+
+ if (!ehca_bmap) {
+ ehca_bmap = kmalloc(sizeof(struct ehca_bmap), GFP_KERNEL);
+ if (!ehca_bmap)
+ return -ENOMEM;
+ /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
+ memset(ehca_bmap, 0xFF, EHCA_TOP_MAP_SIZE);
+ }
+
+ start_section = (pfn * PAGE_SIZE) / EHCA_SECTSIZE;
+ end_section = ((pfn + nr_pages) * PAGE_SIZE) / EHCA_SECTSIZE;
+ for (i = start_section; i < end_section; i++) {
+ int ret;
+ top = ehca_calc_index(i, EHCA_TOP_INDEX_SHIFT);
+ dir = ehca_calc_index(i, EHCA_DIR_INDEX_SHIFT);
+ idx = i & EHCA_INDEX_MASK;
+
+ ret = ehca_init_bmap(ehca_bmap, top, dir);
+ if (ret) {
+ ehca_destroy_busmap();
+ return ret;
+ }
+ ehca_bmap->top[top]->dir[dir]->ent[idx] = ehca_mr_len;
+ ehca_mr_len += EHCA_SECTSIZE;
+ }
+ return 0;
+}
+
+static int ehca_is_hugepage(unsigned long pfn)
+{
+ int page_order;
+
+ if (pfn & EHCA_HUGEPAGE_PFN_MASK)
+ return 0;
+
+ page_order = compound_order(pfn_to_page(pfn));
+ if (page_order + PAGE_SHIFT != EHCA_HUGEPAGESHIFT)
+ return 0;
+
+ return 1;
+}
+
+static int ehca_create_busmap_callback(unsigned long initial_pfn,
+ unsigned long total_nr_pages, void *arg)
+{
+ int ret;
+ unsigned long pfn, start_pfn, end_pfn, nr_pages;
+
+ if ((total_nr_pages * PAGE_SIZE) < EHCA_HUGEPAGE_SIZE)
+ return ehca_update_busmap(initial_pfn, total_nr_pages);
+
+ /* Given chunk is >= 16GB -> check for hugepages */
+ start_pfn = initial_pfn;
+ end_pfn = initial_pfn + total_nr_pages;
+ pfn = start_pfn;
+
+ while (pfn < end_pfn) {
+ if (ehca_is_hugepage(pfn)) {
+ /* Add mem found in front of the hugepage */
+ nr_pages = pfn - start_pfn;
+ ret = ehca_update_busmap(start_pfn, nr_pages);
+ if (ret)
+ return ret;
+ /* Skip the hugepage */
+ pfn += (EHCA_HUGEPAGE_SIZE / PAGE_SIZE);
+ start_pfn = pfn;
+ } else
+ pfn += (EHCA_SECTSIZE / PAGE_SIZE);
+ }
+
+ /* Add mem found behind the hugepage(s) */
+ nr_pages = pfn - start_pfn;
+ return ehca_update_busmap(start_pfn, nr_pages);
+}
+
+int ehca_create_busmap(void)
+{
+ int ret;
+
+ ehca_mr_len = 0;
+ ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL,
+ ehca_create_busmap_callback);
+ return ret;
+}
+
+static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ int top;
+ u64 hret, *kpage;
+
+ kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!kpage) {
+ ehca_err(&shca->ib_device, "kpage alloc failed");
+ return -ENOMEM;
+ }
+ for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]))
+ continue;
+ hret = ehca_reg_mr_dir_sections(top, kpage, shca, e_mr, pginfo);
+ if ((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS))
+ break;
+ }
+
+ ehca_free_fw_ctrlblock(kpage);
+
+ if (hret == H_SUCCESS)
+ return 0; /* Everything is fine */
+ else {
+ ehca_err(&shca->ib_device, "ehca_reg_bmap_mr_rpages failed, "
+ "h_ret=%lli e_mr=%p top=%x lkey=%x "
+ "hca_hndl=%llx mr_hndl=%llx", hret, e_mr, top,
+ e_mr->ib.ib_mr.lkey,
+ shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle);
+ return ehca2ib_return_code(hret);
+ }
+}
+
+static u64 ehca_map_vaddr(void *caddr)
+{
+ int top, dir, idx;
+ unsigned long abs_addr, offset;
+ u64 entry;
+
+ if (!ehca_bmap)
+ return EHCA_INVAL_ADDR;
+
+ abs_addr = __pa(caddr);
+ top = ehca_calc_index(abs_addr, EHCA_TOP_INDEX_SHIFT + EHCA_SECTSHIFT);
+ if (!ehca_bmap_valid(ehca_bmap->top[top]))
+ return EHCA_INVAL_ADDR;
+
+ dir = ehca_calc_index(abs_addr, EHCA_DIR_INDEX_SHIFT + EHCA_SECTSHIFT);
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
+ return EHCA_INVAL_ADDR;
+
+ idx = ehca_calc_index(abs_addr, EHCA_SECTSHIFT);
+
+ entry = ehca_bmap->top[top]->dir[dir]->ent[idx];
+ if (ehca_bmap_valid(entry)) {
+ offset = (unsigned long)caddr & (EHCA_SECTSIZE - 1);
+ return entry | offset;
+ } else
+ return EHCA_INVAL_ADDR;
+}
+
+static int ehca_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
+{
+ return dma_addr == EHCA_INVAL_ADDR;
+}
+
+static u64 ehca_dma_map_single(struct ib_device *dev, void *cpu_addr,
+ size_t size, enum dma_data_direction direction)
+{
+ if (cpu_addr)
+ return ehca_map_vaddr(cpu_addr);
+ else
+ return EHCA_INVAL_ADDR;
+}
+
+static void ehca_dma_unmap_single(struct ib_device *dev, u64 addr, size_t size,
+ enum dma_data_direction direction)
+{
+ /* This is only a stub; nothing to be done here */
+}
+
+static u64 ehca_dma_map_page(struct ib_device *dev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction direction)
+{
+ u64 addr;
+
+ if (offset + size > PAGE_SIZE)
+ return EHCA_INVAL_ADDR;
+
+ addr = ehca_map_vaddr(page_address(page));
+ if (!ehca_dma_mapping_error(dev, addr))
+ addr += offset;
+
+ return addr;
+}
+
+static void ehca_dma_unmap_page(struct ib_device *dev, u64 addr, size_t size,
+ enum dma_data_direction direction)
+{
+ /* This is only a stub; nothing to be done here */
+}
+
+static int ehca_dma_map_sg(struct ib_device *dev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction direction)
+{
+ struct scatterlist *sg;
+ int i;
+
+ for_each_sg(sgl, sg, nents, i) {
+ u64 addr;
+ addr = ehca_map_vaddr(sg_virt(sg));
+ if (ehca_dma_mapping_error(dev, addr))
+ return 0;
+
+ sg->dma_address = addr;
+ sg->dma_length = sg->length;
+ }
+ return nents;
+}
+
+static void ehca_dma_unmap_sg(struct ib_device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction direction)
+{
+ /* This is only a stub; nothing to be done here */
+}
+
+static void ehca_dma_sync_single_for_cpu(struct ib_device *dev, u64 addr,
+ size_t size,
+ enum dma_data_direction dir)
+{
+ dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
+}
+
+static void ehca_dma_sync_single_for_device(struct ib_device *dev, u64 addr,
+ size_t size,
+ enum dma_data_direction dir)
+{
+ dma_sync_single_for_device(dev->dma_device, addr, size, dir);
+}
+
+static void *ehca_dma_alloc_coherent(struct ib_device *dev, size_t size,
+ u64 *dma_handle, gfp_t flag)
+{
+ struct page *p;
+ void *addr = NULL;
+ u64 dma_addr;
+
+ p = alloc_pages(flag, get_order(size));
+ if (p) {
+ addr = page_address(p);
+ dma_addr = ehca_map_vaddr(addr);
+ if (ehca_dma_mapping_error(dev, dma_addr)) {
+ free_pages((unsigned long)addr, get_order(size));
+ return NULL;
+ }
+ if (dma_handle)
+ *dma_handle = dma_addr;
+ return addr;
+ }
+ return NULL;
+}
+
+static void ehca_dma_free_coherent(struct ib_device *dev, size_t size,
+ void *cpu_addr, u64 dma_handle)
+{
+ if (cpu_addr && size)
+ free_pages((unsigned long)cpu_addr, get_order(size));
+}
+
+
+struct ib_dma_mapping_ops ehca_dma_mapping_ops = {
+ .mapping_error = ehca_dma_mapping_error,
+ .map_single = ehca_dma_map_single,
+ .unmap_single = ehca_dma_unmap_single,
+ .map_page = ehca_dma_map_page,
+ .unmap_page = ehca_dma_unmap_page,
+ .map_sg = ehca_dma_map_sg,
+ .unmap_sg = ehca_dma_unmap_sg,
+ .sync_single_for_cpu = ehca_dma_sync_single_for_cpu,
+ .sync_single_for_device = ehca_dma_sync_single_for_device,
+ .alloc_coherent = ehca_dma_alloc_coherent,
+ .free_coherent = ehca_dma_free_coherent,
+};
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * MR/MW declarations and inline functions
+ *
+ * Authors: Dietmar Decker <ddecker@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _EHCA_MRMW_H_
+#define _EHCA_MRMW_H_
+
+enum ehca_reg_type {
+ EHCA_REG_MR,
+ EHCA_REG_BUSMAP_MR
+};
+
+int ehca_reg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey,
+ u32 *rkey,
+ enum ehca_reg_type reg_type);
+
+int ehca_reg_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo);
+
+int ehca_rereg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int mr_access_flags,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey,
+ u32 *rkey);
+
+int ehca_unmap_one_fmr(struct ehca_shca *shca,
+ struct ehca_mr *e_fmr);
+
+int ehca_reg_smr(struct ehca_shca *shca,
+ struct ehca_mr *e_origmr,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey,
+ u32 *rkey);
+
+int ehca_reg_internal_maxmr(struct ehca_shca *shca,
+ struct ehca_pd *e_pd,
+ struct ehca_mr **maxmr);
+
+int ehca_reg_maxmr(struct ehca_shca *shca,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey,
+ u32 *rkey);
+
+int ehca_dereg_internal_maxmr(struct ehca_shca *shca);
+
+int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ u64 *iova_start,
+ u64 *size);
+
+int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
+ u64 *page_list,
+ int list_len);
+
+int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage);
+
+int ehca_mr_is_maxmr(u64 size,
+ u64 *iova_start);
+
+void ehca_mrmw_map_acl(int ib_acl,
+ u32 *hipz_acl);
+
+void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl);
+
+void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
+ int *ib_acl);
+
+void ehca_mr_deletenew(struct ehca_mr *mr);
+
+int ehca_create_busmap(void);
+
+void ehca_destroy_busmap(void);
+
+extern struct ib_dma_mapping_ops ehca_dma_mapping_ops;
+#endif /*_EHCA_MRMW_H_*/
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * PD functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+
+static struct kmem_cache *pd_cache;
+
+struct ib_pd *ehca_alloc_pd(struct ib_device *device,
+ struct ib_ucontext *context, struct ib_udata *udata)
+{
+ struct ehca_pd *pd;
+ int i;
+
+ pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL);
+ if (!pd) {
+ ehca_err(device, "device=%p context=%p out of memory",
+ device, context);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ for (i = 0; i < 2; i++) {
+ INIT_LIST_HEAD(&pd->free[i]);
+ INIT_LIST_HEAD(&pd->full[i]);
+ }
+ mutex_init(&pd->lock);
+
+ /*
+ * Kernel PD: when device = -1, 0
+ * User PD: when context != -1
+ */
+ if (!context) {
+ /*
+ * Kernel PDs after init reuses always
+ * the one created in ehca_shca_reopen()
+ */
+ struct ehca_shca *shca = container_of(device, struct ehca_shca,
+ ib_device);
+ pd->fw_pd.value = shca->pd->fw_pd.value;
+ } else
+ pd->fw_pd.value = (u64)pd;
+
+ return &pd->ib_pd;
+}
+
+int ehca_dealloc_pd(struct ib_pd *pd)
+{
+ struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
+ int i, leftovers = 0;
+ struct ipz_small_queue_page *page, *tmp;
+
+ for (i = 0; i < 2; i++) {
+ list_splice(&my_pd->full[i], &my_pd->free[i]);
+ list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) {
+ leftovers = 1;
+ free_page(page->page);
+ kmem_cache_free(small_qp_cache, page);
+ }
+ }
+
+ if (leftovers)
+ ehca_warn(pd->device,
+ "Some small queue pages were not freed");
+
+ kmem_cache_free(pd_cache, my_pd);
+
+ return 0;
+}
+
+int ehca_init_pd_cache(void)
+{
+ pd_cache = kmem_cache_create("ehca_cache_pd",
+ sizeof(struct ehca_pd), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!pd_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_pd_cache(void)
+{
+ if (pd_cache)
+ kmem_cache_destroy(pd_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Hardware request structures
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef _EHCA_QES_H_
+#define _EHCA_QES_H_
+
+#include "ehca_tools.h"
+
+/* virtual scatter gather entry to specify remote addresses with length */
+struct ehca_vsgentry {
+ u64 vaddr;
+ u32 lkey;
+ u32 length;
+};
+
+#define GRH_FLAG_MASK EHCA_BMASK_IBM( 7, 7)
+#define GRH_IPVERSION_MASK EHCA_BMASK_IBM( 0, 3)
+#define GRH_TCLASS_MASK EHCA_BMASK_IBM( 4, 12)
+#define GRH_FLOWLABEL_MASK EHCA_BMASK_IBM(13, 31)
+#define GRH_PAYLEN_MASK EHCA_BMASK_IBM(32, 47)
+#define GRH_NEXTHEADER_MASK EHCA_BMASK_IBM(48, 55)
+#define GRH_HOPLIMIT_MASK EHCA_BMASK_IBM(56, 63)
+
+/*
+ * Unreliable Datagram Address Vector Format
+ * see IBTA Vol1 chapter 8.3 Global Routing Header
+ */
+struct ehca_ud_av {
+ u8 sl;
+ u8 lnh;
+ u16 dlid;
+ u8 reserved1;
+ u8 reserved2;
+ u8 reserved3;
+ u8 slid_path_bits;
+ u8 reserved4;
+ u8 ipd;
+ u8 reserved5;
+ u8 pmtu;
+ u32 reserved6;
+ u64 reserved7;
+ union {
+ struct {
+ u64 word_0; /* always set to 6 */
+ /*should be 0x1B for IB transport */
+ u64 word_1;
+ u64 word_2;
+ u64 word_3;
+ u64 word_4;
+ } grh;
+ struct {
+ u32 wd_0;
+ u32 wd_1;
+ /* DWord_1 --> SGID */
+
+ u32 sgid_wd3;
+ u32 sgid_wd2;
+
+ u32 sgid_wd1;
+ u32 sgid_wd0;
+ /* DWord_3 --> DGID */
+
+ u32 dgid_wd3;
+ u32 dgid_wd2;
+
+ u32 dgid_wd1;
+ u32 dgid_wd0;
+ } grh_l;
+ };
+};
+
+/* maximum number of sg entries allowed in a WQE */
+#define MAX_WQE_SG_ENTRIES 252
+
+#define WQE_OPTYPE_SEND 0x80
+#define WQE_OPTYPE_RDMAREAD 0x40
+#define WQE_OPTYPE_RDMAWRITE 0x20
+#define WQE_OPTYPE_CMPSWAP 0x10
+#define WQE_OPTYPE_FETCHADD 0x08
+#define WQE_OPTYPE_BIND 0x04
+
+#define WQE_WRFLAG_REQ_SIGNAL_COM 0x80
+#define WQE_WRFLAG_FENCE 0x40
+#define WQE_WRFLAG_IMM_DATA_PRESENT 0x20
+#define WQE_WRFLAG_SOLIC_EVENT 0x10
+
+#define WQEF_CACHE_HINT 0x80
+#define WQEF_CACHE_HINT_RD_WR 0x40
+#define WQEF_TIMED_WQE 0x20
+#define WQEF_PURGE 0x08
+#define WQEF_HIGH_NIBBLE 0xF0
+
+#define MW_BIND_ACCESSCTRL_R_WRITE 0x40
+#define MW_BIND_ACCESSCTRL_R_READ 0x20
+#define MW_BIND_ACCESSCTRL_R_ATOMIC 0x10
+
+struct ehca_wqe {
+ u64 work_request_id;
+ u8 optype;
+ u8 wr_flag;
+ u16 pkeyi;
+ u8 wqef;
+ u8 nr_of_data_seg;
+ u16 wqe_provided_slid;
+ u32 destination_qp_number;
+ u32 resync_psn_sqp;
+ u32 local_ee_context_qkey;
+ u32 immediate_data;
+ union {
+ struct {
+ u64 remote_virtual_address;
+ u32 rkey;
+ u32 reserved;
+ u64 atomic_1st_op_dma_len;
+ u64 atomic_2nd_op;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+
+ } nud;
+ struct {
+ u64 ehca_ud_av_ptr;
+ u64 reserved1;
+ u64 reserved2;
+ u64 reserved3;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+ } ud_avp;
+ struct {
+ struct ehca_ud_av ud_av;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES -
+ 2];
+ } ud_av;
+ struct {
+ u64 reserved0;
+ u64 reserved1;
+ u64 reserved2;
+ u64 reserved3;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+ } all_rcv;
+
+ struct {
+ u64 reserved;
+ u32 rkey;
+ u32 old_rkey;
+ u64 reserved1;
+ u64 reserved2;
+ u64 virtual_address;
+ u32 reserved3;
+ u32 length;
+ u32 reserved4;
+ u16 reserved5;
+ u8 reserved6;
+ u8 lr_ctl;
+ u32 lkey;
+ u32 reserved7;
+ u64 reserved8;
+ u64 reserved9;
+ u64 reserved10;
+ u64 reserved11;
+ } bind;
+ struct {
+ u64 reserved12;
+ u64 reserved13;
+ u32 size;
+ u32 start;
+ } inline_data;
+ } u;
+
+};
+
+#define WC_SEND_RECEIVE EHCA_BMASK_IBM(0, 0)
+#define WC_IMM_DATA EHCA_BMASK_IBM(1, 1)
+#define WC_GRH_PRESENT EHCA_BMASK_IBM(2, 2)
+#define WC_SE_BIT EHCA_BMASK_IBM(3, 3)
+#define WC_STATUS_ERROR_BIT 0x80000000
+#define WC_STATUS_REMOTE_ERROR_FLAGS 0x0000F800
+#define WC_STATUS_PURGE_BIT 0x10
+#define WC_SEND_RECEIVE_BIT 0x80
+
+struct ehca_cqe {
+ u64 work_request_id;
+ u8 optype;
+ u8 w_completion_flags;
+ u16 reserved1;
+ u32 nr_bytes_transferred;
+ u32 immediate_data;
+ u32 local_qp_number;
+ u8 freed_resource_count;
+ u8 service_level;
+ u16 wqe_count;
+ u32 qp_token;
+ u32 qkey_ee_token;
+ u32 remote_qp_number;
+ u16 dlid;
+ u16 rlid;
+ u16 reserved2;
+ u16 pkey_index;
+ u32 cqe_timestamp;
+ u32 wqe_timestamp;
+ u8 wqe_timestamp_valid;
+ u8 reserved3;
+ u8 reserved4;
+ u8 cqe_flags;
+ u32 status;
+};
+
+struct ehca_eqe {
+ u64 entry;
+};
+
+struct ehca_mrte {
+ u64 starting_va;
+ u64 length; /* length of memory region in bytes*/
+ u32 pd;
+ u8 key_instance;
+ u8 pagesize;
+ u8 mr_control;
+ u8 local_remote_access_ctrl;
+ u8 reserved[0x20 - 0x18];
+ u64 at_pointer[4];
+};
+#endif /*_EHCA_QES_H_*/
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * QP functions
+ *
+ * Authors: Joachim Fenkes <fenkes@de.ibm.com>
+ * Stefan Roscher <stefan.roscher@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+
+static struct kmem_cache *qp_cache;
+
+/*
+ * attributes not supported by query qp
+ */
+#define QP_ATTR_QUERY_NOT_SUPPORTED (IB_QP_ACCESS_FLAGS | \
+ IB_QP_EN_SQD_ASYNC_NOTIFY)
+
+/*
+ * ehca (internal) qp state values
+ */
+enum ehca_qp_state {
+ EHCA_QPS_RESET = 1,
+ EHCA_QPS_INIT = 2,
+ EHCA_QPS_RTR = 3,
+ EHCA_QPS_RTS = 5,
+ EHCA_QPS_SQD = 6,
+ EHCA_QPS_SQE = 8,
+ EHCA_QPS_ERR = 128
+};
+
+/*
+ * qp state transitions as defined by IB Arch Rel 1.1 page 431
+ */
+enum ib_qp_statetrans {
+ IB_QPST_ANY2RESET,
+ IB_QPST_ANY2ERR,
+ IB_QPST_RESET2INIT,
+ IB_QPST_INIT2RTR,
+ IB_QPST_INIT2INIT,
+ IB_QPST_RTR2RTS,
+ IB_QPST_RTS2SQD,
+ IB_QPST_RTS2RTS,
+ IB_QPST_SQD2RTS,
+ IB_QPST_SQE2RTS,
+ IB_QPST_SQD2SQD,
+ IB_QPST_MAX /* nr of transitions, this must be last!!! */
+};
+
+/*
+ * ib2ehca_qp_state maps IB to ehca qp_state
+ * returns ehca qp state corresponding to given ib qp state
+ */
+static inline enum ehca_qp_state ib2ehca_qp_state(enum ib_qp_state ib_qp_state)
+{
+ switch (ib_qp_state) {
+ case IB_QPS_RESET:
+ return EHCA_QPS_RESET;
+ case IB_QPS_INIT:
+ return EHCA_QPS_INIT;
+ case IB_QPS_RTR:
+ return EHCA_QPS_RTR;
+ case IB_QPS_RTS:
+ return EHCA_QPS_RTS;
+ case IB_QPS_SQD:
+ return EHCA_QPS_SQD;
+ case IB_QPS_SQE:
+ return EHCA_QPS_SQE;
+ case IB_QPS_ERR:
+ return EHCA_QPS_ERR;
+ default:
+ ehca_gen_err("invalid ib_qp_state=%x", ib_qp_state);
+ return -EINVAL;
+ }
+}
+
+/*
+ * ehca2ib_qp_state maps ehca to IB qp_state
+ * returns ib qp state corresponding to given ehca qp state
+ */
+static inline enum ib_qp_state ehca2ib_qp_state(enum ehca_qp_state
+ ehca_qp_state)
+{
+ switch (ehca_qp_state) {
+ case EHCA_QPS_RESET:
+ return IB_QPS_RESET;
+ case EHCA_QPS_INIT:
+ return IB_QPS_INIT;
+ case EHCA_QPS_RTR:
+ return IB_QPS_RTR;
+ case EHCA_QPS_RTS:
+ return IB_QPS_RTS;
+ case EHCA_QPS_SQD:
+ return IB_QPS_SQD;
+ case EHCA_QPS_SQE:
+ return IB_QPS_SQE;
+ case EHCA_QPS_ERR:
+ return IB_QPS_ERR;
+ default:
+ ehca_gen_err("invalid ehca_qp_state=%x", ehca_qp_state);
+ return -EINVAL;
+ }
+}
+
+/*
+ * ehca_qp_type used as index for req_attr and opt_attr of
+ * struct ehca_modqp_statetrans
+ */
+enum ehca_qp_type {
+ QPT_RC = 0,
+ QPT_UC = 1,
+ QPT_UD = 2,
+ QPT_SQP = 3,
+ QPT_MAX
+};
+
+/*
+ * ib2ehcaqptype maps Ib to ehca qp_type
+ * returns ehca qp type corresponding to ib qp type
+ */
+static inline enum ehca_qp_type ib2ehcaqptype(enum ib_qp_type ibqptype)
+{
+ switch (ibqptype) {
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ return QPT_SQP;
+ case IB_QPT_RC:
+ return QPT_RC;
+ case IB_QPT_UC:
+ return QPT_UC;
+ case IB_QPT_UD:
+ return QPT_UD;
+ default:
+ ehca_gen_err("Invalid ibqptype=%x", ibqptype);
+ return -EINVAL;
+ }
+}
+
+static inline enum ib_qp_statetrans get_modqp_statetrans(int ib_fromstate,
+ int ib_tostate)
+{
+ int index = -EINVAL;
+ switch (ib_tostate) {
+ case IB_QPS_RESET:
+ index = IB_QPST_ANY2RESET;
+ break;
+ case IB_QPS_INIT:
+ switch (ib_fromstate) {
+ case IB_QPS_RESET:
+ index = IB_QPST_RESET2INIT;
+ break;
+ case IB_QPS_INIT:
+ index = IB_QPST_INIT2INIT;
+ break;
+ }
+ break;
+ case IB_QPS_RTR:
+ if (ib_fromstate == IB_QPS_INIT)
+ index = IB_QPST_INIT2RTR;
+ break;
+ case IB_QPS_RTS:
+ switch (ib_fromstate) {
+ case IB_QPS_RTR:
+ index = IB_QPST_RTR2RTS;
+ break;
+ case IB_QPS_RTS:
+ index = IB_QPST_RTS2RTS;
+ break;
+ case IB_QPS_SQD:
+ index = IB_QPST_SQD2RTS;
+ break;
+ case IB_QPS_SQE:
+ index = IB_QPST_SQE2RTS;
+ break;
+ }
+ break;
+ case IB_QPS_SQD:
+ if (ib_fromstate == IB_QPS_RTS)
+ index = IB_QPST_RTS2SQD;
+ break;
+ case IB_QPS_SQE:
+ break;
+ case IB_QPS_ERR:
+ index = IB_QPST_ANY2ERR;
+ break;
+ default:
+ break;
+ }
+ return index;
+}
+
+/*
+ * ibqptype2servicetype returns hcp service type corresponding to given
+ * ib qp type used by create_qp()
+ */
+static inline int ibqptype2servicetype(enum ib_qp_type ibqptype)
+{
+ switch (ibqptype) {
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ return ST_UD;
+ case IB_QPT_RC:
+ return ST_RC;
+ case IB_QPT_UC:
+ return ST_UC;
+ case IB_QPT_UD:
+ return ST_UD;
+ case IB_QPT_RAW_IPV6:
+ return -EINVAL;
+ case IB_QPT_RAW_ETHERTYPE:
+ return -EINVAL;
+ default:
+ ehca_gen_err("Invalid ibqptype=%x", ibqptype);
+ return -EINVAL;
+ }
+}
+
+/*
+ * init userspace queue info from ipz_queue data
+ */
+static inline void queue2resp(struct ipzu_queue_resp *resp,
+ struct ipz_queue *queue)
+{
+ resp->qe_size = queue->qe_size;
+ resp->act_nr_of_sg = queue->act_nr_of_sg;
+ resp->queue_length = queue->queue_length;
+ resp->pagesize = queue->pagesize;
+ resp->toggle_state = queue->toggle_state;
+ resp->offset = queue->offset;
+}
+
+/*
+ * init_qp_queue initializes/constructs r/squeue and registers queue pages.
+ */
+static inline int init_qp_queue(struct ehca_shca *shca,
+ struct ehca_pd *pd,
+ struct ehca_qp *my_qp,
+ struct ipz_queue *queue,
+ int q_type,
+ u64 expected_hret,
+ struct ehca_alloc_queue_parms *parms,
+ int wqe_size)
+{
+ int ret, cnt, ipz_rc, nr_q_pages;
+ void *vpage;
+ u64 rpage, h_ret;
+ struct ib_device *ib_dev = &shca->ib_device;
+ struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle;
+
+ if (!parms->queue_size)
+ return 0;
+
+ if (parms->is_small) {
+ nr_q_pages = 1;
+ ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
+ 128 << parms->page_size,
+ wqe_size, parms->act_nr_sges, 1);
+ } else {
+ nr_q_pages = parms->queue_size;
+ ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
+ EHCA_PAGESIZE, wqe_size,
+ parms->act_nr_sges, 0);
+ }
+
+ if (!ipz_rc) {
+ ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%i",
+ ipz_rc);
+ return -EBUSY;
+ }
+
+ /* register queue pages */
+ for (cnt = 0; cnt < nr_q_pages; cnt++) {
+ vpage = ipz_qpageit_get_inc(queue);
+ if (!vpage) {
+ ehca_err(ib_dev, "ipz_qpageit_get_inc() "
+ "failed p_vpage= %p", vpage);
+ ret = -EINVAL;
+ goto init_qp_queue1;
+ }
+ rpage = __pa(vpage);
+
+ h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ NULL, 0, q_type,
+ rpage, parms->is_small ? 0 : 1,
+ my_qp->galpas.kernel);
+ if (cnt == (nr_q_pages - 1)) { /* last page! */
+ if (h_ret != expected_hret) {
+ ehca_err(ib_dev, "hipz_qp_register_rpage() "
+ "h_ret=%lli", h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto init_qp_queue1;
+ }
+ vpage = ipz_qpageit_get_inc(&my_qp->ipz_rqueue);
+ if (vpage) {
+ ehca_err(ib_dev, "ipz_qpageit_get_inc() "
+ "should not succeed vpage=%p", vpage);
+ ret = -EINVAL;
+ goto init_qp_queue1;
+ }
+ } else {
+ if (h_ret != H_PAGE_REGISTERED) {
+ ehca_err(ib_dev, "hipz_qp_register_rpage() "
+ "h_ret=%lli", h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto init_qp_queue1;
+ }
+ }
+ }
+
+ ipz_qeit_reset(queue);
+
+ return 0;
+
+init_qp_queue1:
+ ipz_queue_dtor(pd, queue);
+ return ret;
+}
+
+static inline int ehca_calc_wqe_size(int act_nr_sge, int is_llqp)
+{
+ if (is_llqp)
+ return 128 << act_nr_sge;
+ else
+ return offsetof(struct ehca_wqe,
+ u.nud.sg_list[act_nr_sge]);
+}
+
+static void ehca_determine_small_queue(struct ehca_alloc_queue_parms *queue,
+ int req_nr_sge, int is_llqp)
+{
+ u32 wqe_size, q_size;
+ int act_nr_sge = req_nr_sge;
+
+ if (!is_llqp)
+ /* round up #SGEs so WQE size is a power of 2 */
+ for (act_nr_sge = 4; act_nr_sge <= 252;
+ act_nr_sge = 4 + 2 * act_nr_sge)
+ if (act_nr_sge >= req_nr_sge)
+ break;
+
+ wqe_size = ehca_calc_wqe_size(act_nr_sge, is_llqp);
+ q_size = wqe_size * (queue->max_wr + 1);
+
+ if (q_size <= 512)
+ queue->page_size = 2;
+ else if (q_size <= 1024)
+ queue->page_size = 3;
+ else
+ queue->page_size = 0;
+
+ queue->is_small = (queue->page_size != 0);
+}
+
+/* needs to be called with cq->spinlock held */
+void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq)
+{
+ struct list_head *list, *node;
+
+ /* TODO: support low latency QPs */
+ if (qp->ext_type == EQPT_LLQP)
+ return;
+
+ if (on_sq) {
+ list = &qp->send_cq->sqp_err_list;
+ node = &qp->sq_err_node;
+ } else {
+ list = &qp->recv_cq->rqp_err_list;
+ node = &qp->rq_err_node;
+ }
+
+ if (list_empty(node))
+ list_add_tail(node, list);
+
+ return;
+}
+
+static void del_from_err_list(struct ehca_cq *cq, struct list_head *node)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->spinlock, flags);
+
+ if (!list_empty(node))
+ list_del_init(node);
+
+ spin_unlock_irqrestore(&cq->spinlock, flags);
+}
+
+static void reset_queue_map(struct ehca_queue_map *qmap)
+{
+ int i;
+
+ qmap->tail = qmap->entries - 1;
+ qmap->left_to_poll = 0;
+ qmap->next_wqe_idx = 0;
+ for (i = 0; i < qmap->entries; i++) {
+ qmap->map[i].reported = 1;
+ qmap->map[i].cqe_req = 0;
+ }
+}
+
+/*
+ * Create an ib_qp struct that is either a QP or an SRQ, depending on
+ * the value of the is_srq parameter. If init_attr and srq_init_attr share
+ * fields, the field out of init_attr is used.
+ */
+static struct ehca_qp *internal_create_qp(
+ struct ib_pd *pd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata, int is_srq)
+{
+ struct ehca_qp *my_qp, *my_srq = NULL;
+ struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+ ib_device);
+ struct ib_ucontext *context = NULL;
+ u64 h_ret;
+ int is_llqp = 0, has_srq = 0, is_user = 0;
+ int qp_type, max_send_sge, max_recv_sge, ret;
+
+ /* h_call's out parameters */
+ struct ehca_alloc_qp_parms parms;
+ u32 swqe_size = 0, rwqe_size = 0, ib_qp_num;
+ unsigned long flags;
+
+ if (!atomic_add_unless(&shca->num_qps, 1, shca->max_num_qps)) {
+ ehca_err(pd->device, "Unable to create QP, max number of %i "
+ "QPs reached.", shca->max_num_qps);
+ ehca_err(pd->device, "To increase the maximum number of QPs "
+ "use the number_of_qps module parameter.\n");
+ return ERR_PTR(-ENOSPC);
+ }
+
+ if (init_attr->create_flags) {
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ memset(&parms, 0, sizeof(parms));
+ qp_type = init_attr->qp_type;
+
+ if (init_attr->sq_sig_type != IB_SIGNAL_REQ_WR &&
+ init_attr->sq_sig_type != IB_SIGNAL_ALL_WR) {
+ ehca_err(pd->device, "init_attr->sg_sig_type=%x not allowed",
+ init_attr->sq_sig_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ /* save LLQP info */
+ if (qp_type & 0x80) {
+ is_llqp = 1;
+ parms.ext_type = EQPT_LLQP;
+ parms.ll_comp_flags = qp_type & LLQP_COMP_MASK;
+ }
+ qp_type &= 0x1F;
+ init_attr->qp_type &= 0x1F;
+
+ /* handle SRQ base QPs */
+ if (init_attr->srq) {
+ my_srq = container_of(init_attr->srq, struct ehca_qp, ib_srq);
+
+ if (qp_type == IB_QPT_UC) {
+ ehca_err(pd->device, "UC with SRQ not supported");
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ has_srq = 1;
+ parms.ext_type = EQPT_SRQBASE;
+ parms.srq_qpn = my_srq->real_qp_num;
+ }
+
+ if (is_llqp && has_srq) {
+ ehca_err(pd->device, "LLQPs can't have an SRQ");
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ /* handle SRQs */
+ if (is_srq) {
+ parms.ext_type = EQPT_SRQ;
+ parms.srq_limit = srq_init_attr->attr.srq_limit;
+ if (init_attr->cap.max_recv_sge > 3) {
+ ehca_err(pd->device, "no more than three SGEs "
+ "supported for SRQ pd=%p max_sge=%x",
+ pd, init_attr->cap.max_recv_sge);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ }
+
+ /* check QP type */
+ if (qp_type != IB_QPT_UD &&
+ qp_type != IB_QPT_UC &&
+ qp_type != IB_QPT_RC &&
+ qp_type != IB_QPT_SMI &&
+ qp_type != IB_QPT_GSI) {
+ ehca_err(pd->device, "wrong QP Type=%x", qp_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ if (is_llqp) {
+ switch (qp_type) {
+ case IB_QPT_RC:
+ if ((init_attr->cap.max_send_wr > 255) ||
+ (init_attr->cap.max_recv_wr > 255)) {
+ ehca_err(pd->device,
+ "Invalid Number of max_sq_wr=%x "
+ "or max_rq_wr=%x for RC LLQP",
+ init_attr->cap.max_send_wr,
+ init_attr->cap.max_recv_wr);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ break;
+ case IB_QPT_UD:
+ if (!EHCA_BMASK_GET(HCA_CAP_UD_LL_QP, shca->hca_cap)) {
+ ehca_err(pd->device, "UD LLQP not supported "
+ "by this adapter");
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-ENOSYS);
+ }
+ if (!(init_attr->cap.max_send_sge <= 5
+ && init_attr->cap.max_send_sge >= 1
+ && init_attr->cap.max_recv_sge <= 5
+ && init_attr->cap.max_recv_sge >= 1)) {
+ ehca_err(pd->device,
+ "Invalid Number of max_send_sge=%x "
+ "or max_recv_sge=%x for UD LLQP",
+ init_attr->cap.max_send_sge,
+ init_attr->cap.max_recv_sge);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ } else if (init_attr->cap.max_send_wr > 255) {
+ ehca_err(pd->device,
+ "Invalid Number of "
+ "max_send_wr=%x for UD QP_TYPE=%x",
+ init_attr->cap.max_send_wr, qp_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ break;
+ default:
+ ehca_err(pd->device, "unsupported LL QP Type=%x",
+ qp_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ } else {
+ int max_sge = (qp_type == IB_QPT_UD || qp_type == IB_QPT_SMI
+ || qp_type == IB_QPT_GSI) ? 250 : 252;
+
+ if (init_attr->cap.max_send_sge > max_sge
+ || init_attr->cap.max_recv_sge > max_sge) {
+ ehca_err(pd->device, "Invalid number of SGEs requested "
+ "send_sge=%x recv_sge=%x max_sge=%x",
+ init_attr->cap.max_send_sge,
+ init_attr->cap.max_recv_sge, max_sge);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ }
+
+ my_qp = kmem_cache_zalloc(qp_cache, GFP_KERNEL);
+ if (!my_qp) {
+ ehca_err(pd->device, "pd=%p not enough memory to alloc qp", pd);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ if (pd->uobject && udata) {
+ is_user = 1;
+ context = pd->uobject->context;
+ }
+
+ atomic_set(&my_qp->nr_events, 0);
+ init_waitqueue_head(&my_qp->wait_completion);
+ spin_lock_init(&my_qp->spinlock_s);
+ spin_lock_init(&my_qp->spinlock_r);
+ my_qp->qp_type = qp_type;
+ my_qp->ext_type = parms.ext_type;
+ my_qp->state = IB_QPS_RESET;
+
+ if (init_attr->recv_cq)
+ my_qp->recv_cq =
+ container_of(init_attr->recv_cq, struct ehca_cq, ib_cq);
+ if (init_attr->send_cq)
+ my_qp->send_cq =
+ container_of(init_attr->send_cq, struct ehca_cq, ib_cq);
+
+ idr_preload(GFP_KERNEL);
+ write_lock_irqsave(&ehca_qp_idr_lock, flags);
+
+ ret = idr_alloc(&ehca_qp_idr, my_qp, 0, 0x2000000, GFP_NOWAIT);
+ if (ret >= 0)
+ my_qp->token = ret;
+
+ write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+ idr_preload_end();
+ if (ret < 0) {
+ if (ret == -ENOSPC) {
+ ret = -EINVAL;
+ ehca_err(pd->device, "Invalid number of qp");
+ } else {
+ ret = -ENOMEM;
+ ehca_err(pd->device, "Can't allocate new idr entry.");
+ }
+ goto create_qp_exit0;
+ }
+
+ if (has_srq)
+ parms.srq_token = my_qp->token;
+
+ parms.servicetype = ibqptype2servicetype(qp_type);
+ if (parms.servicetype < 0) {
+ ret = -EINVAL;
+ ehca_err(pd->device, "Invalid qp_type=%x", qp_type);
+ goto create_qp_exit1;
+ }
+
+ /* Always signal by WQE so we can hide circ. WQEs */
+ parms.sigtype = HCALL_SIGT_BY_WQE;
+
+ /* UD_AV CIRCUMVENTION */
+ max_send_sge = init_attr->cap.max_send_sge;
+ max_recv_sge = init_attr->cap.max_recv_sge;
+ if (parms.servicetype == ST_UD && !is_llqp) {
+ max_send_sge += 2;
+ max_recv_sge += 2;
+ }
+
+ parms.token = my_qp->token;
+ parms.eq_handle = shca->eq.ipz_eq_handle;
+ parms.pd = my_pd->fw_pd;
+ if (my_qp->send_cq)
+ parms.send_cq_handle = my_qp->send_cq->ipz_cq_handle;
+ if (my_qp->recv_cq)
+ parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle;
+
+ parms.squeue.max_wr = init_attr->cap.max_send_wr;
+ parms.rqueue.max_wr = init_attr->cap.max_recv_wr;
+ parms.squeue.max_sge = max_send_sge;
+ parms.rqueue.max_sge = max_recv_sge;
+
+ /* RC QPs need one more SWQE for unsolicited ack circumvention */
+ if (qp_type == IB_QPT_RC)
+ parms.squeue.max_wr++;
+
+ if (EHCA_BMASK_GET(HCA_CAP_MINI_QP, shca->hca_cap)) {
+ if (HAS_SQ(my_qp))
+ ehca_determine_small_queue(
+ &parms.squeue, max_send_sge, is_llqp);
+ if (HAS_RQ(my_qp))
+ ehca_determine_small_queue(
+ &parms.rqueue, max_recv_sge, is_llqp);
+ parms.qp_storage =
+ (parms.squeue.is_small || parms.rqueue.is_small);
+ }
+
+ h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms, is_user);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(pd->device, "h_alloc_resource_qp() failed h_ret=%lli",
+ h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto create_qp_exit1;
+ }
+
+ ib_qp_num = my_qp->real_qp_num = parms.real_qp_num;
+ my_qp->ipz_qp_handle = parms.qp_handle;
+ my_qp->galpas = parms.galpas;
+
+ swqe_size = ehca_calc_wqe_size(parms.squeue.act_nr_sges, is_llqp);
+ rwqe_size = ehca_calc_wqe_size(parms.rqueue.act_nr_sges, is_llqp);
+
+ switch (qp_type) {
+ case IB_QPT_RC:
+ if (is_llqp) {
+ parms.squeue.act_nr_sges = 1;
+ parms.rqueue.act_nr_sges = 1;
+ }
+ /* hide the extra WQE */
+ parms.squeue.act_nr_wqes--;
+ break;
+ case IB_QPT_UD:
+ case IB_QPT_GSI:
+ case IB_QPT_SMI:
+ /* UD circumvention */
+ if (is_llqp) {
+ parms.squeue.act_nr_sges = 1;
+ parms.rqueue.act_nr_sges = 1;
+ } else {
+ parms.squeue.act_nr_sges -= 2;
+ parms.rqueue.act_nr_sges -= 2;
+ }
+
+ if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) {
+ parms.squeue.act_nr_wqes = init_attr->cap.max_send_wr;
+ parms.rqueue.act_nr_wqes = init_attr->cap.max_recv_wr;
+ parms.squeue.act_nr_sges = init_attr->cap.max_send_sge;
+ parms.rqueue.act_nr_sges = init_attr->cap.max_recv_sge;
+ ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+
+ /* initialize r/squeue and register queue pages */
+ if (HAS_SQ(my_qp)) {
+ ret = init_qp_queue(
+ shca, my_pd, my_qp, &my_qp->ipz_squeue, 0,
+ HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS,
+ &parms.squeue, swqe_size);
+ if (ret) {
+ ehca_err(pd->device, "Couldn't initialize squeue "
+ "and pages ret=%i", ret);
+ goto create_qp_exit2;
+ }
+
+ if (!is_user) {
+ my_qp->sq_map.entries = my_qp->ipz_squeue.queue_length /
+ my_qp->ipz_squeue.qe_size;
+ my_qp->sq_map.map = vmalloc(my_qp->sq_map.entries *
+ sizeof(struct ehca_qmap_entry));
+ if (!my_qp->sq_map.map) {
+ ehca_err(pd->device, "Couldn't allocate squeue "
+ "map ret=%i", ret);
+ goto create_qp_exit3;
+ }
+ INIT_LIST_HEAD(&my_qp->sq_err_node);
+ /* to avoid the generation of bogus flush CQEs */
+ reset_queue_map(&my_qp->sq_map);
+ }
+ }
+
+ if (HAS_RQ(my_qp)) {
+ ret = init_qp_queue(
+ shca, my_pd, my_qp, &my_qp->ipz_rqueue, 1,
+ H_SUCCESS, &parms.rqueue, rwqe_size);
+ if (ret) {
+ ehca_err(pd->device, "Couldn't initialize rqueue "
+ "and pages ret=%i", ret);
+ goto create_qp_exit4;
+ }
+ if (!is_user) {
+ my_qp->rq_map.entries = my_qp->ipz_rqueue.queue_length /
+ my_qp->ipz_rqueue.qe_size;
+ my_qp->rq_map.map = vmalloc(my_qp->rq_map.entries *
+ sizeof(struct ehca_qmap_entry));
+ if (!my_qp->rq_map.map) {
+ ehca_err(pd->device, "Couldn't allocate squeue "
+ "map ret=%i", ret);
+ goto create_qp_exit5;
+ }
+ INIT_LIST_HEAD(&my_qp->rq_err_node);
+ /* to avoid the generation of bogus flush CQEs */
+ reset_queue_map(&my_qp->rq_map);
+ }
+ } else if (init_attr->srq && !is_user) {
+ /* this is a base QP, use the queue map of the SRQ */
+ my_qp->rq_map = my_srq->rq_map;
+ INIT_LIST_HEAD(&my_qp->rq_err_node);
+
+ my_qp->ipz_rqueue = my_srq->ipz_rqueue;
+ }
+
+ if (is_srq) {
+ my_qp->ib_srq.pd = &my_pd->ib_pd;
+ my_qp->ib_srq.device = my_pd->ib_pd.device;
+
+ my_qp->ib_srq.srq_context = init_attr->qp_context;
+ my_qp->ib_srq.event_handler = init_attr->event_handler;
+ } else {
+ my_qp->ib_qp.qp_num = ib_qp_num;
+ my_qp->ib_qp.pd = &my_pd->ib_pd;
+ my_qp->ib_qp.device = my_pd->ib_pd.device;
+
+ my_qp->ib_qp.recv_cq = init_attr->recv_cq;
+ my_qp->ib_qp.send_cq = init_attr->send_cq;
+
+ my_qp->ib_qp.qp_type = qp_type;
+ my_qp->ib_qp.srq = init_attr->srq;
+
+ my_qp->ib_qp.qp_context = init_attr->qp_context;
+ my_qp->ib_qp.event_handler = init_attr->event_handler;
+ }
+
+ init_attr->cap.max_inline_data = 0; /* not supported yet */
+ init_attr->cap.max_recv_sge = parms.rqueue.act_nr_sges;
+ init_attr->cap.max_recv_wr = parms.rqueue.act_nr_wqes;
+ init_attr->cap.max_send_sge = parms.squeue.act_nr_sges;
+ init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes;
+ my_qp->init_attr = *init_attr;
+
+ if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
+ shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
+ &my_qp->ib_qp;
+ if (ehca_nr_ports < 0) {
+ /* alloc array to cache subsequent modify qp parms
+ * for autodetect mode
+ */
+ my_qp->mod_qp_parm =
+ kzalloc(EHCA_MOD_QP_PARM_MAX *
+ sizeof(*my_qp->mod_qp_parm),
+ GFP_KERNEL);
+ if (!my_qp->mod_qp_parm) {
+ ehca_err(pd->device,
+ "Could not alloc mod_qp_parm");
+ goto create_qp_exit5;
+ }
+ }
+ }
+
+ /* NOTE: define_apq0() not supported yet */
+ if (qp_type == IB_QPT_GSI) {
+ h_ret = ehca_define_sqp(shca, my_qp, init_attr);
+ if (h_ret != H_SUCCESS) {
+ kfree(my_qp->mod_qp_parm);
+ my_qp->mod_qp_parm = NULL;
+ /* the QP pointer is no longer valid */
+ shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
+ NULL;
+ ret = ehca2ib_return_code(h_ret);
+ goto create_qp_exit6;
+ }
+ }
+
+ if (my_qp->send_cq) {
+ ret = ehca_cq_assign_qp(my_qp->send_cq, my_qp);
+ if (ret) {
+ ehca_err(pd->device,
+ "Couldn't assign qp to send_cq ret=%i", ret);
+ goto create_qp_exit7;
+ }
+ }
+
+ /* copy queues, galpa data to user space */
+ if (context && udata) {
+ struct ehca_create_qp_resp resp;
+ memset(&resp, 0, sizeof(resp));
+
+ resp.qp_num = my_qp->real_qp_num;
+ resp.token = my_qp->token;
+ resp.qp_type = my_qp->qp_type;
+ resp.ext_type = my_qp->ext_type;
+ resp.qkey = my_qp->qkey;
+ resp.real_qp_num = my_qp->real_qp_num;
+
+ if (HAS_SQ(my_qp))
+ queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue);
+ if (HAS_RQ(my_qp))
+ queue2resp(&resp.ipz_rqueue, &my_qp->ipz_rqueue);
+ resp.fw_handle_ofs = (u32)
+ (my_qp->galpas.user.fw_handle & (PAGE_SIZE - 1));
+
+ if (ib_copy_to_udata(udata, &resp, sizeof resp)) {
+ ehca_err(pd->device, "Copy to udata failed");
+ ret = -EINVAL;
+ goto create_qp_exit8;
+ }
+ }
+
+ return my_qp;
+
+create_qp_exit8:
+ ehca_cq_unassign_qp(my_qp->send_cq, my_qp->real_qp_num);
+
+create_qp_exit7:
+ kfree(my_qp->mod_qp_parm);
+
+create_qp_exit6:
+ if (HAS_RQ(my_qp) && !is_user)
+ vfree(my_qp->rq_map.map);
+
+create_qp_exit5:
+ if (HAS_RQ(my_qp))
+ ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
+
+create_qp_exit4:
+ if (HAS_SQ(my_qp) && !is_user)
+ vfree(my_qp->sq_map.map);
+
+create_qp_exit3:
+ if (HAS_SQ(my_qp))
+ ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
+
+create_qp_exit2:
+ hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
+
+create_qp_exit1:
+ write_lock_irqsave(&ehca_qp_idr_lock, flags);
+ idr_remove(&ehca_qp_idr, my_qp->token);
+ write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+create_qp_exit0:
+ kmem_cache_free(qp_cache, my_qp);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(ret);
+}
+
+struct ib_qp *ehca_create_qp(struct ib_pd *pd,
+ struct ib_qp_init_attr *qp_init_attr,
+ struct ib_udata *udata)
+{
+ struct ehca_qp *ret;
+
+ ret = internal_create_qp(pd, qp_init_attr, NULL, udata, 0);
+ return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp;
+}
+
+static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
+ struct ib_uobject *uobject);
+
+struct ib_srq *ehca_create_srq(struct ib_pd *pd,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata)
+{
+ struct ib_qp_init_attr qp_init_attr;
+ struct ehca_qp *my_qp;
+ struct ib_srq *ret;
+ struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+ ib_device);
+ struct hcp_modify_qp_control_block *mqpcb;
+ u64 hret, update_mask;
+
+ if (srq_init_attr->srq_type != IB_SRQT_BASIC)
+ return ERR_PTR(-ENOSYS);
+
+ /* For common attributes, internal_create_qp() takes its info
+ * out of qp_init_attr, so copy all common attrs there.
+ */
+ memset(&qp_init_attr, 0, sizeof(qp_init_attr));
+ qp_init_attr.event_handler = srq_init_attr->event_handler;
+ qp_init_attr.qp_context = srq_init_attr->srq_context;
+ qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
+ qp_init_attr.qp_type = IB_QPT_RC;
+ qp_init_attr.cap.max_recv_wr = srq_init_attr->attr.max_wr;
+ qp_init_attr.cap.max_recv_sge = srq_init_attr->attr.max_sge;
+
+ my_qp = internal_create_qp(pd, &qp_init_attr, srq_init_attr, udata, 1);
+ if (IS_ERR(my_qp))
+ return (struct ib_srq *)my_qp;
+
+ /* copy back return values */
+ srq_init_attr->attr.max_wr = qp_init_attr.cap.max_recv_wr;
+ srq_init_attr->attr.max_sge = 3;
+
+ /* drive SRQ into RTR state */
+ mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!mqpcb) {
+ ehca_err(pd->device, "Could not get zeroed page for mqpcb "
+ "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
+ ret = ERR_PTR(-ENOMEM);
+ goto create_srq1;
+ }
+
+ mqpcb->qp_state = EHCA_QPS_INIT;
+ mqpcb->prim_phys_port = 1;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
+ hret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+ if (hret != H_SUCCESS) {
+ ehca_err(pd->device, "Could not modify SRQ to INIT "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, hret);
+ goto create_srq2;
+ }
+
+ mqpcb->qp_enable = 1;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
+ hret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+ if (hret != H_SUCCESS) {
+ ehca_err(pd->device, "Could not enable SRQ "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, hret);
+ goto create_srq2;
+ }
+
+ mqpcb->qp_state = EHCA_QPS_RTR;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
+ hret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+ if (hret != H_SUCCESS) {
+ ehca_err(pd->device, "Could not modify SRQ to RTR "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, hret);
+ goto create_srq2;
+ }
+
+ ehca_free_fw_ctrlblock(mqpcb);
+
+ return &my_qp->ib_srq;
+
+create_srq2:
+ ret = ERR_PTR(ehca2ib_return_code(hret));
+ ehca_free_fw_ctrlblock(mqpcb);
+
+create_srq1:
+ internal_destroy_qp(pd->device, my_qp, my_qp->ib_srq.uobject);
+
+ return ret;
+}
+
+/*
+ * prepare_sqe_rts called by internal_modify_qp() at trans sqe -> rts
+ * set purge bit of bad wqe and subsequent wqes to avoid reentering sqe
+ * returns total number of bad wqes in bad_wqe_cnt
+ */
+static int prepare_sqe_rts(struct ehca_qp *my_qp, struct ehca_shca *shca,
+ int *bad_wqe_cnt)
+{
+ u64 h_ret;
+ struct ipz_queue *squeue;
+ void *bad_send_wqe_p, *bad_send_wqe_v;
+ u64 q_ofs;
+ struct ehca_wqe *wqe;
+ int qp_num = my_qp->ib_qp.qp_num;
+
+ /* get send wqe pointer */
+ h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle, &my_qp->pf,
+ &bad_send_wqe_p, NULL, 2);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_h_disable_and_get_wqe() failed"
+ " ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, qp_num, h_ret);
+ return ehca2ib_return_code(h_ret);
+ }
+ bad_send_wqe_p = (void *)((u64)bad_send_wqe_p & (~(1L << 63)));
+ ehca_dbg(&shca->ib_device, "qp_num=%x bad_send_wqe_p=%p",
+ qp_num, bad_send_wqe_p);
+ /* convert wqe pointer to vadr */
+ bad_send_wqe_v = __va((u64)bad_send_wqe_p);
+ if (ehca_debug_level >= 2)
+ ehca_dmp(bad_send_wqe_v, 32, "qp_num=%x bad_wqe", qp_num);
+ squeue = &my_qp->ipz_squeue;
+ if (ipz_queue_abs_to_offset(squeue, (u64)bad_send_wqe_p, &q_ofs)) {
+ ehca_err(&shca->ib_device, "failed to get wqe offset qp_num=%x"
+ " bad_send_wqe_p=%p", qp_num, bad_send_wqe_p);
+ return -EFAULT;
+ }
+
+ /* loop sets wqe's purge bit */
+ wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
+ *bad_wqe_cnt = 0;
+ while (wqe->optype != 0xff && wqe->wqef != 0xff) {
+ if (ehca_debug_level >= 2)
+ ehca_dmp(wqe, 32, "qp_num=%x wqe", qp_num);
+ wqe->nr_of_data_seg = 0; /* suppress data access */
+ wqe->wqef = WQEF_PURGE; /* WQE to be purged */
+ q_ofs = ipz_queue_advance_offset(squeue, q_ofs);
+ wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
+ *bad_wqe_cnt = (*bad_wqe_cnt)+1;
+ }
+ /*
+ * bad wqe will be reprocessed and ignored when pol_cq() is called,
+ * i.e. nr of wqes with flush error status is one less
+ */
+ ehca_dbg(&shca->ib_device, "qp_num=%x flusherr_wqe_cnt=%x",
+ qp_num, (*bad_wqe_cnt)-1);
+ wqe->wqef = 0;
+
+ return 0;
+}
+
+static int calc_left_cqes(u64 wqe_p, struct ipz_queue *ipz_queue,
+ struct ehca_queue_map *qmap)
+{
+ void *wqe_v;
+ u64 q_ofs;
+ u32 wqe_idx;
+ unsigned int tail_idx;
+
+ /* convert real to abs address */
+ wqe_p = wqe_p & (~(1UL << 63));
+
+ wqe_v = __va(wqe_p);
+
+ if (ipz_queue_abs_to_offset(ipz_queue, wqe_p, &q_ofs)) {
+ ehca_gen_err("Invalid offset for calculating left cqes "
+ "wqe_p=%#llx wqe_v=%p\n", wqe_p, wqe_v);
+ return -EFAULT;
+ }
+
+ tail_idx = next_index(qmap->tail, qmap->entries);
+ wqe_idx = q_ofs / ipz_queue->qe_size;
+
+ /* check all processed wqes, whether a cqe is requested or not */
+ while (tail_idx != wqe_idx) {
+ if (qmap->map[tail_idx].cqe_req)
+ qmap->left_to_poll++;
+ tail_idx = next_index(tail_idx, qmap->entries);
+ }
+ /* save index in queue, where we have to start flushing */
+ qmap->next_wqe_idx = wqe_idx;
+ return 0;
+}
+
+static int check_for_left_cqes(struct ehca_qp *my_qp, struct ehca_shca *shca)
+{
+ u64 h_ret;
+ void *send_wqe_p, *recv_wqe_p;
+ int ret;
+ unsigned long flags;
+ int qp_num = my_qp->ib_qp.qp_num;
+
+ /* this hcall is not supported on base QPs */
+ if (my_qp->ext_type != EQPT_SRQBASE) {
+ /* get send and receive wqe pointer */
+ h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle, &my_qp->pf,
+ &send_wqe_p, &recv_wqe_p, 4);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "disable_and_get_wqe() "
+ "failed ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, qp_num, h_ret);
+ return ehca2ib_return_code(h_ret);
+ }
+
+ /*
+ * acquire lock to ensure that nobody is polling the cq which
+ * could mean that the qmap->tail pointer is in an
+ * inconsistent state.
+ */
+ spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
+ ret = calc_left_cqes((u64)send_wqe_p, &my_qp->ipz_squeue,
+ &my_qp->sq_map);
+ spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
+ if (ret)
+ return ret;
+
+
+ spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
+ ret = calc_left_cqes((u64)recv_wqe_p, &my_qp->ipz_rqueue,
+ &my_qp->rq_map);
+ spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
+ if (ret)
+ return ret;
+ } else {
+ spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
+ my_qp->sq_map.left_to_poll = 0;
+ my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
+ my_qp->sq_map.entries);
+ spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
+
+ spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
+ my_qp->rq_map.left_to_poll = 0;
+ my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
+ my_qp->rq_map.entries);
+ spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
+ }
+
+ /* this assures flush cqes being generated only for pending wqes */
+ if ((my_qp->sq_map.left_to_poll == 0) &&
+ (my_qp->rq_map.left_to_poll == 0)) {
+ spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
+ ehca_add_to_err_list(my_qp, 1);
+ spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
+
+ if (HAS_RQ(my_qp)) {
+ spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
+ ehca_add_to_err_list(my_qp, 0);
+ spin_unlock_irqrestore(&my_qp->recv_cq->spinlock,
+ flags);
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * internal_modify_qp with circumvention to handle aqp0 properly
+ * smi_reset2init indicates if this is an internal reset-to-init-call for
+ * smi. This flag must always be zero if called from ehca_modify_qp()!
+ * This internal func was intorduced to avoid recursion of ehca_modify_qp()!
+ */
+static int internal_modify_qp(struct ib_qp *ibqp,
+ struct ib_qp_attr *attr,
+ int attr_mask, int smi_reset2init)
+{
+ enum ib_qp_state qp_cur_state, qp_new_state;
+ int cnt, qp_attr_idx, ret = 0;
+ enum ib_qp_statetrans statetrans;
+ struct hcp_modify_qp_control_block *mqpcb;
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca =
+ container_of(ibqp->pd->device, struct ehca_shca, ib_device);
+ u64 update_mask;
+ u64 h_ret;
+ int bad_wqe_cnt = 0;
+ int is_user = 0;
+ int squeue_locked = 0;
+ unsigned long flags = 0;
+
+ /* do query_qp to obtain current attr values */
+ mqpcb = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
+ if (!mqpcb) {
+ ehca_err(ibqp->device, "Could not get zeroed page for mqpcb "
+ "ehca_qp=%p qp_num=%x ", my_qp, ibqp->qp_num);
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ mqpcb, my_qp->galpas.kernel);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(ibqp->device, "hipz_h_query_qp() failed "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, ibqp->qp_num, h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto modify_qp_exit1;
+ }
+ if (ibqp->uobject)
+ is_user = 1;
+
+ qp_cur_state = ehca2ib_qp_state(mqpcb->qp_state);
+
+ if (qp_cur_state == -EINVAL) { /* invalid qp state */
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid current ehca_qp_state=%x "
+ "ehca_qp=%p qp_num=%x",
+ mqpcb->qp_state, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+ /*
+ * circumvention to set aqp0 initial state to init
+ * as expected by IB spec
+ */
+ if (smi_reset2init == 0 &&
+ ibqp->qp_type == IB_QPT_SMI &&
+ qp_cur_state == IB_QPS_RESET &&
+ (attr_mask & IB_QP_STATE) &&
+ attr->qp_state == IB_QPS_INIT) { /* RESET -> INIT */
+ struct ib_qp_attr smiqp_attr = {
+ .qp_state = IB_QPS_INIT,
+ .port_num = my_qp->init_attr.port_num,
+ .pkey_index = 0,
+ .qkey = 0
+ };
+ int smiqp_attr_mask = IB_QP_STATE | IB_QP_PORT |
+ IB_QP_PKEY_INDEX | IB_QP_QKEY;
+ int smirc = internal_modify_qp(
+ ibqp, &smiqp_attr, smiqp_attr_mask, 1);
+ if (smirc) {
+ ehca_err(ibqp->device, "SMI RESET -> INIT failed. "
+ "ehca_modify_qp() rc=%i", smirc);
+ ret = H_PARAMETER;
+ goto modify_qp_exit1;
+ }
+ qp_cur_state = IB_QPS_INIT;
+ ehca_dbg(ibqp->device, "SMI RESET -> INIT succeeded");
+ }
+ /* is transmitted current state equal to "real" current state */
+ if ((attr_mask & IB_QP_CUR_STATE) &&
+ qp_cur_state != attr->cur_qp_state) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device,
+ "Invalid IB_QP_CUR_STATE attr->curr_qp_state=%x <>"
+ " actual cur_qp_state=%x. ehca_qp=%p qp_num=%x",
+ attr->cur_qp_state, qp_cur_state, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ ehca_dbg(ibqp->device, "ehca_qp=%p qp_num=%x current qp_state=%x "
+ "new qp_state=%x attribute_mask=%x",
+ my_qp, ibqp->qp_num, qp_cur_state, attr->qp_state, attr_mask);
+
+ qp_new_state = attr_mask & IB_QP_STATE ? attr->qp_state : qp_cur_state;
+ if (!smi_reset2init &&
+ !ib_modify_qp_is_ok(qp_cur_state, qp_new_state, ibqp->qp_type,
+ attr_mask, IB_LINK_LAYER_UNSPECIFIED)) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device,
+ "Invalid qp transition new_state=%x cur_state=%x "
+ "ehca_qp=%p qp_num=%x attr_mask=%x", qp_new_state,
+ qp_cur_state, my_qp, ibqp->qp_num, attr_mask);
+ goto modify_qp_exit1;
+ }
+
+ mqpcb->qp_state = ib2ehca_qp_state(qp_new_state);
+ if (mqpcb->qp_state)
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
+ else {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid new qp state=%x "
+ "ehca_qp=%p qp_num=%x",
+ qp_new_state, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ /* retrieve state transition struct to get req and opt attrs */
+ statetrans = get_modqp_statetrans(qp_cur_state, qp_new_state);
+ if (statetrans < 0) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "<INVALID STATE CHANGE> qp_cur_state=%x "
+ "new_qp_state=%x State_xsition=%x ehca_qp=%p "
+ "qp_num=%x", qp_cur_state, qp_new_state,
+ statetrans, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ qp_attr_idx = ib2ehcaqptype(ibqp->qp_type);
+
+ if (qp_attr_idx < 0) {
+ ret = qp_attr_idx;
+ ehca_err(ibqp->device,
+ "Invalid QP type=%x ehca_qp=%p qp_num=%x",
+ ibqp->qp_type, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ ehca_dbg(ibqp->device,
+ "ehca_qp=%p qp_num=%x <VALID STATE CHANGE> qp_state_xsit=%x",
+ my_qp, ibqp->qp_num, statetrans);
+
+ /* eHCA2 rev2 and higher require the SEND_GRH_FLAG to be set
+ * in non-LL UD QPs.
+ */
+ if ((my_qp->qp_type == IB_QPT_UD) &&
+ (my_qp->ext_type != EQPT_LLQP) &&
+ (statetrans == IB_QPST_INIT2RTR) &&
+ (shca->hw_level >= 0x22)) {
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
+ mqpcb->send_grh_flag = 1;
+ }
+
+ /* sqe -> rts: set purge bit of bad wqe before actual trans */
+ if ((my_qp->qp_type == IB_QPT_UD ||
+ my_qp->qp_type == IB_QPT_GSI ||
+ my_qp->qp_type == IB_QPT_SMI) &&
+ statetrans == IB_QPST_SQE2RTS) {
+ /* mark next free wqe if kernel */
+ if (!ibqp->uobject) {
+ struct ehca_wqe *wqe;
+ /* lock send queue */
+ spin_lock_irqsave(&my_qp->spinlock_s, flags);
+ squeue_locked = 1;
+ /* mark next free wqe */
+ wqe = (struct ehca_wqe *)
+ ipz_qeit_get(&my_qp->ipz_squeue);
+ wqe->optype = wqe->wqef = 0xff;
+ ehca_dbg(ibqp->device, "qp_num=%x next_free_wqe=%p",
+ ibqp->qp_num, wqe);
+ }
+ ret = prepare_sqe_rts(my_qp, shca, &bad_wqe_cnt);
+ if (ret) {
+ ehca_err(ibqp->device, "prepare_sqe_rts() failed "
+ "ehca_qp=%p qp_num=%x ret=%i",
+ my_qp, ibqp->qp_num, ret);
+ goto modify_qp_exit2;
+ }
+ }
+
+ /*
+ * enable RDMA_Atomic_Control if reset->init und reliable con
+ * this is necessary since gen2 does not provide that flag,
+ * but pHyp requires it
+ */
+ if (statetrans == IB_QPST_RESET2INIT &&
+ (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_UC)) {
+ mqpcb->rdma_atomic_ctrl = 3;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RDMA_ATOMIC_CTRL, 1);
+ }
+ /* circ. pHyp requires #RDMA/Atomic Resp Res for UC INIT -> RTR */
+ if (statetrans == IB_QPST_INIT2RTR &&
+ (ibqp->qp_type == IB_QPT_UC) &&
+ !(attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)) {
+ mqpcb->rdma_nr_atomic_resp_res = 1; /* default to 1 */
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
+ }
+
+ if (attr_mask & IB_QP_PKEY_INDEX) {
+ if (attr->pkey_index >= 16) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid pkey_index=%x. "
+ "ehca_qp=%p qp_num=%x max_pkey_index=f",
+ attr->pkey_index, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+ mqpcb->prim_p_key_idx = attr->pkey_index;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_P_KEY_IDX, 1);
+ }
+ if (attr_mask & IB_QP_PORT) {
+ struct ehca_sport *sport;
+ struct ehca_qp *aqp1;
+ if (attr->port_num < 1 || attr->port_num > shca->num_ports) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid port=%x. "
+ "ehca_qp=%p qp_num=%x num_ports=%x",
+ attr->port_num, my_qp, ibqp->qp_num,
+ shca->num_ports);
+ goto modify_qp_exit2;
+ }
+ sport = &shca->sport[attr->port_num - 1];
+ if (!sport->ibqp_sqp[IB_QPT_GSI]) {
+ /* should not occur */
+ ret = -EFAULT;
+ ehca_err(ibqp->device, "AQP1 was not created for "
+ "port=%x", attr->port_num);
+ goto modify_qp_exit2;
+ }
+ aqp1 = container_of(sport->ibqp_sqp[IB_QPT_GSI],
+ struct ehca_qp, ib_qp);
+ if (ibqp->qp_type != IB_QPT_GSI &&
+ ibqp->qp_type != IB_QPT_SMI &&
+ aqp1->mod_qp_parm) {
+ /*
+ * firmware will reject this modify_qp() because
+ * port is not activated/initialized fully
+ */
+ ret = -EFAULT;
+ ehca_warn(ibqp->device, "Couldn't modify qp port=%x: "
+ "either port is being activated (try again) "
+ "or cabling issue", attr->port_num);
+ goto modify_qp_exit2;
+ }
+ mqpcb->prim_phys_port = attr->port_num;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_PHYS_PORT, 1);
+ }
+ if (attr_mask & IB_QP_QKEY) {
+ mqpcb->qkey = attr->qkey;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_QKEY, 1);
+ }
+ if (attr_mask & IB_QP_AV) {
+ mqpcb->dlid = attr->ah_attr.dlid;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DLID, 1);
+ mqpcb->source_path_bits = attr->ah_attr.src_path_bits;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS, 1);
+ mqpcb->service_level = attr->ah_attr.sl;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL, 1);
+
+ if (ehca_calc_ipd(shca, mqpcb->prim_phys_port,
+ attr->ah_attr.static_rate,
+ &mqpcb->max_static_rate)) {
+ ret = -EINVAL;
+ goto modify_qp_exit2;
+ }
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE, 1);
+
+ /*
+ * Always supply the GRH flag, even if it's zero, to give the
+ * hypervisor a clear "yes" or "no" instead of a "perhaps"
+ */
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
+
+ /*
+ * only if GRH is TRUE we might consider SOURCE_GID_IDX
+ * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
+ */
+ if (attr->ah_attr.ah_flags == IB_AH_GRH) {
+ mqpcb->send_grh_flag = 1;
+
+ mqpcb->source_gid_idx = attr->ah_attr.grh.sgid_index;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX, 1);
+
+ for (cnt = 0; cnt < 16; cnt++)
+ mqpcb->dest_gid.byte[cnt] =
+ attr->ah_attr.grh.dgid.raw[cnt];
+
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_GID, 1);
+ mqpcb->flow_label = attr->ah_attr.grh.flow_label;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL, 1);
+ mqpcb->hop_limit = attr->ah_attr.grh.hop_limit;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT, 1);
+ mqpcb->traffic_class = attr->ah_attr.grh.traffic_class;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS, 1);
+ }
+ }
+
+ if (attr_mask & IB_QP_PATH_MTU) {
+ /* store ld(MTU) */
+ my_qp->mtu_shift = attr->path_mtu + 7;
+ mqpcb->path_mtu = attr->path_mtu;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PATH_MTU, 1);
+ }
+ if (attr_mask & IB_QP_TIMEOUT) {
+ mqpcb->timeout = attr->timeout;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT, 1);
+ }
+ if (attr_mask & IB_QP_RETRY_CNT) {
+ mqpcb->retry_count = attr->retry_cnt;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT, 1);
+ }
+ if (attr_mask & IB_QP_RNR_RETRY) {
+ mqpcb->rnr_retry_count = attr->rnr_retry;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT, 1);
+ }
+ if (attr_mask & IB_QP_RQ_PSN) {
+ mqpcb->receive_psn = attr->rq_psn;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RECEIVE_PSN, 1);
+ }
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+ mqpcb->rdma_nr_atomic_resp_res = attr->max_dest_rd_atomic < 3 ?
+ attr->max_dest_rd_atomic : 2;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
+ }
+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ mqpcb->rdma_atomic_outst_dest_qp = attr->max_rd_atomic < 3 ?
+ attr->max_rd_atomic : 2;
+ update_mask |=
+ EHCA_BMASK_SET
+ (MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP, 1);
+ }
+ if (attr_mask & IB_QP_ALT_PATH) {
+ if (attr->alt_port_num < 1
+ || attr->alt_port_num > shca->num_ports) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid alt_port=%x. "
+ "ehca_qp=%p qp_num=%x num_ports=%x",
+ attr->alt_port_num, my_qp, ibqp->qp_num,
+ shca->num_ports);
+ goto modify_qp_exit2;
+ }
+ mqpcb->alt_phys_port = attr->alt_port_num;
+
+ if (attr->alt_pkey_index >= 16) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid alt_pkey_index=%x. "
+ "ehca_qp=%p qp_num=%x max_pkey_index=f",
+ attr->pkey_index, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+ mqpcb->alt_p_key_idx = attr->alt_pkey_index;
+
+ mqpcb->timeout_al = attr->alt_timeout;
+ mqpcb->dlid_al = attr->alt_ah_attr.dlid;
+ mqpcb->source_path_bits_al = attr->alt_ah_attr.src_path_bits;
+ mqpcb->service_level_al = attr->alt_ah_attr.sl;
+
+ if (ehca_calc_ipd(shca, mqpcb->alt_phys_port,
+ attr->alt_ah_attr.static_rate,
+ &mqpcb->max_static_rate_al)) {
+ ret = -EINVAL;
+ goto modify_qp_exit2;
+ }
+
+ /* OpenIB doesn't support alternate retry counts - copy them */
+ mqpcb->retry_count_al = mqpcb->retry_count;
+ mqpcb->rnr_retry_count_al = mqpcb->rnr_retry_count;
+
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_ALT_PHYS_PORT, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_ALT_P_KEY_IDX, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_DLID_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT_AL, 1);
+
+ /*
+ * Always supply the GRH flag, even if it's zero, to give the
+ * hypervisor a clear "yes" or "no" instead of a "perhaps"
+ */
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG_AL, 1);
+
+ /*
+ * only if GRH is TRUE we might consider SOURCE_GID_IDX
+ * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
+ */
+ if (attr->alt_ah_attr.ah_flags == IB_AH_GRH) {
+ mqpcb->send_grh_flag_al = 1;
+
+ for (cnt = 0; cnt < 16; cnt++)
+ mqpcb->dest_gid_al.byte[cnt] =
+ attr->alt_ah_attr.grh.dgid.raw[cnt];
+ mqpcb->source_gid_idx_al =
+ attr->alt_ah_attr.grh.sgid_index;
+ mqpcb->flow_label_al = attr->alt_ah_attr.grh.flow_label;
+ mqpcb->hop_limit_al = attr->alt_ah_attr.grh.hop_limit;
+ mqpcb->traffic_class_al =
+ attr->alt_ah_attr.grh.traffic_class;
+
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_DEST_GID_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT_AL, 1) |
+ EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS_AL, 1);
+ }
+ }
+
+ if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+ mqpcb->min_rnr_nak_timer_field = attr->min_rnr_timer;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD, 1);
+ }
+
+ if (attr_mask & IB_QP_SQ_PSN) {
+ mqpcb->send_psn = attr->sq_psn;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_PSN, 1);
+ }
+
+ if (attr_mask & IB_QP_DEST_QPN) {
+ mqpcb->dest_qp_nr = attr->dest_qp_num;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_QP_NR, 1);
+ }
+
+ if (attr_mask & IB_QP_PATH_MIG_STATE) {
+ if (attr->path_mig_state != IB_MIG_REARM
+ && attr->path_mig_state != IB_MIG_MIGRATED) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid mig_state=%x",
+ attr->path_mig_state);
+ goto modify_qp_exit2;
+ }
+ mqpcb->path_migration_state = attr->path_mig_state + 1;
+ if (attr->path_mig_state == IB_MIG_REARM)
+ my_qp->mig_armed = 1;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_PATH_MIGRATION_STATE, 1);
+ }
+
+ if (attr_mask & IB_QP_CAP) {
+ mqpcb->max_nr_outst_send_wr = attr->cap.max_send_wr+1;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_SEND_WR, 1);
+ mqpcb->max_nr_outst_recv_wr = attr->cap.max_recv_wr+1;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_RECV_WR, 1);
+ /* no support for max_send/recv_sge yet */
+ }
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(mqpcb, 4*70, "qp_num=%x", ibqp->qp_num);
+
+ h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(ibqp->device, "hipz_h_modify_qp() failed h_ret=%lli "
+ "ehca_qp=%p qp_num=%x", h_ret, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+
+ if ((my_qp->qp_type == IB_QPT_UD ||
+ my_qp->qp_type == IB_QPT_GSI ||
+ my_qp->qp_type == IB_QPT_SMI) &&
+ statetrans == IB_QPST_SQE2RTS) {
+ /* doorbell to reprocessing wqes */
+ iosync(); /* serialize GAL register access */
+ hipz_update_sqa(my_qp, bad_wqe_cnt-1);
+ ehca_gen_dbg("doorbell for %x wqes", bad_wqe_cnt);
+ }
+
+ if (statetrans == IB_QPST_RESET2INIT ||
+ statetrans == IB_QPST_INIT2INIT) {
+ mqpcb->qp_enable = 1;
+ mqpcb->qp_state = EHCA_QPS_INIT;
+ update_mask = 0;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
+
+ h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb,
+ my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(ibqp->device, "ENABLE in context of "
+ "RESET_2_INIT failed! Maybe you didn't get "
+ "a LID h_ret=%lli ehca_qp=%p qp_num=%x",
+ h_ret, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+ }
+ if ((qp_new_state == IB_QPS_ERR) && (qp_cur_state != IB_QPS_ERR)
+ && !is_user) {
+ ret = check_for_left_cqes(my_qp, shca);
+ if (ret)
+ goto modify_qp_exit2;
+ }
+
+ if (statetrans == IB_QPST_ANY2RESET) {
+ ipz_qeit_reset(&my_qp->ipz_rqueue);
+ ipz_qeit_reset(&my_qp->ipz_squeue);
+
+ if (qp_cur_state == IB_QPS_ERR && !is_user) {
+ del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
+
+ if (HAS_RQ(my_qp))
+ del_from_err_list(my_qp->recv_cq,
+ &my_qp->rq_err_node);
+ }
+ if (!is_user)
+ reset_queue_map(&my_qp->sq_map);
+
+ if (HAS_RQ(my_qp) && !is_user)
+ reset_queue_map(&my_qp->rq_map);
+ }
+
+ if (attr_mask & IB_QP_QKEY)
+ my_qp->qkey = attr->qkey;
+
+modify_qp_exit2:
+ if (squeue_locked) { /* this means: sqe -> rts */
+ spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
+ my_qp->sqerr_purgeflag = 1;
+ }
+
+modify_qp_exit1:
+ ehca_free_fw_ctrlblock(mqpcb);
+
+ return ret;
+}
+
+int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+
+ struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
+ ib_device);
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+
+ /* The if-block below caches qp_attr to be modified for GSI and SMI
+ * qps during the initialization by ib_mad. When the respective port
+ * is activated, ie we got an event PORT_ACTIVE, we'll replay the
+ * cached modify calls sequence, see ehca_recover_sqs() below.
+ * Why that is required:
+ * 1) If one port is connected, older code requires that port one
+ * to be connected and module option nr_ports=1 to be given by
+ * user, which is very inconvenient for end user.
+ * 2) Firmware accepts modify_qp() only if respective port has become
+ * active. Older code had a wait loop of 30sec create_qp()/
+ * define_aqp1(), which is not appropriate in practice. This
+ * code now removes that wait loop, see define_aqp1(), and always
+ * reports all ports to ib_mad resp. users. Only activated ports
+ * will then usable for the users.
+ */
+ if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) {
+ int port = my_qp->init_attr.port_num;
+ struct ehca_sport *sport = &shca->sport[port - 1];
+ unsigned long flags;
+ spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+ /* cache qp_attr only during init */
+ if (my_qp->mod_qp_parm) {
+ struct ehca_mod_qp_parm *p;
+ if (my_qp->mod_qp_parm_idx >= EHCA_MOD_QP_PARM_MAX) {
+ ehca_err(&shca->ib_device,
+ "mod_qp_parm overflow state=%x port=%x"
+ " type=%x", attr->qp_state,
+ my_qp->init_attr.port_num,
+ ibqp->qp_type);
+ spin_unlock_irqrestore(&sport->mod_sqp_lock,
+ flags);
+ return -EINVAL;
+ }
+ p = &my_qp->mod_qp_parm[my_qp->mod_qp_parm_idx];
+ p->mask = attr_mask;
+ p->attr = *attr;
+ my_qp->mod_qp_parm_idx++;
+ ehca_dbg(&shca->ib_device,
+ "Saved qp_attr for state=%x port=%x type=%x",
+ attr->qp_state, my_qp->init_attr.port_num,
+ ibqp->qp_type);
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+ goto out;
+ }
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+ }
+
+ ret = internal_modify_qp(ibqp, attr, attr_mask, 0);
+
+out:
+ if ((ret == 0) && (attr_mask & IB_QP_STATE))
+ my_qp->state = attr->qp_state;
+
+ return ret;
+}
+
+void ehca_recover_sqp(struct ib_qp *sqp)
+{
+ struct ehca_qp *my_sqp = container_of(sqp, struct ehca_qp, ib_qp);
+ int port = my_sqp->init_attr.port_num;
+ struct ib_qp_attr attr;
+ struct ehca_mod_qp_parm *qp_parm;
+ int i, qp_parm_idx, ret;
+ unsigned long flags, wr_cnt;
+
+ if (!my_sqp->mod_qp_parm)
+ return;
+ ehca_dbg(sqp->device, "SQP port=%x qp_num=%x", port, sqp->qp_num);
+
+ qp_parm = my_sqp->mod_qp_parm;
+ qp_parm_idx = my_sqp->mod_qp_parm_idx;
+ for (i = 0; i < qp_parm_idx; i++) {
+ attr = qp_parm[i].attr;
+ ret = internal_modify_qp(sqp, &attr, qp_parm[i].mask, 0);
+ if (ret) {
+ ehca_err(sqp->device, "Could not modify SQP port=%x "
+ "qp_num=%x ret=%x", port, sqp->qp_num, ret);
+ goto free_qp_parm;
+ }
+ ehca_dbg(sqp->device, "SQP port=%x qp_num=%x in state=%x",
+ port, sqp->qp_num, attr.qp_state);
+ }
+
+ /* re-trigger posted recv wrs */
+ wr_cnt = my_sqp->ipz_rqueue.current_q_offset /
+ my_sqp->ipz_rqueue.qe_size;
+ if (wr_cnt) {
+ spin_lock_irqsave(&my_sqp->spinlock_r, flags);
+ hipz_update_rqa(my_sqp, wr_cnt);
+ spin_unlock_irqrestore(&my_sqp->spinlock_r, flags);
+ ehca_dbg(sqp->device, "doorbell port=%x qp_num=%x wr_cnt=%lx",
+ port, sqp->qp_num, wr_cnt);
+ }
+
+free_qp_parm:
+ kfree(qp_parm);
+ /* this prevents subsequent calls to modify_qp() to cache qp_attr */
+ my_sqp->mod_qp_parm = NULL;
+}
+
+int ehca_query_qp(struct ib_qp *qp,
+ struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+ struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca = container_of(qp->device, struct ehca_shca,
+ ib_device);
+ struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+ struct hcp_modify_qp_control_block *qpcb;
+ int cnt, ret = 0;
+ u64 h_ret;
+
+ if (qp_attr_mask & QP_ATTR_QUERY_NOT_SUPPORTED) {
+ ehca_err(qp->device, "Invalid attribute mask "
+ "ehca_qp=%p qp_num=%x qp_attr_mask=%x ",
+ my_qp, qp->qp_num, qp_attr_mask);
+ return -EINVAL;
+ }
+
+ qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!qpcb) {
+ ehca_err(qp->device, "Out of memory for qpcb "
+ "ehca_qp=%p qp_num=%x", my_qp, qp->qp_num);
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_qp(adapter_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ qpcb, my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(qp->device, "hipz_h_query_qp() failed "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, qp->qp_num, h_ret);
+ goto query_qp_exit1;
+ }
+
+ qp_attr->cur_qp_state = ehca2ib_qp_state(qpcb->qp_state);
+ qp_attr->qp_state = qp_attr->cur_qp_state;
+
+ if (qp_attr->cur_qp_state == -EINVAL) {
+ ret = -EINVAL;
+ ehca_err(qp->device, "Got invalid ehca_qp_state=%x "
+ "ehca_qp=%p qp_num=%x",
+ qpcb->qp_state, my_qp, qp->qp_num);
+ goto query_qp_exit1;
+ }
+
+ if (qp_attr->qp_state == IB_QPS_SQD)
+ qp_attr->sq_draining = 1;
+
+ qp_attr->qkey = qpcb->qkey;
+ qp_attr->path_mtu = qpcb->path_mtu;
+ qp_attr->path_mig_state = qpcb->path_migration_state - 1;
+ qp_attr->rq_psn = qpcb->receive_psn;
+ qp_attr->sq_psn = qpcb->send_psn;
+ qp_attr->min_rnr_timer = qpcb->min_rnr_nak_timer_field;
+ qp_attr->cap.max_send_wr = qpcb->max_nr_outst_send_wr-1;
+ qp_attr->cap.max_recv_wr = qpcb->max_nr_outst_recv_wr-1;
+ /* UD_AV CIRCUMVENTION */
+ if (my_qp->qp_type == IB_QPT_UD) {
+ qp_attr->cap.max_send_sge =
+ qpcb->actual_nr_sges_in_sq_wqe - 2;
+ qp_attr->cap.max_recv_sge =
+ qpcb->actual_nr_sges_in_rq_wqe - 2;
+ } else {
+ qp_attr->cap.max_send_sge =
+ qpcb->actual_nr_sges_in_sq_wqe;
+ qp_attr->cap.max_recv_sge =
+ qpcb->actual_nr_sges_in_rq_wqe;
+ }
+
+ qp_attr->cap.max_inline_data = my_qp->sq_max_inline_data_size;
+ qp_attr->dest_qp_num = qpcb->dest_qp_nr;
+
+ qp_attr->pkey_index = qpcb->prim_p_key_idx;
+ qp_attr->port_num = qpcb->prim_phys_port;
+ qp_attr->timeout = qpcb->timeout;
+ qp_attr->retry_cnt = qpcb->retry_count;
+ qp_attr->rnr_retry = qpcb->rnr_retry_count;
+
+ qp_attr->alt_pkey_index = qpcb->alt_p_key_idx;
+ qp_attr->alt_port_num = qpcb->alt_phys_port;
+ qp_attr->alt_timeout = qpcb->timeout_al;
+
+ qp_attr->max_dest_rd_atomic = qpcb->rdma_nr_atomic_resp_res;
+ qp_attr->max_rd_atomic = qpcb->rdma_atomic_outst_dest_qp;
+
+ /* primary av */
+ qp_attr->ah_attr.sl = qpcb->service_level;
+
+ if (qpcb->send_grh_flag) {
+ qp_attr->ah_attr.ah_flags = IB_AH_GRH;
+ }
+
+ qp_attr->ah_attr.static_rate = qpcb->max_static_rate;
+ qp_attr->ah_attr.dlid = qpcb->dlid;
+ qp_attr->ah_attr.src_path_bits = qpcb->source_path_bits;
+ qp_attr->ah_attr.port_num = qp_attr->port_num;
+
+ /* primary GRH */
+ qp_attr->ah_attr.grh.traffic_class = qpcb->traffic_class;
+ qp_attr->ah_attr.grh.hop_limit = qpcb->hop_limit;
+ qp_attr->ah_attr.grh.sgid_index = qpcb->source_gid_idx;
+ qp_attr->ah_attr.grh.flow_label = qpcb->flow_label;
+
+ for (cnt = 0; cnt < 16; cnt++)
+ qp_attr->ah_attr.grh.dgid.raw[cnt] =
+ qpcb->dest_gid.byte[cnt];
+
+ /* alternate AV */
+ qp_attr->alt_ah_attr.sl = qpcb->service_level_al;
+ if (qpcb->send_grh_flag_al) {
+ qp_attr->alt_ah_attr.ah_flags = IB_AH_GRH;
+ }
+
+ qp_attr->alt_ah_attr.static_rate = qpcb->max_static_rate_al;
+ qp_attr->alt_ah_attr.dlid = qpcb->dlid_al;
+ qp_attr->alt_ah_attr.src_path_bits = qpcb->source_path_bits_al;
+
+ /* alternate GRH */
+ qp_attr->alt_ah_attr.grh.traffic_class = qpcb->traffic_class_al;
+ qp_attr->alt_ah_attr.grh.hop_limit = qpcb->hop_limit_al;
+ qp_attr->alt_ah_attr.grh.sgid_index = qpcb->source_gid_idx_al;
+ qp_attr->alt_ah_attr.grh.flow_label = qpcb->flow_label_al;
+
+ for (cnt = 0; cnt < 16; cnt++)
+ qp_attr->alt_ah_attr.grh.dgid.raw[cnt] =
+ qpcb->dest_gid_al.byte[cnt];
+
+ /* return init attributes given in ehca_create_qp */
+ if (qp_init_attr)
+ *qp_init_attr = my_qp->init_attr;
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(qpcb, 4*70, "qp_num=%x", qp->qp_num);
+
+query_qp_exit1:
+ ehca_free_fw_ctrlblock(qpcb);
+
+ return ret;
+}
+
+int ehca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
+{
+ struct ehca_qp *my_qp =
+ container_of(ibsrq, struct ehca_qp, ib_srq);
+ struct ehca_shca *shca =
+ container_of(ibsrq->pd->device, struct ehca_shca, ib_device);
+ struct hcp_modify_qp_control_block *mqpcb;
+ u64 update_mask;
+ u64 h_ret;
+ int ret = 0;
+
+ mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!mqpcb) {
+ ehca_err(ibsrq->device, "Could not get zeroed page for mqpcb "
+ "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
+ return -ENOMEM;
+ }
+
+ update_mask = 0;
+ if (attr_mask & IB_SRQ_LIMIT) {
+ attr_mask &= ~IB_SRQ_LIMIT;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_CURR_SRQ_LIMIT, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG, 1);
+ mqpcb->curr_srq_limit = attr->srq_limit;
+ mqpcb->qp_aff_asyn_ev_log_reg =
+ EHCA_BMASK_SET(QPX_AAELOG_RESET_SRQ_LIMIT, 1);
+ }
+
+ /* by now, all bits in attr_mask should have been cleared */
+ if (attr_mask) {
+ ehca_err(ibsrq->device, "invalid attribute mask bits set "
+ "attr_mask=%x", attr_mask);
+ ret = -EINVAL;
+ goto modify_srq_exit0;
+ }
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(mqpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
+
+ h_ret = hipz_h_modify_qp(shca->ipz_hca_handle, my_qp->ipz_qp_handle,
+ NULL, update_mask, mqpcb,
+ my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(ibsrq->device, "hipz_h_modify_qp() failed h_ret=%lli "
+ "ehca_qp=%p qp_num=%x",
+ h_ret, my_qp, my_qp->real_qp_num);
+ }
+
+modify_srq_exit0:
+ ehca_free_fw_ctrlblock(mqpcb);
+
+ return ret;
+}
+
+int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr)
+{
+ struct ehca_qp *my_qp = container_of(srq, struct ehca_qp, ib_srq);
+ struct ehca_shca *shca = container_of(srq->device, struct ehca_shca,
+ ib_device);
+ struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+ struct hcp_modify_qp_control_block *qpcb;
+ int ret = 0;
+ u64 h_ret;
+
+ qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!qpcb) {
+ ehca_err(srq->device, "Out of memory for qpcb "
+ "ehca_qp=%p qp_num=%x", my_qp, my_qp->real_qp_num);
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_qp(adapter_handle, my_qp->ipz_qp_handle,
+ NULL, qpcb, my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(srq->device, "hipz_h_query_qp() failed "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, h_ret);
+ goto query_srq_exit1;
+ }
+
+ srq_attr->max_wr = qpcb->max_nr_outst_recv_wr - 1;
+ srq_attr->max_sge = 3;
+ srq_attr->srq_limit = qpcb->curr_srq_limit;
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(qpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
+
+query_srq_exit1:
+ ehca_free_fw_ctrlblock(qpcb);
+
+ return ret;
+}
+
+static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
+ struct ib_uobject *uobject)
+{
+ struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device);
+ struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
+ ib_pd);
+ struct ehca_sport *sport = &shca->sport[my_qp->init_attr.port_num - 1];
+ u32 qp_num = my_qp->real_qp_num;
+ int ret;
+ u64 h_ret;
+ u8 port_num;
+ int is_user = 0;
+ enum ib_qp_type qp_type;
+ unsigned long flags;
+
+ if (uobject) {
+ is_user = 1;
+ if (my_qp->mm_count_galpa ||
+ my_qp->mm_count_rqueue || my_qp->mm_count_squeue) {
+ ehca_err(dev, "Resources still referenced in "
+ "user space qp_num=%x", qp_num);
+ return -EINVAL;
+ }
+ }
+
+ if (my_qp->send_cq) {
+ ret = ehca_cq_unassign_qp(my_qp->send_cq, qp_num);
+ if (ret) {
+ ehca_err(dev, "Couldn't unassign qp from "
+ "send_cq ret=%i qp_num=%x cq_num=%x", ret,
+ qp_num, my_qp->send_cq->cq_number);
+ return ret;
+ }
+ }
+
+ write_lock_irqsave(&ehca_qp_idr_lock, flags);
+ idr_remove(&ehca_qp_idr, my_qp->token);
+ write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+ /*
+ * SRQs will never get into an error list and do not have a recv_cq,
+ * so we need to skip them here.
+ */
+ if (HAS_RQ(my_qp) && !IS_SRQ(my_qp) && !is_user)
+ del_from_err_list(my_qp->recv_cq, &my_qp->rq_err_node);
+
+ if (HAS_SQ(my_qp) && !is_user)
+ del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
+
+ /* now wait until all pending events have completed */
+ wait_event(my_qp->wait_completion, !atomic_read(&my_qp->nr_events));
+
+ h_ret = hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(dev, "hipz_h_destroy_qp() failed h_ret=%lli "
+ "ehca_qp=%p qp_num=%x", h_ret, my_qp, qp_num);
+ return ehca2ib_return_code(h_ret);
+ }
+
+ port_num = my_qp->init_attr.port_num;
+ qp_type = my_qp->init_attr.qp_type;
+
+ if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
+ spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+ kfree(my_qp->mod_qp_parm);
+ my_qp->mod_qp_parm = NULL;
+ shca->sport[port_num - 1].ibqp_sqp[qp_type] = NULL;
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+ }
+
+ /* no support for IB_QPT_SMI yet */
+ if (qp_type == IB_QPT_GSI) {
+ struct ib_event event;
+ ehca_info(dev, "device %s: port %x is inactive.",
+ shca->ib_device.name, port_num);
+ event.device = &shca->ib_device;
+ event.event = IB_EVENT_PORT_ERR;
+ event.element.port_num = port_num;
+ shca->sport[port_num - 1].port_state = IB_PORT_DOWN;
+ ib_dispatch_event(&event);
+ }
+
+ if (HAS_RQ(my_qp)) {
+ ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
+ if (!is_user)
+ vfree(my_qp->rq_map.map);
+ }
+ if (HAS_SQ(my_qp)) {
+ ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
+ if (!is_user)
+ vfree(my_qp->sq_map.map);
+ }
+ kmem_cache_free(qp_cache, my_qp);
+ atomic_dec(&shca->num_qps);
+ return 0;
+}
+
+int ehca_destroy_qp(struct ib_qp *qp)
+{
+ return internal_destroy_qp(qp->device,
+ container_of(qp, struct ehca_qp, ib_qp),
+ qp->uobject);
+}
+
+int ehca_destroy_srq(struct ib_srq *srq)
+{
+ return internal_destroy_qp(srq->device,
+ container_of(srq, struct ehca_qp, ib_srq),
+ srq->uobject);
+}
+
+int ehca_init_qp_cache(void)
+{
+ qp_cache = kmem_cache_create("ehca_cache_qp",
+ sizeof(struct ehca_qp), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!qp_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_qp_cache(void)
+{
+ if (qp_cache)
+ kmem_cache_destroy(qp_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * post_send/recv, poll_cq, req_notify
+ *
+ * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+
+/* in RC traffic, insert an empty RDMA READ every this many packets */
+#define ACK_CIRC_THRESHOLD 2000000
+
+static u64 replace_wr_id(u64 wr_id, u16 idx)
+{
+ u64 ret;
+
+ ret = wr_id & ~QMAP_IDX_MASK;
+ ret |= idx & QMAP_IDX_MASK;
+
+ return ret;
+}
+
+static u16 get_app_wr_id(u64 wr_id)
+{
+ return wr_id & QMAP_IDX_MASK;
+}
+
+static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
+ struct ehca_wqe *wqe_p,
+ struct ib_recv_wr *recv_wr,
+ u32 rq_map_idx)
+{
+ u8 cnt_ds;
+ if (unlikely((recv_wr->num_sge < 0) ||
+ (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
+ ehca_gen_err("Invalid number of WQE SGE. "
+ "num_sqe=%x max_nr_of_sg=%x",
+ recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
+ return -EINVAL; /* invalid SG list length */
+ }
+
+ /* clear wqe header until sglist */
+ memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
+
+ wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
+ wqe_p->nr_of_data_seg = recv_wr->num_sge;
+
+ for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
+ wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
+ recv_wr->sg_list[cnt_ds].addr;
+ wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
+ recv_wr->sg_list[cnt_ds].lkey;
+ wqe_p->u.all_rcv.sg_list[cnt_ds].length =
+ recv_wr->sg_list[cnt_ds].length;
+ }
+
+ if (ehca_debug_level >= 3) {
+ ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
+ ipz_rqueue);
+ ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
+ }
+
+ return 0;
+}
+
+#if defined(DEBUG_GSI_SEND_WR)
+
+/* need ib_mad struct */
+#include <rdma/ib_mad.h>
+
+static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
+{
+ int idx;
+ int j;
+ while (send_wr) {
+ struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
+ struct ib_sge *sge = send_wr->sg_list;
+ ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
+ "send_flags=%x opcode=%x", idx, send_wr->wr_id,
+ send_wr->num_sge, send_wr->send_flags,
+ send_wr->opcode);
+ if (mad_hdr) {
+ ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
+ "mgmt_class=%x class_version=%x method=%x "
+ "status=%x class_specific=%x tid=%lx "
+ "attr_id=%x resv=%x attr_mod=%x",
+ idx, mad_hdr->base_version,
+ mad_hdr->mgmt_class,
+ mad_hdr->class_version, mad_hdr->method,
+ mad_hdr->status, mad_hdr->class_specific,
+ mad_hdr->tid, mad_hdr->attr_id,
+ mad_hdr->resv,
+ mad_hdr->attr_mod);
+ }
+ for (j = 0; j < send_wr->num_sge; j++) {
+ u8 *data = __va(sge->addr);
+ ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
+ "lkey=%x",
+ idx, j, data, sge->length, sge->lkey);
+ /* assume length is n*16 */
+ ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
+ idx, j);
+ sge++;
+ } /* eof for j */
+ idx++;
+ send_wr = send_wr->next;
+ } /* eof while send_wr */
+}
+
+#endif /* DEBUG_GSI_SEND_WR */
+
+static inline int ehca_write_swqe(struct ehca_qp *qp,
+ struct ehca_wqe *wqe_p,
+ const struct ib_send_wr *send_wr,
+ u32 sq_map_idx,
+ int hidden)
+{
+ u32 idx;
+ u64 dma_length;
+ struct ehca_av *my_av;
+ u32 remote_qkey = send_wr->wr.ud.remote_qkey;
+ struct ehca_qmap_entry *qmap_entry = &qp->sq_map.map[sq_map_idx];
+
+ if (unlikely((send_wr->num_sge < 0) ||
+ (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
+ ehca_gen_err("Invalid number of WQE SGE. "
+ "num_sqe=%x max_nr_of_sg=%x",
+ send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
+ return -EINVAL; /* invalid SG list length */
+ }
+
+ /* clear wqe header until sglist */
+ memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
+
+ wqe_p->work_request_id = replace_wr_id(send_wr->wr_id, sq_map_idx);
+
+ qmap_entry->app_wr_id = get_app_wr_id(send_wr->wr_id);
+ qmap_entry->reported = 0;
+ qmap_entry->cqe_req = 0;
+
+ switch (send_wr->opcode) {
+ case IB_WR_SEND:
+ case IB_WR_SEND_WITH_IMM:
+ wqe_p->optype = WQE_OPTYPE_SEND;
+ break;
+ case IB_WR_RDMA_WRITE:
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
+ break;
+ case IB_WR_RDMA_READ:
+ wqe_p->optype = WQE_OPTYPE_RDMAREAD;
+ break;
+ default:
+ ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
+ return -EINVAL; /* invalid opcode */
+ }
+
+ wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
+
+ wqe_p->wr_flag = 0;
+
+ if ((send_wr->send_flags & IB_SEND_SIGNALED ||
+ qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
+ && !hidden) {
+ wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
+ qmap_entry->cqe_req = 1;
+ }
+
+ if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
+ send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
+ /* this might not work as long as HW does not support it */
+ wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
+ wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
+ }
+
+ wqe_p->nr_of_data_seg = send_wr->num_sge;
+
+ switch (qp->qp_type) {
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ /* no break is intential here */
+ case IB_QPT_UD:
+ /* IB 1.2 spec C10-15 compliance */
+ if (send_wr->wr.ud.remote_qkey & 0x80000000)
+ remote_qkey = qp->qkey;
+
+ wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
+ wqe_p->local_ee_context_qkey = remote_qkey;
+ if (unlikely(!send_wr->wr.ud.ah)) {
+ ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
+ return -EINVAL;
+ }
+ if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
+ ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
+ return -EINVAL;
+ }
+ my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
+ wqe_p->u.ud_av.ud_av = my_av->av;
+
+ /*
+ * omitted check of IB_SEND_INLINE
+ * since HW does not support it
+ */
+ for (idx = 0; idx < send_wr->num_sge; idx++) {
+ wqe_p->u.ud_av.sg_list[idx].vaddr =
+ send_wr->sg_list[idx].addr;
+ wqe_p->u.ud_av.sg_list[idx].lkey =
+ send_wr->sg_list[idx].lkey;
+ wqe_p->u.ud_av.sg_list[idx].length =
+ send_wr->sg_list[idx].length;
+ } /* eof for idx */
+ if (qp->qp_type == IB_QPT_SMI ||
+ qp->qp_type == IB_QPT_GSI)
+ wqe_p->u.ud_av.ud_av.pmtu = 1;
+ if (qp->qp_type == IB_QPT_GSI) {
+ wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
+#ifdef DEBUG_GSI_SEND_WR
+ trace_send_wr_ud(send_wr);
+#endif /* DEBUG_GSI_SEND_WR */
+ }
+ break;
+
+ case IB_QPT_UC:
+ if (send_wr->send_flags & IB_SEND_FENCE)
+ wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
+ /* no break is intentional here */
+ case IB_QPT_RC:
+ /* TODO: atomic not implemented */
+ wqe_p->u.nud.remote_virtual_address =
+ send_wr->wr.rdma.remote_addr;
+ wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
+
+ /*
+ * omitted checking of IB_SEND_INLINE
+ * since HW does not support it
+ */
+ dma_length = 0;
+ for (idx = 0; idx < send_wr->num_sge; idx++) {
+ wqe_p->u.nud.sg_list[idx].vaddr =
+ send_wr->sg_list[idx].addr;
+ wqe_p->u.nud.sg_list[idx].lkey =
+ send_wr->sg_list[idx].lkey;
+ wqe_p->u.nud.sg_list[idx].length =
+ send_wr->sg_list[idx].length;
+ dma_length += send_wr->sg_list[idx].length;
+ } /* eof idx */
+ wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
+
+ /* unsolicited ack circumvention */
+ if (send_wr->opcode == IB_WR_RDMA_READ) {
+ /* on RDMA read, switch on and reset counters */
+ qp->message_count = qp->packet_count = 0;
+ qp->unsol_ack_circ = 1;
+ } else
+ /* else estimate #packets */
+ qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
+
+ break;
+
+ default:
+ ehca_gen_err("Invalid qptype=%x", qp->qp_type);
+ return -EINVAL;
+ }
+
+ if (ehca_debug_level >= 3) {
+ ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
+ ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
+ }
+ return 0;
+}
+
+/* map_ib_wc_status converts raw cqe_status to ib_wc_status */
+static inline void map_ib_wc_status(u32 cqe_status,
+ enum ib_wc_status *wc_status)
+{
+ if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
+ switch (cqe_status & 0x3F) {
+ case 0x01:
+ case 0x21:
+ *wc_status = IB_WC_LOC_LEN_ERR;
+ break;
+ case 0x02:
+ case 0x22:
+ *wc_status = IB_WC_LOC_QP_OP_ERR;
+ break;
+ case 0x03:
+ case 0x23:
+ *wc_status = IB_WC_LOC_EEC_OP_ERR;
+ break;
+ case 0x04:
+ case 0x24:
+ *wc_status = IB_WC_LOC_PROT_ERR;
+ break;
+ case 0x05:
+ case 0x25:
+ *wc_status = IB_WC_WR_FLUSH_ERR;
+ break;
+ case 0x06:
+ *wc_status = IB_WC_MW_BIND_ERR;
+ break;
+ case 0x07: /* remote error - look into bits 20:24 */
+ switch ((cqe_status
+ & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
+ case 0x0:
+ /*
+ * PSN Sequence Error!
+ * couldn't find a matching status!
+ */
+ *wc_status = IB_WC_GENERAL_ERR;
+ break;
+ case 0x1:
+ *wc_status = IB_WC_REM_INV_REQ_ERR;
+ break;
+ case 0x2:
+ *wc_status = IB_WC_REM_ACCESS_ERR;
+ break;
+ case 0x3:
+ *wc_status = IB_WC_REM_OP_ERR;
+ break;
+ case 0x4:
+ *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
+ break;
+ }
+ break;
+ case 0x08:
+ *wc_status = IB_WC_RETRY_EXC_ERR;
+ break;
+ case 0x09:
+ *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
+ break;
+ case 0x0A:
+ case 0x2D:
+ *wc_status = IB_WC_REM_ABORT_ERR;
+ break;
+ case 0x0B:
+ case 0x2E:
+ *wc_status = IB_WC_INV_EECN_ERR;
+ break;
+ case 0x0C:
+ case 0x2F:
+ *wc_status = IB_WC_INV_EEC_STATE_ERR;
+ break;
+ case 0x0D:
+ *wc_status = IB_WC_BAD_RESP_ERR;
+ break;
+ case 0x10:
+ /* WQE purged */
+ *wc_status = IB_WC_WR_FLUSH_ERR;
+ break;
+ default:
+ *wc_status = IB_WC_FATAL_ERR;
+
+ }
+ } else
+ *wc_status = IB_WC_SUCCESS;
+}
+
+static inline int post_one_send(struct ehca_qp *my_qp,
+ struct ib_send_wr *cur_send_wr,
+ int hidden)
+{
+ struct ehca_wqe *wqe_p;
+ int ret;
+ u32 sq_map_idx;
+ u64 start_offset = my_qp->ipz_squeue.current_q_offset;
+
+ /* get pointer next to free WQE */
+ wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
+ if (unlikely(!wqe_p)) {
+ /* too many posted work requests: queue overflow */
+ ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
+ "qp_num=%x", my_qp->ib_qp.qp_num);
+ return -ENOMEM;
+ }
+
+ /*
+ * Get the index of the WQE in the send queue. The same index is used
+ * for writing into the sq_map.
+ */
+ sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
+
+ /* write a SEND WQE into the QUEUE */
+ ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
+ /*
+ * if something failed,
+ * reset the free entry pointer to the start value
+ */
+ if (unlikely(ret)) {
+ my_qp->ipz_squeue.current_q_offset = start_offset;
+ ehca_err(my_qp->ib_qp.device, "Could not write WQE "
+ "qp_num=%x", my_qp->ib_qp.qp_num);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int ehca_post_send(struct ib_qp *qp,
+ struct ib_send_wr *send_wr,
+ struct ib_send_wr **bad_send_wr)
+{
+ struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+ int wqe_cnt = 0;
+ int ret = 0;
+ unsigned long flags;
+
+ /* Reject WR if QP is in RESET, INIT or RTR state */
+ if (unlikely(my_qp->state < IB_QPS_RTS)) {
+ ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
+ my_qp->state, qp->qp_num);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* LOCK the QUEUE */
+ spin_lock_irqsave(&my_qp->spinlock_s, flags);
+
+ /* Send an empty extra RDMA read if:
+ * 1) there has been an RDMA read on this connection before
+ * 2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
+ * 3) we can be sure that any previous extra RDMA read has been
+ * processed so we don't overflow the SQ
+ */
+ if (unlikely(my_qp->unsol_ack_circ &&
+ my_qp->packet_count > ACK_CIRC_THRESHOLD &&
+ my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
+ /* insert an empty RDMA READ to fix up the remote QP state */
+ struct ib_send_wr circ_wr;
+ memset(&circ_wr, 0, sizeof(circ_wr));
+ circ_wr.opcode = IB_WR_RDMA_READ;
+ post_one_send(my_qp, &circ_wr, 1); /* ignore retcode */
+ wqe_cnt++;
+ ehca_dbg(qp->device, "posted circ wr qp_num=%x", qp->qp_num);
+ my_qp->message_count = my_qp->packet_count = 0;
+ }
+
+ /* loop processes list of send reqs */
+ while (send_wr) {
+ ret = post_one_send(my_qp, send_wr, 0);
+ if (unlikely(ret)) {
+ goto post_send_exit0;
+ }
+ wqe_cnt++;
+ send_wr = send_wr->next;
+ }
+
+post_send_exit0:
+ iosync(); /* serialize GAL register access */
+ hipz_update_sqa(my_qp, wqe_cnt);
+ if (unlikely(ret || ehca_debug_level >= 2))
+ ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
+ my_qp, qp->qp_num, wqe_cnt, ret);
+ my_qp->message_count += wqe_cnt;
+ spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
+
+out:
+ if (ret)
+ *bad_send_wr = send_wr;
+ return ret;
+}
+
+static int internal_post_recv(struct ehca_qp *my_qp,
+ struct ib_device *dev,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr)
+{
+ struct ehca_wqe *wqe_p;
+ int wqe_cnt = 0;
+ int ret = 0;
+ u32 rq_map_idx;
+ unsigned long flags;
+ struct ehca_qmap_entry *qmap_entry;
+
+ if (unlikely(!HAS_RQ(my_qp))) {
+ ehca_err(dev, "QP has no RQ ehca_qp=%p qp_num=%x ext_type=%d",
+ my_qp, my_qp->real_qp_num, my_qp->ext_type);
+ ret = -ENODEV;
+ goto out;
+ }
+
+ /* LOCK the QUEUE */
+ spin_lock_irqsave(&my_qp->spinlock_r, flags);
+
+ /* loop processes list of recv reqs */
+ while (recv_wr) {
+ u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
+ /* get pointer next to free WQE */
+ wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
+ if (unlikely(!wqe_p)) {
+ /* too many posted work requests: queue overflow */
+ ret = -ENOMEM;
+ ehca_err(dev, "Too many posted WQEs "
+ "qp_num=%x", my_qp->real_qp_num);
+ goto post_recv_exit0;
+ }
+ /*
+ * Get the index of the WQE in the recv queue. The same index
+ * is used for writing into the rq_map.
+ */
+ rq_map_idx = start_offset / my_qp->ipz_rqueue.qe_size;
+
+ /* write a RECV WQE into the QUEUE */
+ ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, recv_wr,
+ rq_map_idx);
+ /*
+ * if something failed,
+ * reset the free entry pointer to the start value
+ */
+ if (unlikely(ret)) {
+ my_qp->ipz_rqueue.current_q_offset = start_offset;
+ ret = -EINVAL;
+ ehca_err(dev, "Could not write WQE "
+ "qp_num=%x", my_qp->real_qp_num);
+ goto post_recv_exit0;
+ }
+
+ qmap_entry = &my_qp->rq_map.map[rq_map_idx];
+ qmap_entry->app_wr_id = get_app_wr_id(recv_wr->wr_id);
+ qmap_entry->reported = 0;
+ qmap_entry->cqe_req = 1;
+
+ wqe_cnt++;
+ recv_wr = recv_wr->next;
+ } /* eof for recv_wr */
+
+post_recv_exit0:
+ iosync(); /* serialize GAL register access */
+ hipz_update_rqa(my_qp, wqe_cnt);
+ if (unlikely(ret || ehca_debug_level >= 2))
+ ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
+ my_qp, my_qp->real_qp_num, wqe_cnt, ret);
+ spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
+
+out:
+ if (ret)
+ *bad_recv_wr = recv_wr;
+
+ return ret;
+}
+
+int ehca_post_recv(struct ib_qp *qp,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr)
+{
+ struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+
+ /* Reject WR if QP is in RESET state */
+ if (unlikely(my_qp->state == IB_QPS_RESET)) {
+ ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
+ my_qp->state, qp->qp_num);
+ *bad_recv_wr = recv_wr;
+ return -EINVAL;
+ }
+
+ return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
+}
+
+int ehca_post_srq_recv(struct ib_srq *srq,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr)
+{
+ return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
+ srq->device, recv_wr, bad_recv_wr);
+}
+
+/*
+ * ib_wc_opcode table converts ehca wc opcode to ib
+ * Since we use zero to indicate invalid opcode, the actual ib opcode must
+ * be decremented!!!
+ */
+static const u8 ib_wc_opcode[255] = {
+ [0x01] = IB_WC_RECV+1,
+ [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
+ [0x04] = IB_WC_BIND_MW+1,
+ [0x08] = IB_WC_FETCH_ADD+1,
+ [0x10] = IB_WC_COMP_SWAP+1,
+ [0x20] = IB_WC_RDMA_WRITE+1,
+ [0x40] = IB_WC_RDMA_READ+1,
+ [0x80] = IB_WC_SEND+1
+};
+
+/* internal function to poll one entry of cq */
+static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
+{
+ int ret = 0, qmap_tail_idx;
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ struct ehca_cqe *cqe;
+ struct ehca_qp *my_qp;
+ struct ehca_qmap_entry *qmap_entry;
+ struct ehca_queue_map *qmap;
+ int cqe_count = 0, is_error;
+
+repoll:
+ cqe = (struct ehca_cqe *)
+ ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
+ if (!cqe) {
+ ret = -EAGAIN;
+ if (ehca_debug_level >= 3)
+ ehca_dbg(cq->device, "Completion queue is empty "
+ "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
+ goto poll_cq_one_exit0;
+ }
+
+ /* prevents loads being reordered across this point */
+ rmb();
+
+ cqe_count++;
+ if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
+ struct ehca_qp *qp;
+ int purgeflag;
+ unsigned long flags;
+
+ qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
+ if (!qp) {
+ ehca_err(cq->device, "cq_num=%x qp_num=%x "
+ "could not find qp -> ignore cqe",
+ my_cq->cq_number, cqe->local_qp_number);
+ ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
+ my_cq->cq_number, cqe->local_qp_number);
+ /* ignore this purged cqe */
+ goto repoll;
+ }
+ spin_lock_irqsave(&qp->spinlock_s, flags);
+ purgeflag = qp->sqerr_purgeflag;
+ spin_unlock_irqrestore(&qp->spinlock_s, flags);
+
+ if (purgeflag) {
+ ehca_dbg(cq->device,
+ "Got CQE with purged bit qp_num=%x src_qp=%x",
+ cqe->local_qp_number, cqe->remote_qp_number);
+ if (ehca_debug_level >= 2)
+ ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
+ cqe->local_qp_number,
+ cqe->remote_qp_number);
+ /*
+ * ignore this to avoid double cqes of bad wqe
+ * that caused sqe and turn off purge flag
+ */
+ qp->sqerr_purgeflag = 0;
+ goto repoll;
+ }
+ }
+
+ is_error = cqe->status & WC_STATUS_ERROR_BIT;
+
+ /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
+ if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
+ ehca_dbg(cq->device,
+ "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
+ is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
+ ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
+ my_cq, my_cq->cq_number);
+ ehca_dbg(cq->device,
+ "ehca_cq=%p cq_num=%x -------------------------",
+ my_cq, my_cq->cq_number);
+ }
+
+ read_lock(&ehca_qp_idr_lock);
+ my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
+ read_unlock(&ehca_qp_idr_lock);
+ if (!my_qp)
+ goto repoll;
+ wc->qp = &my_qp->ib_qp;
+
+ qmap_tail_idx = get_app_wr_id(cqe->work_request_id);
+ if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT))
+ /* We got a send completion. */
+ qmap = &my_qp->sq_map;
+ else
+ /* We got a receive completion. */
+ qmap = &my_qp->rq_map;
+
+ /* advance the tail pointer */
+ qmap->tail = qmap_tail_idx;
+
+ if (is_error) {
+ /*
+ * set left_to_poll to 0 because in error state, we will not
+ * get any additional CQEs
+ */
+ my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
+ my_qp->sq_map.entries);
+ my_qp->sq_map.left_to_poll = 0;
+ ehca_add_to_err_list(my_qp, 1);
+
+ my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
+ my_qp->rq_map.entries);
+ my_qp->rq_map.left_to_poll = 0;
+ if (HAS_RQ(my_qp))
+ ehca_add_to_err_list(my_qp, 0);
+ }
+
+ qmap_entry = &qmap->map[qmap_tail_idx];
+ if (qmap_entry->reported) {
+ ehca_warn(cq->device, "Double cqe on qp_num=%#x",
+ my_qp->real_qp_num);
+ /* found a double cqe, discard it and read next one */
+ goto repoll;
+ }
+
+ wc->wr_id = replace_wr_id(cqe->work_request_id, qmap_entry->app_wr_id);
+ qmap_entry->reported = 1;
+
+ /* if left_to_poll is decremented to 0, add the QP to the error list */
+ if (qmap->left_to_poll > 0) {
+ qmap->left_to_poll--;
+ if ((my_qp->sq_map.left_to_poll == 0) &&
+ (my_qp->rq_map.left_to_poll == 0)) {
+ ehca_add_to_err_list(my_qp, 1);
+ if (HAS_RQ(my_qp))
+ ehca_add_to_err_list(my_qp, 0);
+ }
+ }
+
+ /* eval ib_wc_opcode */
+ wc->opcode = ib_wc_opcode[cqe->optype]-1;
+ if (unlikely(wc->opcode == -1)) {
+ ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
+ "ehca_cq=%p cq_num=%x",
+ cqe->optype, cqe->status, my_cq, my_cq->cq_number);
+ /* dump cqe for other infos */
+ ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
+ my_cq, my_cq->cq_number);
+ /* update also queue adder to throw away this entry!!! */
+ goto repoll;
+ }
+
+ /* eval ib_wc_status */
+ if (unlikely(is_error)) {
+ /* complete with errors */
+ map_ib_wc_status(cqe->status, &wc->status);
+ wc->vendor_err = wc->status;
+ } else
+ wc->status = IB_WC_SUCCESS;
+
+ wc->byte_len = cqe->nr_bytes_transferred;
+ wc->pkey_index = cqe->pkey_index;
+ wc->slid = cqe->rlid;
+ wc->dlid_path_bits = cqe->dlid;
+ wc->src_qp = cqe->remote_qp_number;
+ /*
+ * HW has "Immed data present" and "GRH present" in bits 6 and 5.
+ * SW defines those in bits 1 and 0, so we can just shift and mask.
+ */
+ wc->wc_flags = (cqe->w_completion_flags >> 5) & 3;
+ wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
+ wc->sl = cqe->service_level;
+
+poll_cq_one_exit0:
+ if (cqe_count > 0)
+ hipz_update_feca(my_cq, cqe_count);
+
+ return ret;
+}
+
+static int generate_flush_cqes(struct ehca_qp *my_qp, struct ib_cq *cq,
+ struct ib_wc *wc, int num_entries,
+ struct ipz_queue *ipz_queue, int on_sq)
+{
+ int nr = 0;
+ struct ehca_wqe *wqe;
+ u64 offset;
+ struct ehca_queue_map *qmap;
+ struct ehca_qmap_entry *qmap_entry;
+
+ if (on_sq)
+ qmap = &my_qp->sq_map;
+ else
+ qmap = &my_qp->rq_map;
+
+ qmap_entry = &qmap->map[qmap->next_wqe_idx];
+
+ while ((nr < num_entries) && (qmap_entry->reported == 0)) {
+ /* generate flush CQE */
+
+ memset(wc, 0, sizeof(*wc));
+
+ offset = qmap->next_wqe_idx * ipz_queue->qe_size;
+ wqe = (struct ehca_wqe *)ipz_qeit_calc(ipz_queue, offset);
+ if (!wqe) {
+ ehca_err(cq->device, "Invalid wqe offset=%#llx on "
+ "qp_num=%#x", offset, my_qp->real_qp_num);
+ return nr;
+ }
+
+ wc->wr_id = replace_wr_id(wqe->work_request_id,
+ qmap_entry->app_wr_id);
+
+ if (on_sq) {
+ switch (wqe->optype) {
+ case WQE_OPTYPE_SEND:
+ wc->opcode = IB_WC_SEND;
+ break;
+ case WQE_OPTYPE_RDMAWRITE:
+ wc->opcode = IB_WC_RDMA_WRITE;
+ break;
+ case WQE_OPTYPE_RDMAREAD:
+ wc->opcode = IB_WC_RDMA_READ;
+ break;
+ default:
+ ehca_err(cq->device, "Invalid optype=%x",
+ wqe->optype);
+ return nr;
+ }
+ } else
+ wc->opcode = IB_WC_RECV;
+
+ if (wqe->wr_flag & WQE_WRFLAG_IMM_DATA_PRESENT) {
+ wc->ex.imm_data = wqe->immediate_data;
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ }
+
+ wc->status = IB_WC_WR_FLUSH_ERR;
+
+ wc->qp = &my_qp->ib_qp;
+
+ /* mark as reported and advance next_wqe pointer */
+ qmap_entry->reported = 1;
+ qmap->next_wqe_idx = next_index(qmap->next_wqe_idx,
+ qmap->entries);
+ qmap_entry = &qmap->map[qmap->next_wqe_idx];
+
+ wc++; nr++;
+ }
+
+ return nr;
+
+}
+
+int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
+{
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ int nr;
+ struct ehca_qp *err_qp;
+ struct ib_wc *current_wc = wc;
+ int ret = 0;
+ unsigned long flags;
+ int entries_left = num_entries;
+
+ if (num_entries < 1) {
+ ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
+ "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
+ ret = -EINVAL;
+ goto poll_cq_exit0;
+ }
+
+ spin_lock_irqsave(&my_cq->spinlock, flags);
+
+ /* generate flush cqes for send queues */
+ list_for_each_entry(err_qp, &my_cq->sqp_err_list, sq_err_node) {
+ nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
+ &err_qp->ipz_squeue, 1);
+ entries_left -= nr;
+ current_wc += nr;
+
+ if (entries_left == 0)
+ break;
+ }
+
+ /* generate flush cqes for receive queues */
+ list_for_each_entry(err_qp, &my_cq->rqp_err_list, rq_err_node) {
+ nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
+ &err_qp->ipz_rqueue, 0);
+ entries_left -= nr;
+ current_wc += nr;
+
+ if (entries_left == 0)
+ break;
+ }
+
+ for (nr = 0; nr < entries_left; nr++) {
+ ret = ehca_poll_cq_one(cq, current_wc);
+ if (ret)
+ break;
+ current_wc++;
+ } /* eof for nr */
+ entries_left -= nr;
+
+ spin_unlock_irqrestore(&my_cq->spinlock, flags);
+ if (ret == -EAGAIN || !ret)
+ ret = num_entries - entries_left;
+
+poll_cq_exit0:
+ return ret;
+}
+
+int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
+{
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ int ret = 0;
+
+ switch (notify_flags & IB_CQ_SOLICITED_MASK) {
+ case IB_CQ_SOLICITED:
+ hipz_set_cqx_n0(my_cq, 1);
+ break;
+ case IB_CQ_NEXT_COMP:
+ hipz_set_cqx_n1(my_cq, 1);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
+ unsigned long spl_flags;
+ spin_lock_irqsave(&my_cq->spinlock, spl_flags);
+ ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
+ spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
+ }
+
+ return ret;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * SQP functions
+ *
+ * Authors: Khadija Souissi <souissi@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rdma/ib_mad.h>
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+#define IB_MAD_STATUS_REDIRECT cpu_to_be16(0x0002)
+#define IB_MAD_STATUS_UNSUP_VERSION cpu_to_be16(0x0004)
+#define IB_MAD_STATUS_UNSUP_METHOD cpu_to_be16(0x0008)
+
+#define IB_PMA_CLASS_PORT_INFO cpu_to_be16(0x0001)
+
+/**
+ * ehca_define_sqp - Defines special queue pair 1 (GSI QP). When special queue
+ * pair is created successfully, the corresponding port gets active.
+ *
+ * Define Special Queue pair 0 (SMI QP) is still not supported.
+ *
+ * @qp_init_attr: Queue pair init attributes with port and queue pair type
+ */
+
+u64 ehca_define_sqp(struct ehca_shca *shca,
+ struct ehca_qp *ehca_qp,
+ struct ib_qp_init_attr *qp_init_attr)
+{
+ u32 pma_qp_nr, bma_qp_nr;
+ u64 ret;
+ u8 port = qp_init_attr->port_num;
+ int counter;
+
+ shca->sport[port - 1].port_state = IB_PORT_DOWN;
+
+ switch (qp_init_attr->qp_type) {
+ case IB_QPT_SMI:
+ /* function not supported yet */
+ break;
+ case IB_QPT_GSI:
+ ret = hipz_h_define_aqp1(shca->ipz_hca_handle,
+ ehca_qp->ipz_qp_handle,
+ ehca_qp->galpas.kernel,
+ (u32) qp_init_attr->port_num,
+ &pma_qp_nr, &bma_qp_nr);
+
+ if (ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device,
+ "Can't define AQP1 for port %x. h_ret=%lli",
+ port, ret);
+ return ret;
+ }
+ shca->sport[port - 1].pma_qp_nr = pma_qp_nr;
+ ehca_dbg(&shca->ib_device, "port=%x pma_qp_nr=%x",
+ port, pma_qp_nr);
+ break;
+ default:
+ ehca_err(&shca->ib_device, "invalid qp_type=%x",
+ qp_init_attr->qp_type);
+ return H_PARAMETER;
+ }
+
+ if (ehca_nr_ports < 0) /* autodetect mode */
+ return H_SUCCESS;
+
+ for (counter = 0;
+ shca->sport[port - 1].port_state != IB_PORT_ACTIVE &&
+ counter < ehca_port_act_time;
+ counter++) {
+ ehca_dbg(&shca->ib_device, "... wait until port %x is active",
+ port);
+ msleep_interruptible(1000);
+ }
+
+ if (counter == ehca_port_act_time) {
+ ehca_err(&shca->ib_device, "Port %x is not active.", port);
+ return H_HARDWARE;
+ }
+
+ return H_SUCCESS;
+}
+
+struct ib_perf {
+ struct ib_mad_hdr mad_hdr;
+ u8 reserved[40];
+ u8 data[192];
+} __attribute__ ((packed));
+
+/* TC/SL/FL packed into 32 bits, as in ClassPortInfo */
+struct tcslfl {
+ u32 tc:8;
+ u32 sl:4;
+ u32 fl:20;
+} __attribute__ ((packed));
+
+/* IP Version/TC/FL packed into 32 bits, as in GRH */
+struct vertcfl {
+ u32 ver:4;
+ u32 tc:8;
+ u32 fl:20;
+} __attribute__ ((packed));
+
+static int ehca_process_perf(struct ib_device *ibdev, u8 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad *in_mad, struct ib_mad *out_mad)
+{
+ const struct ib_perf *in_perf = (const struct ib_perf *)in_mad;
+ struct ib_perf *out_perf = (struct ib_perf *)out_mad;
+ struct ib_class_port_info *poi =
+ (struct ib_class_port_info *)out_perf->data;
+ struct tcslfl *tcslfl =
+ (struct tcslfl *)&poi->redirect_tcslfl;
+ struct ehca_shca *shca =
+ container_of(ibdev, struct ehca_shca, ib_device);
+ struct ehca_sport *sport = &shca->sport[port_num - 1];
+
+ ehca_dbg(ibdev, "method=%x", in_perf->mad_hdr.method);
+
+ *out_mad = *in_mad;
+
+ if (in_perf->mad_hdr.class_version != 1) {
+ ehca_warn(ibdev, "Unsupported class_version=%x",
+ in_perf->mad_hdr.class_version);
+ out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_VERSION;
+ goto perf_reply;
+ }
+
+ switch (in_perf->mad_hdr.method) {
+ case IB_MGMT_METHOD_GET:
+ case IB_MGMT_METHOD_SET:
+ /* set class port info for redirection */
+ out_perf->mad_hdr.attr_id = IB_PMA_CLASS_PORT_INFO;
+ out_perf->mad_hdr.status = IB_MAD_STATUS_REDIRECT;
+ memset(poi, 0, sizeof(*poi));
+ poi->base_version = 1;
+ poi->class_version = 1;
+ poi->resp_time_value = 18;
+
+ /* copy local routing information from WC where applicable */
+ tcslfl->sl = in_wc->sl;
+ poi->redirect_lid =
+ sport->saved_attr.lid | in_wc->dlid_path_bits;
+ poi->redirect_qp = sport->pma_qp_nr;
+ poi->redirect_qkey = IB_QP1_QKEY;
+
+ ehca_query_pkey(ibdev, port_num, in_wc->pkey_index,
+ &poi->redirect_pkey);
+
+ /* if request was globally routed, copy route info */
+ if (in_grh) {
+ const struct vertcfl *vertcfl =
+ (const struct vertcfl *)&in_grh->version_tclass_flow;
+ memcpy(poi->redirect_gid, in_grh->dgid.raw,
+ sizeof(poi->redirect_gid));
+ tcslfl->tc = vertcfl->tc;
+ tcslfl->fl = vertcfl->fl;
+ } else
+ /* else only fill in default GID */
+ ehca_query_gid(ibdev, port_num, 0,
+ (union ib_gid *)&poi->redirect_gid);
+
+ ehca_dbg(ibdev, "ehca_pma_lid=%x ehca_pma_qp=%x",
+ sport->saved_attr.lid, sport->pma_qp_nr);
+ break;
+
+ case IB_MGMT_METHOD_GET_RESP:
+ return IB_MAD_RESULT_FAILURE;
+
+ default:
+ out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_METHOD;
+ break;
+ }
+
+perf_reply:
+ out_perf->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
+
+ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad_hdr *in, size_t in_mad_size,
+ struct ib_mad_hdr *out, size_t *out_mad_size,
+ u16 *out_mad_pkey_index)
+{
+ int ret;
+ const struct ib_mad *in_mad = (const struct ib_mad *)in;
+ struct ib_mad *out_mad = (struct ib_mad *)out;
+
+ if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||
+ *out_mad_size != sizeof(*out_mad)))
+ return IB_MAD_RESULT_FAILURE;
+
+ if (!port_num || port_num > ibdev->phys_port_cnt || !in_wc)
+ return IB_MAD_RESULT_FAILURE;
+
+ /* accept only pma request */
+ if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
+ return IB_MAD_RESULT_SUCCESS;
+
+ ehca_dbg(ibdev, "port_num=%x src_qp=%x", port_num, in_wc->src_qp);
+ ret = ehca_process_perf(ibdev, port_num, in_wc, in_grh,
+ in_mad, out_mad);
+
+ return ret;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * auxiliary functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Khadija Souissi <souissik@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef EHCA_TOOLS_H
+#define EHCA_TOOLS_H
+
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/idr.h>
+#include <linux/kthread.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/vmalloc.h>
+#include <linux/notifier.h>
+#include <linux/cpu.h>
+#include <linux/device.h>
+
+#include <linux/atomic.h>
+#include <asm/ibmebus.h>
+#include <asm/io.h>
+#include <asm/pgtable.h>
+#include <asm/hvcall.h>
+
+extern int ehca_debug_level;
+
+#define ehca_dbg(ib_dev, format, arg...) \
+ do { \
+ if (unlikely(ehca_debug_level)) \
+ dev_printk(KERN_DEBUG, (ib_dev)->dma_device, \
+ "PU%04x EHCA_DBG:%s " format "\n", \
+ raw_smp_processor_id(), __func__, \
+ ## arg); \
+ } while (0)
+
+#define ehca_info(ib_dev, format, arg...) \
+ dev_info((ib_dev)->dma_device, "PU%04x EHCA_INFO:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+#define ehca_warn(ib_dev, format, arg...) \
+ dev_warn((ib_dev)->dma_device, "PU%04x EHCA_WARN:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+#define ehca_err(ib_dev, format, arg...) \
+ dev_err((ib_dev)->dma_device, "PU%04x EHCA_ERR:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+/* use this one only if no ib_dev available */
+#define ehca_gen_dbg(format, arg...) \
+ do { \
+ if (unlikely(ehca_debug_level)) \
+ printk(KERN_DEBUG "PU%04x EHCA_DBG:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg); \
+ } while (0)
+
+#define ehca_gen_warn(format, arg...) \
+ printk(KERN_INFO "PU%04x EHCA_WARN:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+#define ehca_gen_err(format, arg...) \
+ printk(KERN_ERR "PU%04x EHCA_ERR:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+/**
+ * ehca_dmp - printk a memory block, whose length is n*8 bytes.
+ * Each line has the following layout:
+ * <format string> adr=X ofs=Y <8 bytes hex> <8 bytes hex>
+ */
+#define ehca_dmp(adr, len, format, args...) \
+ do { \
+ unsigned int x; \
+ unsigned int l = (unsigned int)(len); \
+ unsigned char *deb = (unsigned char *)(adr); \
+ for (x = 0; x < l; x += 16) { \
+ printk(KERN_INFO "EHCA_DMP:%s " format \
+ " adr=%p ofs=%04x %016llx %016llx\n", \
+ __func__, ##args, deb, x, \
+ *((u64 *)&deb[0]), *((u64 *)&deb[8])); \
+ deb += 16; \
+ } \
+ } while (0)
+
+/* define a bitmask, little endian version */
+#define EHCA_BMASK(pos, length) (((pos) << 16) + (length))
+
+/* define a bitmask, the ibm way... */
+#define EHCA_BMASK_IBM(from, to) (((63 - to) << 16) + ((to) - (from) + 1))
+
+/* internal function, don't use */
+#define EHCA_BMASK_SHIFTPOS(mask) (((mask) >> 16) & 0xffff)
+
+/* internal function, don't use */
+#define EHCA_BMASK_MASK(mask) (~0ULL >> ((64 - (mask)) & 0xffff))
+
+/**
+ * EHCA_BMASK_SET - return value shifted and masked by mask
+ * variable|=EHCA_BMASK_SET(MY_MASK,0x4711) ORs the bits in variable
+ * variable&=~EHCA_BMASK_SET(MY_MASK,-1) clears the bits from the mask
+ * in variable
+ */
+#define EHCA_BMASK_SET(mask, value) \
+ ((EHCA_BMASK_MASK(mask) & ((u64)(value))) << EHCA_BMASK_SHIFTPOS(mask))
+
+/**
+ * EHCA_BMASK_GET - extract a parameter from value by mask
+ */
+#define EHCA_BMASK_GET(mask, value) \
+ (EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask)))
+
+/* Converts ehca to ib return code */
+int ehca2ib_return_code(u64 ehca_rc);
+
+#endif /* EHCA_TOOLS_H */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * userspace support verbs
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_classes.h"
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+
+struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
+ struct ib_udata *udata)
+{
+ struct ehca_ucontext *my_context;
+
+ my_context = kzalloc(sizeof *my_context, GFP_KERNEL);
+ if (!my_context) {
+ ehca_err(device, "Out of memory device=%p", device);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ return &my_context->ib_ucontext;
+}
+
+int ehca_dealloc_ucontext(struct ib_ucontext *context)
+{
+ kfree(container_of(context, struct ehca_ucontext, ib_ucontext));
+ return 0;
+}
+
+static void ehca_mm_open(struct vm_area_struct *vma)
+{
+ u32 *count = (u32 *)vma->vm_private_data;
+ if (!count) {
+ ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
+ vma->vm_start, vma->vm_end);
+ return;
+ }
+ (*count)++;
+ if (!(*count))
+ ehca_gen_err("Use count overflow vm_start=%lx vm_end=%lx",
+ vma->vm_start, vma->vm_end);
+ ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
+ vma->vm_start, vma->vm_end, *count);
+}
+
+static void ehca_mm_close(struct vm_area_struct *vma)
+{
+ u32 *count = (u32 *)vma->vm_private_data;
+ if (!count) {
+ ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
+ vma->vm_start, vma->vm_end);
+ return;
+ }
+ (*count)--;
+ ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
+ vma->vm_start, vma->vm_end, *count);
+}
+
+static const struct vm_operations_struct vm_ops = {
+ .open = ehca_mm_open,
+ .close = ehca_mm_close,
+};
+
+static int ehca_mmap_fw(struct vm_area_struct *vma, struct h_galpas *galpas,
+ u32 *mm_count)
+{
+ int ret;
+ u64 vsize, physical;
+
+ vsize = vma->vm_end - vma->vm_start;
+ if (vsize < EHCA_PAGESIZE) {
+ ehca_gen_err("invalid vsize=%lx", vma->vm_end - vma->vm_start);
+ return -EINVAL;
+ }
+
+ physical = galpas->user.fw_handle;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ ehca_gen_dbg("vsize=%llx physical=%llx", vsize, physical);
+ /* VM_IO | VM_DONTEXPAND | VM_DONTDUMP are set by remap_pfn_range() */
+ ret = remap_4k_pfn(vma, vma->vm_start, physical >> EHCA_PAGESHIFT,
+ vma->vm_page_prot);
+ if (unlikely(ret)) {
+ ehca_gen_err("remap_pfn_range() failed ret=%i", ret);
+ return -ENOMEM;
+ }
+
+ vma->vm_private_data = mm_count;
+ (*mm_count)++;
+ vma->vm_ops = &vm_ops;
+
+ return 0;
+}
+
+static int ehca_mmap_queue(struct vm_area_struct *vma, struct ipz_queue *queue,
+ u32 *mm_count)
+{
+ int ret;
+ u64 start, ofs;
+ struct page *page;
+
+ vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
+ start = vma->vm_start;
+ for (ofs = 0; ofs < queue->queue_length; ofs += PAGE_SIZE) {
+ u64 virt_addr = (u64)ipz_qeit_calc(queue, ofs);
+ page = virt_to_page(virt_addr);
+ ret = vm_insert_page(vma, start, page);
+ if (unlikely(ret)) {
+ ehca_gen_err("vm_insert_page() failed rc=%i", ret);
+ return ret;
+ }
+ start += PAGE_SIZE;
+ }
+ vma->vm_private_data = mm_count;
+ (*mm_count)++;
+ vma->vm_ops = &vm_ops;
+
+ return 0;
+}
+
+static int ehca_mmap_cq(struct vm_area_struct *vma, struct ehca_cq *cq,
+ u32 rsrc_type)
+{
+ int ret;
+
+ switch (rsrc_type) {
+ case 0: /* galpa fw handle */
+ ehca_dbg(cq->ib_cq.device, "cq_num=%x fw", cq->cq_number);
+ ret = ehca_mmap_fw(vma, &cq->galpas, &cq->mm_count_galpa);
+ if (unlikely(ret)) {
+ ehca_err(cq->ib_cq.device,
+ "ehca_mmap_fw() failed rc=%i cq_num=%x",
+ ret, cq->cq_number);
+ return ret;
+ }
+ break;
+
+ case 1: /* cq queue_addr */
+ ehca_dbg(cq->ib_cq.device, "cq_num=%x queue", cq->cq_number);
+ ret = ehca_mmap_queue(vma, &cq->ipz_queue, &cq->mm_count_queue);
+ if (unlikely(ret)) {
+ ehca_err(cq->ib_cq.device,
+ "ehca_mmap_queue() failed rc=%i cq_num=%x",
+ ret, cq->cq_number);
+ return ret;
+ }
+ break;
+
+ default:
+ ehca_err(cq->ib_cq.device, "bad resource type=%x cq_num=%x",
+ rsrc_type, cq->cq_number);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int ehca_mmap_qp(struct vm_area_struct *vma, struct ehca_qp *qp,
+ u32 rsrc_type)
+{
+ int ret;
+
+ switch (rsrc_type) {
+ case 0: /* galpa fw handle */
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x fw", qp->ib_qp.qp_num);
+ ret = ehca_mmap_fw(vma, &qp->galpas, &qp->mm_count_galpa);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "remap_pfn_range() failed ret=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return -ENOMEM;
+ }
+ break;
+
+ case 1: /* qp rqueue_addr */
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x rq", qp->ib_qp.qp_num);
+ ret = ehca_mmap_queue(vma, &qp->ipz_rqueue,
+ &qp->mm_count_rqueue);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "ehca_mmap_queue(rq) failed rc=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return ret;
+ }
+ break;
+
+ case 2: /* qp squeue_addr */
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x sq", qp->ib_qp.qp_num);
+ ret = ehca_mmap_queue(vma, &qp->ipz_squeue,
+ &qp->mm_count_squeue);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "ehca_mmap_queue(sq) failed rc=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return ret;
+ }
+ break;
+
+ default:
+ ehca_err(qp->ib_qp.device, "bad resource type=%x qp=num=%x",
+ rsrc_type, qp->ib_qp.qp_num);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+ u64 fileoffset = vma->vm_pgoff;
+ u32 idr_handle = fileoffset & 0x1FFFFFF;
+ u32 q_type = (fileoffset >> 27) & 0x1; /* CQ, QP,... */
+ u32 rsrc_type = (fileoffset >> 25) & 0x3; /* sq,rq,cmnd_window */
+ u32 ret;
+ struct ehca_cq *cq;
+ struct ehca_qp *qp;
+ struct ib_uobject *uobject;
+
+ switch (q_type) {
+ case 0: /* CQ */
+ read_lock(&ehca_cq_idr_lock);
+ cq = idr_find(&ehca_cq_idr, idr_handle);
+ read_unlock(&ehca_cq_idr_lock);
+
+ /* make sure this mmap really belongs to the authorized user */
+ if (!cq)
+ return -EINVAL;
+
+ if (!cq->ib_cq.uobject || cq->ib_cq.uobject->context != context)
+ return -EINVAL;
+
+ ret = ehca_mmap_cq(vma, cq, rsrc_type);
+ if (unlikely(ret)) {
+ ehca_err(cq->ib_cq.device,
+ "ehca_mmap_cq() failed rc=%i cq_num=%x",
+ ret, cq->cq_number);
+ return ret;
+ }
+ break;
+
+ case 1: /* QP */
+ read_lock(&ehca_qp_idr_lock);
+ qp = idr_find(&ehca_qp_idr, idr_handle);
+ read_unlock(&ehca_qp_idr_lock);
+
+ /* make sure this mmap really belongs to the authorized user */
+ if (!qp)
+ return -EINVAL;
+
+ uobject = IS_SRQ(qp) ? qp->ib_srq.uobject : qp->ib_qp.uobject;
+ if (!uobject || uobject->context != context)
+ return -EINVAL;
+
+ ret = ehca_mmap_qp(vma, qp, rsrc_type);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "ehca_mmap_qp() failed rc=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return ret;
+ }
+ break;
+
+ default:
+ ehca_gen_err("bad queue type %x", q_type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Firmware Infiniband Interface code for POWER
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ * Gerd Bayer <gerd.bayer@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/hvcall.h>
+#include "ehca_tools.h"
+#include "hcp_if.h"
+#include "hcp_phyp.h"
+#include "hipz_fns.h"
+#include "ipz_pt_fn.h"
+
+#define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11)
+#define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12)
+#define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15)
+#define H_ALL_RES_QP_STORAGE EHCA_BMASK_IBM(16, 17)
+#define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18)
+#define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21)
+#define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23)
+#define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31)
+#define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35)
+#define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39)
+#define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63)
+
+#define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15)
+#define H_ALL_RES_QP_MAX_OUTST_RECV_WR EHCA_BMASK_IBM(16, 31)
+#define H_ALL_RES_QP_MAX_SEND_SGE EHCA_BMASK_IBM(32, 39)
+#define H_ALL_RES_QP_MAX_RECV_SGE EHCA_BMASK_IBM(40, 47)
+
+#define H_ALL_RES_QP_UD_AV_LKEY EHCA_BMASK_IBM(32, 63)
+#define H_ALL_RES_QP_SRQ_QP_TOKEN EHCA_BMASK_IBM(0, 31)
+#define H_ALL_RES_QP_SRQ_QP_HANDLE EHCA_BMASK_IBM(0, 64)
+#define H_ALL_RES_QP_SRQ_LIMIT EHCA_BMASK_IBM(48, 63)
+#define H_ALL_RES_QP_SRQ_QPN EHCA_BMASK_IBM(40, 63)
+
+#define H_ALL_RES_QP_ACT_OUTST_SEND_WR EHCA_BMASK_IBM(16, 31)
+#define H_ALL_RES_QP_ACT_OUTST_RECV_WR EHCA_BMASK_IBM(48, 63)
+#define H_ALL_RES_QP_ACT_SEND_SGE EHCA_BMASK_IBM(8, 15)
+#define H_ALL_RES_QP_ACT_RECV_SGE EHCA_BMASK_IBM(24, 31)
+
+#define H_ALL_RES_QP_SQUEUE_SIZE_PAGES EHCA_BMASK_IBM(0, 31)
+#define H_ALL_RES_QP_RQUEUE_SIZE_PAGES EHCA_BMASK_IBM(32, 63)
+
+#define H_MP_INIT_TYPE EHCA_BMASK_IBM(44, 47)
+#define H_MP_SHUTDOWN EHCA_BMASK_IBM(48, 48)
+#define H_MP_RESET_QKEY_CTR EHCA_BMASK_IBM(49, 49)
+
+#define HCALL4_REGS_FORMAT "r4=%lx r5=%lx r6=%lx r7=%lx"
+#define HCALL7_REGS_FORMAT HCALL4_REGS_FORMAT " r8=%lx r9=%lx r10=%lx"
+#define HCALL9_REGS_FORMAT HCALL7_REGS_FORMAT " r11=%lx r12=%lx"
+
+static DEFINE_SPINLOCK(hcall_lock);
+
+static long ehca_plpar_hcall_norets(unsigned long opcode,
+ unsigned long arg1,
+ unsigned long arg2,
+ unsigned long arg3,
+ unsigned long arg4,
+ unsigned long arg5,
+ unsigned long arg6,
+ unsigned long arg7)
+{
+ long ret;
+ int i, sleep_msecs;
+ unsigned long flags = 0;
+
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("opcode=%lx " HCALL7_REGS_FORMAT,
+ opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+
+ for (i = 0; i < 5; i++) {
+ /* serialize hCalls to work around firmware issue */
+ if (ehca_lock_hcalls)
+ spin_lock_irqsave(&hcall_lock, flags);
+
+ ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
+ arg5, arg6, arg7);
+
+ if (ehca_lock_hcalls)
+ spin_unlock_irqrestore(&hcall_lock, flags);
+
+ if (H_IS_LONG_BUSY(ret)) {
+ sleep_msecs = get_longbusy_msecs(ret);
+ msleep_interruptible(sleep_msecs);
+ continue;
+ }
+
+ if (ret < H_SUCCESS)
+ ehca_gen_err("opcode=%lx ret=%li " HCALL7_REGS_FORMAT,
+ opcode, ret, arg1, arg2, arg3,
+ arg4, arg5, arg6, arg7);
+ else
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("opcode=%lx ret=%li", opcode, ret);
+
+ return ret;
+ }
+
+ return H_BUSY;
+}
+
+static long ehca_plpar_hcall9(unsigned long opcode,
+ unsigned long *outs, /* array of 9 outputs */
+ unsigned long arg1,
+ unsigned long arg2,
+ unsigned long arg3,
+ unsigned long arg4,
+ unsigned long arg5,
+ unsigned long arg6,
+ unsigned long arg7,
+ unsigned long arg8,
+ unsigned long arg9)
+{
+ long ret;
+ int i, sleep_msecs;
+ unsigned long flags = 0;
+
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT, opcode,
+ arg1, arg2, arg3, arg4, arg5,
+ arg6, arg7, arg8, arg9);
+
+ for (i = 0; i < 5; i++) {
+ /* serialize hCalls to work around firmware issue */
+ if (ehca_lock_hcalls)
+ spin_lock_irqsave(&hcall_lock, flags);
+
+ ret = plpar_hcall9(opcode, outs,
+ arg1, arg2, arg3, arg4, arg5,
+ arg6, arg7, arg8, arg9);
+
+ if (ehca_lock_hcalls)
+ spin_unlock_irqrestore(&hcall_lock, flags);
+
+ if (H_IS_LONG_BUSY(ret)) {
+ sleep_msecs = get_longbusy_msecs(ret);
+ msleep_interruptible(sleep_msecs);
+ continue;
+ }
+
+ if (ret < H_SUCCESS) {
+ ehca_gen_err("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT,
+ opcode, arg1, arg2, arg3, arg4, arg5,
+ arg6, arg7, arg8, arg9);
+ ehca_gen_err("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
+ ret, outs[0], outs[1], outs[2], outs[3],
+ outs[4], outs[5], outs[6], outs[7],
+ outs[8]);
+ } else if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
+ ret, outs[0], outs[1], outs[2], outs[3],
+ outs[4], outs[5], outs[6], outs[7],
+ outs[8]);
+ return ret;
+ }
+
+ return H_BUSY;
+}
+
+u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_pfeq *pfeq,
+ const u32 neq_control,
+ const u32 number_of_entries,
+ struct ipz_eq_handle *eq_handle,
+ u32 *act_nr_of_entries,
+ u32 *act_pages,
+ u32 *eq_ist)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+ u64 allocate_controls;
+
+ /* resource type */
+ allocate_controls = 3ULL;
+
+ /* ISN is associated */
+ if (neq_control != 1)
+ allocate_controls = (1ULL << (63 - 7)) | allocate_controls;
+ else /* notification event queue */
+ allocate_controls = (1ULL << 63) | allocate_controls;
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ allocate_controls, /* r5 */
+ number_of_entries, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ eq_handle->handle = outs[0];
+ *act_nr_of_entries = (u32)outs[3];
+ *act_pages = (u32)outs[4];
+ *eq_ist = (u32)outs[5];
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resource - ret=%lli ", ret);
+
+ return ret;
+}
+
+u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
+ struct ipz_eq_handle eq_handle,
+ const u64 event_mask)
+{
+ return ehca_plpar_hcall_norets(H_RESET_EVENTS,
+ adapter_handle.handle, /* r4 */
+ eq_handle.handle, /* r5 */
+ event_mask, /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ struct ehca_alloc_cq_parms *param)
+{
+ int rc;
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ 2, /* r5 */
+ param->eq_handle.handle, /* r6 */
+ cq->token, /* r7 */
+ param->nr_cqe, /* r8 */
+ 0, 0, 0, 0);
+ cq->ipz_cq_handle.handle = outs[0];
+ param->act_nr_of_entries = (u32)outs[3];
+ param->act_pages = (u32)outs[4];
+
+ if (ret == H_SUCCESS) {
+ rc = hcp_galpas_ctor(&cq->galpas, 0, outs[5], outs[6]);
+ if (rc) {
+ ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
+ rc, outs[5]);
+
+ ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ cq->ipz_cq_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+ ret = H_NO_MEM;
+ }
+ }
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resources. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_alloc_qp_parms *parms, int is_user)
+{
+ int rc;
+ u64 ret;
+ u64 allocate_controls, max_r10_reg, r11, r12;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ allocate_controls =
+ EHCA_BMASK_SET(H_ALL_RES_QP_ENHANCED_OPS, parms->ext_type)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE,
+ parms->squeue.page_size)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE,
+ parms->rqueue.page_size)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
+ !!(parms->ll_comp_flags & LLQP_RECV_COMP))
+ | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
+ !!(parms->ll_comp_flags & LLQP_SEND_COMP))
+ | EHCA_BMASK_SET(H_ALL_RES_QP_UD_AV_LKEY_CTRL,
+ parms->ud_av_l_key_ctl)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_RESOURCE_TYPE, 1);
+
+ max_r10_reg =
+ EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
+ parms->squeue.max_wr + 1)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
+ parms->rqueue.max_wr + 1)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
+ parms->squeue.max_sge)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
+ parms->rqueue.max_sge);
+
+ r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token);
+
+ if (parms->ext_type == EQPT_SRQ)
+ r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_LIMIT, parms->srq_limit);
+ else
+ r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QPN, parms->srq_qpn);
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ allocate_controls, /* r5 */
+ parms->send_cq_handle.handle,
+ parms->recv_cq_handle.handle,
+ parms->eq_handle.handle,
+ ((u64)parms->token << 32) | parms->pd.value,
+ max_r10_reg, r11, r12);
+
+ parms->qp_handle.handle = outs[0];
+ parms->real_qp_num = (u32)outs[1];
+ parms->squeue.act_nr_wqes =
+ (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
+ parms->rqueue.act_nr_wqes =
+ (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
+ parms->squeue.act_nr_sges =
+ (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
+ parms->rqueue.act_nr_sges =
+ (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
+ parms->squeue.queue_size =
+ (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
+ parms->rqueue.queue_size =
+ (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
+
+ if (ret == H_SUCCESS) {
+ rc = hcp_galpas_ctor(&parms->galpas, is_user, outs[6], outs[6]);
+ if (rc) {
+ ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
+ rc, outs[6]);
+
+ ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ parms->qp_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+ ret = H_NO_MEM;
+ }
+ }
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resources. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id,
+ struct hipz_query_port *query_port_response_block)
+{
+ u64 ret;
+ u64 r_cb = __pa(query_port_response_block);
+
+ if (r_cb & (EHCA_PAGESIZE-1)) {
+ ehca_gen_err("response block not page aligned");
+ return H_PARAMETER;
+ }
+
+ ret = ehca_plpar_hcall_norets(H_QUERY_PORT,
+ adapter_handle.handle, /* r4 */
+ port_id, /* r5 */
+ r_cb, /* r6 */
+ 0, 0, 0, 0);
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(query_port_response_block, 64, "response_block");
+
+ return ret;
+}
+
+u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id, const u32 port_cap,
+ const u8 init_type, const int modify_mask)
+{
+ u64 port_attributes = port_cap;
+
+ if (modify_mask & IB_PORT_SHUTDOWN)
+ port_attributes |= EHCA_BMASK_SET(H_MP_SHUTDOWN, 1);
+ if (modify_mask & IB_PORT_INIT_TYPE)
+ port_attributes |= EHCA_BMASK_SET(H_MP_INIT_TYPE, init_type);
+ if (modify_mask & IB_PORT_RESET_QKEY_CNTR)
+ port_attributes |= EHCA_BMASK_SET(H_MP_RESET_QKEY_CTR, 1);
+
+ return ehca_plpar_hcall_norets(H_MODIFY_PORT,
+ adapter_handle.handle, /* r4 */
+ port_id, /* r5 */
+ port_attributes, /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
+ struct hipz_query_hca *query_hca_rblock)
+{
+ u64 r_cb = __pa(query_hca_rblock);
+
+ if (r_cb & (EHCA_PAGESIZE-1)) {
+ ehca_gen_err("response_block=%p not page aligned",
+ query_hca_rblock);
+ return H_PARAMETER;
+ }
+
+ return ehca_plpar_hcall_norets(H_QUERY_HCA,
+ adapter_handle.handle, /* r4 */
+ r_cb, /* r5 */
+ 0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 resource_handle,
+ const u64 logical_address_of_page,
+ u64 count)
+{
+ return ehca_plpar_hcall_norets(H_REGISTER_RPAGES,
+ adapter_handle.handle, /* r4 */
+ (u64)queue_type | ((u64)pagesize) << 8,
+ /* r5 */
+ resource_handle, /* r6 */
+ logical_address_of_page, /* r7 */
+ count, /* r8 */
+ 0, 0);
+}
+
+u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_eq_handle eq_handle,
+ struct ehca_pfeq *pfeq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count)
+{
+ if (count != 1) {
+ ehca_gen_err("Ppage counter=%llx", count);
+ return H_PARAMETER;
+ }
+ return hipz_h_register_rpage(adapter_handle,
+ pagesize,
+ queue_type,
+ eq_handle.handle,
+ logical_address_of_page, count);
+}
+
+u64 hipz_h_query_int_state(const struct ipz_adapter_handle adapter_handle,
+ u32 ist)
+{
+ u64 ret;
+ ret = ehca_plpar_hcall_norets(H_QUERY_INT_STATE,
+ adapter_handle.handle, /* r4 */
+ ist, /* r5 */
+ 0, 0, 0, 0, 0);
+
+ if (ret != H_SUCCESS && ret != H_BUSY)
+ ehca_gen_err("Could not query interrupt state.");
+
+ return ret;
+}
+
+u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_cq_handle cq_handle,
+ struct ehca_pfcq *pfcq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa gal)
+{
+ if (count != 1) {
+ ehca_gen_err("Page counter=%llx", count);
+ return H_PARAMETER;
+ }
+
+ return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
+ cq_handle.handle, logical_address_of_page,
+ count);
+}
+
+u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa galpa)
+{
+ if (count > 1) {
+ ehca_gen_err("Page counter=%llx", count);
+ return H_PARAMETER;
+ }
+
+ return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
+ qp_handle.handle, logical_address_of_page,
+ count);
+}
+
+u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ void **log_addr_next_sq_wqe2processed,
+ void **log_addr_next_rq_wqe2processed,
+ int dis_and_get_function_code)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
+ adapter_handle.handle, /* r4 */
+ dis_and_get_function_code, /* r5 */
+ qp_handle.handle, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ if (log_addr_next_sq_wqe2processed)
+ *log_addr_next_sq_wqe2processed = (void *)outs[0];
+ if (log_addr_next_rq_wqe2processed)
+ *log_addr_next_rq_wqe2processed = (void *)outs[1];
+
+ return ret;
+}
+
+u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u64 update_mask,
+ struct hcp_modify_qp_control_block *mqpcb,
+ struct h_galpa gal)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+ ret = ehca_plpar_hcall9(H_MODIFY_QP, outs,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ update_mask, /* r6 */
+ __pa(mqpcb), /* r7 */
+ 0, 0, 0, 0, 0);
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Insufficient resources ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ struct hcp_modify_qp_control_block *qqpcb,
+ struct h_galpa gal)
+{
+ return ehca_plpar_hcall_norets(H_QUERY_QP,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ __pa(qqpcb), /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_qp *qp)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = hcp_galpas_dtor(&qp->galpas);
+ if (ret) {
+ ehca_gen_err("Could not destruct qp->galpas");
+ return H_RESOURCE;
+ }
+ ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
+ adapter_handle.handle, /* r4 */
+ /* function code */
+ 1, /* r5 */
+ qp->ipz_qp_handle.handle, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ if (ret == H_HARDWARE)
+ ehca_gen_err("HCA not operational. ret=%lli", ret);
+
+ ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ qp->ipz_qp_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+
+ if (ret == H_RESOURCE)
+ ehca_gen_err("Resource still in use. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port)
+{
+ return ehca_plpar_hcall_norets(H_DEFINE_AQP0,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ port, /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port, u32 * pma_qp_nr,
+ u32 * bma_qp_nr)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_DEFINE_AQP1, outs,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ port, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ *pma_qp_nr = (u32)outs[0];
+ *bma_qp_nr = (u32)outs[1];
+
+ if (ret == H_ALIAS_EXIST)
+ ehca_gen_err("AQP1 already exists. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id)
+{
+ u64 ret;
+
+ ret = ehca_plpar_hcall_norets(H_ATTACH_MCQP,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ mcg_dlid, /* r6 */
+ interface_id, /* r7 */
+ subnet_prefix, /* r8 */
+ 0, 0);
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resources. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id)
+{
+ return ehca_plpar_hcall_norets(H_DETACH_MCQP,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ mcg_dlid, /* r6 */
+ interface_id, /* r7 */
+ subnet_prefix, /* r8 */
+ 0, 0);
+}
+
+u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ u8 force_flag)
+{
+ u64 ret;
+
+ ret = hcp_galpas_dtor(&cq->galpas);
+ if (ret) {
+ ehca_gen_err("Could not destruct cp->galpas");
+ return H_RESOURCE;
+ }
+
+ ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ cq->ipz_cq_handle.handle, /* r5 */
+ force_flag != 0 ? 1L : 0L, /* r6 */
+ 0, 0, 0, 0);
+
+ if (ret == H_RESOURCE)
+ ehca_gen_err("H_FREE_RESOURCE failed ret=%lli ", ret);
+
+ return ret;
+}
+
+u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_eq *eq)
+{
+ u64 ret;
+
+ ret = hcp_galpas_dtor(&eq->galpas);
+ if (ret) {
+ ehca_gen_err("Could not destruct eq->galpas");
+ return H_RESOURCE;
+ }
+
+ ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ eq->ipz_eq_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+
+ if (ret == H_RESOURCE)
+ ehca_gen_err("Resource in use. ret=%lli ", ret);
+
+ return ret;
+}
+
+u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ 5, /* r5 */
+ vaddr, /* r6 */
+ length, /* r7 */
+ (((u64)access_ctrl) << 32ULL), /* r8 */
+ pd.value, /* r9 */
+ 0, 0, 0);
+ outparms->handle.handle = outs[0];
+ outparms->lkey = (u32)outs[2];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count)
+{
+ u64 ret;
+
+ if (unlikely(ehca_debug_level >= 3)) {
+ if (count > 1) {
+ u64 *kpage;
+ int i;
+ kpage = __va(logical_address_of_page);
+ for (i = 0; i < count; i++)
+ ehca_gen_dbg("kpage[%d]=%p",
+ i, (void *)kpage[i]);
+ } else
+ ehca_gen_dbg("kpage=%p",
+ (void *)logical_address_of_page);
+ }
+
+ if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
+ ehca_gen_err("logical_address_of_page not on a 4k boundary "
+ "adapter_handle=%llx mr=%p mr_handle=%llx "
+ "pagesize=%x queue_type=%x "
+ "logical_address_of_page=%llx count=%llx",
+ adapter_handle.handle, mr,
+ mr->ipz_mr_handle.handle, pagesize, queue_type,
+ logical_address_of_page, count);
+ ret = H_PARAMETER;
+ } else
+ ret = hipz_h_register_rpage(adapter_handle, pagesize,
+ queue_type,
+ mr->ipz_mr_handle.handle,
+ logical_address_of_page, count);
+ return ret;
+}
+
+u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_QUERY_MR, outs,
+ adapter_handle.handle, /* r4 */
+ mr->ipz_mr_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0, 0, 0);
+ outparms->len = outs[0];
+ outparms->vaddr = outs[1];
+ outparms->acl = outs[4] >> 32;
+ outparms->lkey = (u32)(outs[5] >> 32);
+ outparms->rkey = (u32)(outs[5] & (0xffffffff));
+
+ return ret;
+}
+
+u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr)
+{
+ return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ mr->ipz_mr_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr_in,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ const u64 mr_addr_cb,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_REREGISTER_PMR, outs,
+ adapter_handle.handle, /* r4 */
+ mr->ipz_mr_handle.handle, /* r5 */
+ vaddr_in, /* r6 */
+ length, /* r7 */
+ /* r8 */
+ ((((u64)access_ctrl) << 32ULL) | pd.value),
+ mr_addr_cb, /* r9 */
+ 0, 0, 0);
+ outparms->vaddr = outs[1];
+ outparms->lkey = (u32)outs[2];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const struct ehca_mr *orig_mr,
+ const u64 vaddr_in,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_REGISTER_SMR, outs,
+ adapter_handle.handle, /* r4 */
+ orig_mr->ipz_mr_handle.handle, /* r5 */
+ vaddr_in, /* r6 */
+ (((u64)access_ctrl) << 32ULL), /* r7 */
+ pd.value, /* r8 */
+ 0, 0, 0, 0);
+ outparms->handle.handle = outs[0];
+ outparms->lkey = (u32)outs[2];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ const struct ipz_pd pd,
+ struct ehca_mw_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ 6, /* r5 */
+ pd.value, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ outparms->handle.handle = outs[0];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ struct ehca_mw_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_QUERY_MW, outs,
+ adapter_handle.handle, /* r4 */
+ mw->ipz_mw_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0, 0, 0);
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw)
+{
+ return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ mw->ipz_mw_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
+ const u64 ressource_handle,
+ void *rblock,
+ unsigned long *byte_count)
+{
+ u64 r_cb = __pa(rblock);
+
+ if (r_cb & (EHCA_PAGESIZE-1)) {
+ ehca_gen_err("rblock not page aligned.");
+ return H_PARAMETER;
+ }
+
+ return ehca_plpar_hcall_norets(H_ERROR_DATA,
+ adapter_handle.handle,
+ ressource_handle,
+ r_cb,
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_eoi(int irq)
+{
+ unsigned long xirr;
+
+ iosync();
+ xirr = (0xffULL << 24) | irq;
+
+ return plpar_hcall_norets(H_EOI, xirr);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Firmware Infiniband Interface code for POWER
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Gerd Bayer <gerd.bayer@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HCP_IF_H__
+#define __HCP_IF_H__
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "hipz_hw.h"
+
+/*
+ * hipz_h_alloc_resource_eq allocates EQ resources in HW and FW, initialize
+ * resources, create the empty EQPT (ring).
+ */
+u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_pfeq *pfeq,
+ const u32 neq_control,
+ const u32 number_of_entries,
+ struct ipz_eq_handle *eq_handle,
+ u32 * act_nr_of_entries,
+ u32 * act_pages,
+ u32 * eq_ist);
+
+u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
+ struct ipz_eq_handle eq_handle,
+ const u64 event_mask);
+/*
+ * hipz_h_allocate_resource_cq allocates CQ resources in HW and FW, initialize
+ * resources, create the empty CQPT (ring).
+ */
+u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ struct ehca_alloc_cq_parms *param);
+
+
+/*
+ * hipz_h_alloc_resource_qp allocates QP resources in HW and FW,
+ * initialize resources, create empty QPPTs (2 rings).
+ */
+u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_alloc_qp_parms *parms, int is_user);
+
+u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id,
+ struct hipz_query_port *query_port_response_block);
+
+u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id, const u32 port_cap,
+ const u8 init_type, const int modify_mask);
+
+u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
+ struct hipz_query_hca *query_hca_rblock);
+
+/*
+ * hipz_h_register_rpage internal function in hcp_if.h for all
+ * hcp_H_REGISTER_RPAGE calls.
+ */
+u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 resource_handle,
+ const u64 logical_address_of_page,
+ u64 count);
+
+u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_eq_handle eq_handle,
+ struct ehca_pfeq *pfeq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count);
+
+u64 hipz_h_query_int_state(const struct ipz_adapter_handle
+ hcp_adapter_handle,
+ u32 ist);
+
+u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_cq_handle cq_handle,
+ struct ehca_pfcq *pfcq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa gal);
+
+u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa galpa);
+
+u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ void **log_addr_next_sq_wqe_tb_processed,
+ void **log_addr_next_rq_wqe_tb_processed,
+ int dis_and_get_function_code);
+enum hcall_sigt {
+ HCALL_SIGT_NO_CQE = 0,
+ HCALL_SIGT_BY_WQE = 1,
+ HCALL_SIGT_EVERY = 2
+};
+
+u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u64 update_mask,
+ struct hcp_modify_qp_control_block *mqpcb,
+ struct h_galpa gal);
+
+u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ struct hcp_modify_qp_control_block *qqpcb,
+ struct h_galpa gal);
+
+u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_qp *qp);
+
+u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port);
+
+u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port, u32 * pma_qp_nr,
+ u32 * bma_qp_nr);
+
+u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id);
+
+u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id);
+
+u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ u8 force_flag);
+
+u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_eq *eq);
+
+/*
+ * hipz_h_alloc_resource_mr allocates MR resources in HW and FW, initialize
+ * resources.
+ */
+u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_register_rpage_mr registers MR resource pages in HW and FW */
+u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count);
+
+/* hipz_h_query_mr queries MR in HW and FW */
+u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_free_resource_mr frees MR resources in HW and FW */
+u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr);
+
+/* hipz_h_reregister_pmr reregisters MR in HW and FW */
+u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr_in,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ const u64 mr_addr_cb,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_register_smr register shared MR in HW and FW */
+u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const struct ehca_mr *orig_mr,
+ const u64 vaddr_in,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/*
+ * hipz_h_alloc_resource_mw allocates MW resources in HW and FW, initialize
+ * resources.
+ */
+u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ const struct ipz_pd pd,
+ struct ehca_mw_hipzout_parms *outparms);
+
+/* hipz_h_query_mw queries MW in HW and FW */
+u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ struct ehca_mw_hipzout_parms *outparms);
+
+/* hipz_h_free_resource_mw frees MW resources in HW and FW */
+u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw);
+
+u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
+ const u64 ressource_handle,
+ void *rblock,
+ unsigned long *byte_count);
+u64 hipz_h_eoi(int irq);
+
+#endif /* __HCP_IF_H__ */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * load store abstraction for ehca register access with tracing
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "hipz_hw.h"
+
+u64 hcall_map_page(u64 physaddr)
+{
+ return (u64)ioremap(physaddr, EHCA_PAGESIZE);
+}
+
+int hcall_unmap_page(u64 mapaddr)
+{
+ iounmap((volatile void __iomem *) mapaddr);
+ return 0;
+}
+
+int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
+ u64 paddr_kernel, u64 paddr_user)
+{
+ if (!is_user) {
+ galpas->kernel.fw_handle = hcall_map_page(paddr_kernel);
+ if (!galpas->kernel.fw_handle)
+ return -ENOMEM;
+ } else
+ galpas->kernel.fw_handle = 0;
+
+ galpas->user.fw_handle = paddr_user;
+
+ return 0;
+}
+
+int hcp_galpas_dtor(struct h_galpas *galpas)
+{
+ if (galpas->kernel.fw_handle) {
+ int ret = hcall_unmap_page(galpas->kernel.fw_handle);
+ if (ret)
+ return ret;
+ }
+
+ galpas->user.fw_handle = galpas->kernel.fw_handle = 0;
+
+ return 0;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Firmware calls
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Gerd Bayer <gerd.bayer@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HCP_PHYP_H__
+#define __HCP_PHYP_H__
+
+
+/*
+ * eHCA page (mapped into memory)
+ * resource to access eHCA register pages in CPU address space
+*/
+struct h_galpa {
+ u64 fw_handle;
+ /* for pSeries this is a 64bit memory address where
+ I/O memory is mapped into CPU address space (kv) */
+};
+
+/*
+ * resource to access eHCA address space registers, all types
+ */
+struct h_galpas {
+ u32 pid; /*PID of userspace galpa checking */
+ struct h_galpa user; /* user space accessible resource,
+ set to 0 if unused */
+ struct h_galpa kernel; /* kernel space accessible resource,
+ set to 0 if unused */
+};
+
+static inline u64 hipz_galpa_load(struct h_galpa galpa, u32 offset)
+{
+ u64 addr = galpa.fw_handle + offset;
+ return *(volatile u64 __force *)addr;
+}
+
+static inline void hipz_galpa_store(struct h_galpa galpa, u32 offset, u64 value)
+{
+ u64 addr = galpa.fw_handle + offset;
+ *(volatile u64 __force *)addr = value;
+}
+
+int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
+ u64 paddr_kernel, u64 paddr_user);
+
+int hcp_galpas_dtor(struct h_galpas *galpas);
+
+u64 hcall_map_page(u64 physaddr);
+
+int hcall_unmap_page(u64 mapaddr);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * HW abstraction register functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_FNS_H__
+#define __HIPZ_FNS_H__
+
+#include "ehca_classes.h"
+#include "hipz_hw.h"
+
+#include "hipz_fns_core.h"
+
+#define hipz_galpa_store_eq(gal, offset, value) \
+ hipz_galpa_store(gal, EQTEMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_eq(gal, offset) \
+ hipz_galpa_load(gal, EQTEMM_OFFSET(offset))
+
+#define hipz_galpa_store_qped(gal, offset, value) \
+ hipz_galpa_store(gal, QPEDMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_qped(gal, offset) \
+ hipz_galpa_load(gal, QPEDMM_OFFSET(offset))
+
+#define hipz_galpa_store_mrmw(gal, offset, value) \
+ hipz_galpa_store(gal, MRMWMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_mrmw(gal, offset) \
+ hipz_galpa_load(gal, MRMWMM_OFFSET(offset))
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * HW abstraction register functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_FNS_CORE_H__
+#define __HIPZ_FNS_CORE_H__
+
+#include "hcp_phyp.h"
+#include "hipz_hw.h"
+
+#define hipz_galpa_store_cq(gal, offset, value) \
+ hipz_galpa_store(gal, CQTEMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_cq(gal, offset) \
+ hipz_galpa_load(gal, CQTEMM_OFFSET(offset))
+
+#define hipz_galpa_store_qp(gal, offset, value) \
+ hipz_galpa_store(gal, QPTEMM_OFFSET(offset), value)
+#define hipz_galpa_load_qp(gal, offset) \
+ hipz_galpa_load(gal, QPTEMM_OFFSET(offset))
+
+static inline void hipz_update_sqa(struct ehca_qp *qp, u16 nr_wqes)
+{
+ /* ringing doorbell :-) */
+ hipz_galpa_store_qp(qp->galpas.kernel, qpx_sqa,
+ EHCA_BMASK_SET(QPX_SQADDER, nr_wqes));
+}
+
+static inline void hipz_update_rqa(struct ehca_qp *qp, u16 nr_wqes)
+{
+ /* ringing doorbell :-) */
+ hipz_galpa_store_qp(qp->galpas.kernel, qpx_rqa,
+ EHCA_BMASK_SET(QPX_RQADDER, nr_wqes));
+}
+
+static inline void hipz_update_feca(struct ehca_cq *cq, u32 nr_cqes)
+{
+ hipz_galpa_store_cq(cq->galpas.kernel, cqx_feca,
+ EHCA_BMASK_SET(CQX_FECADDER, nr_cqes));
+}
+
+static inline void hipz_set_cqx_n0(struct ehca_cq *cq, u32 value)
+{
+ u64 cqx_n0_reg;
+
+ hipz_galpa_store_cq(cq->galpas.kernel, cqx_n0,
+ EHCA_BMASK_SET(CQX_N0_GENERATE_SOLICITED_COMP_EVENT,
+ value));
+ cqx_n0_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n0);
+}
+
+static inline void hipz_set_cqx_n1(struct ehca_cq *cq, u32 value)
+{
+ u64 cqx_n1_reg;
+
+ hipz_galpa_store_cq(cq->galpas.kernel, cqx_n1,
+ EHCA_BMASK_SET(CQX_N1_GENERATE_COMP_EVENT, value));
+ cqx_n1_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n1);
+}
+
+#endif /* __HIPZ_FNC_CORE_H__ */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * eHCA register definitions
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_HW_H__
+#define __HIPZ_HW_H__
+
+#include "ehca_tools.h"
+
+#define EHCA_MAX_MTU 4
+
+/* QP Table Entry Memory Map */
+struct hipz_qptemm {
+ u64 qpx_hcr;
+ u64 qpx_c;
+ u64 qpx_herr;
+ u64 qpx_aer;
+/* 0x20*/
+ u64 qpx_sqa;
+ u64 qpx_sqc;
+ u64 qpx_rqa;
+ u64 qpx_rqc;
+/* 0x40*/
+ u64 qpx_st;
+ u64 qpx_pmstate;
+ u64 qpx_pmfa;
+ u64 qpx_pkey;
+/* 0x60*/
+ u64 qpx_pkeya;
+ u64 qpx_pkeyb;
+ u64 qpx_pkeyc;
+ u64 qpx_pkeyd;
+/* 0x80*/
+ u64 qpx_qkey;
+ u64 qpx_dqp;
+ u64 qpx_dlidp;
+ u64 qpx_portp;
+/* 0xa0*/
+ u64 qpx_slidp;
+ u64 qpx_slidpp;
+ u64 qpx_dlida;
+ u64 qpx_porta;
+/* 0xc0*/
+ u64 qpx_slida;
+ u64 qpx_slidpa;
+ u64 qpx_slvl;
+ u64 qpx_ipd;
+/* 0xe0*/
+ u64 qpx_mtu;
+ u64 qpx_lato;
+ u64 qpx_rlimit;
+ u64 qpx_rnrlimit;
+/* 0x100*/
+ u64 qpx_t;
+ u64 qpx_sqhp;
+ u64 qpx_sqptp;
+ u64 qpx_nspsn;
+/* 0x120*/
+ u64 qpx_nspsnhwm;
+ u64 reserved1;
+ u64 qpx_sdsi;
+ u64 qpx_sdsbc;
+/* 0x140*/
+ u64 qpx_sqwsize;
+ u64 qpx_sqwts;
+ u64 qpx_lsn;
+ u64 qpx_nssn;
+/* 0x160 */
+ u64 qpx_mor;
+ u64 qpx_cor;
+ u64 qpx_sqsize;
+ u64 qpx_erc;
+/* 0x180*/
+ u64 qpx_rnrrc;
+ u64 qpx_ernrwt;
+ u64 qpx_rnrresp;
+ u64 qpx_lmsna;
+/* 0x1a0 */
+ u64 qpx_sqhpc;
+ u64 qpx_sqcptp;
+ u64 qpx_sigt;
+ u64 qpx_wqecnt;
+/* 0x1c0*/
+ u64 qpx_rqhp;
+ u64 qpx_rqptp;
+ u64 qpx_rqsize;
+ u64 qpx_nrr;
+/* 0x1e0*/
+ u64 qpx_rdmac;
+ u64 qpx_nrpsn;
+ u64 qpx_lapsn;
+ u64 qpx_lcr;
+/* 0x200*/
+ u64 qpx_rwc;
+ u64 qpx_rwva;
+ u64 qpx_rdsi;
+ u64 qpx_rdsbc;
+/* 0x220*/
+ u64 qpx_rqwsize;
+ u64 qpx_crmsn;
+ u64 qpx_rdd;
+ u64 qpx_larpsn;
+/* 0x240*/
+ u64 qpx_pd;
+ u64 qpx_scqn;
+ u64 qpx_rcqn;
+ u64 qpx_aeqn;
+/* 0x260*/
+ u64 qpx_aaelog;
+ u64 qpx_ram;
+ u64 qpx_rdmaqe0;
+ u64 qpx_rdmaqe1;
+/* 0x280*/
+ u64 qpx_rdmaqe2;
+ u64 qpx_rdmaqe3;
+ u64 qpx_nrpsnhwm;
+/* 0x298*/
+ u64 reserved[(0x400 - 0x298) / 8];
+/* 0x400 extended data */
+ u64 reserved_ext[(0x500 - 0x400) / 8];
+/* 0x500 */
+ u64 reserved2[(0x1000 - 0x500) / 8];
+/* 0x1000 */
+};
+
+#define QPX_SQADDER EHCA_BMASK_IBM(48, 63)
+#define QPX_RQADDER EHCA_BMASK_IBM(48, 63)
+#define QPX_AAELOG_RESET_SRQ_LIMIT EHCA_BMASK_IBM(3, 3)
+
+#define QPTEMM_OFFSET(x) offsetof(struct hipz_qptemm, x)
+
+/* MRMWPT Entry Memory Map */
+struct hipz_mrmwmm {
+ /* 0x00 */
+ u64 mrx_hcr;
+
+ u64 mrx_c;
+ u64 mrx_herr;
+ u64 mrx_aer;
+ /* 0x20 */
+ u64 mrx_pp;
+ u64 reserved1;
+ u64 reserved2;
+ u64 reserved3;
+ /* 0x40 */
+ u64 reserved4[(0x200 - 0x40) / 8];
+ /* 0x200 */
+ u64 mrx_ctl[64];
+
+};
+
+#define MRMWMM_OFFSET(x) offsetof(struct hipz_mrmwmm, x)
+
+struct hipz_qpedmm {
+ /* 0x00 */
+ u64 reserved0[(0x400) / 8];
+ /* 0x400 */
+ u64 qpedx_phh;
+ u64 qpedx_ppsgp;
+ /* 0x410 */
+ u64 qpedx_ppsgu;
+ u64 qpedx_ppdgp;
+ /* 0x420 */
+ u64 qpedx_ppdgu;
+ u64 qpedx_aph;
+ /* 0x430 */
+ u64 qpedx_apsgp;
+ u64 qpedx_apsgu;
+ /* 0x440 */
+ u64 qpedx_apdgp;
+ u64 qpedx_apdgu;
+ /* 0x450 */
+ u64 qpedx_apav;
+ u64 qpedx_apsav;
+ /* 0x460 */
+ u64 qpedx_hcr;
+ u64 reserved1[4];
+ /* 0x488 */
+ u64 qpedx_rrl0;
+ /* 0x490 */
+ u64 qpedx_rrrkey0;
+ u64 qpedx_rrva0;
+ /* 0x4a0 */
+ u64 reserved2;
+ u64 qpedx_rrl1;
+ /* 0x4b0 */
+ u64 qpedx_rrrkey1;
+ u64 qpedx_rrva1;
+ /* 0x4c0 */
+ u64 reserved3;
+ u64 qpedx_rrl2;
+ /* 0x4d0 */
+ u64 qpedx_rrrkey2;
+ u64 qpedx_rrva2;
+ /* 0x4e0 */
+ u64 reserved4;
+ u64 qpedx_rrl3;
+ /* 0x4f0 */
+ u64 qpedx_rrrkey3;
+ u64 qpedx_rrva3;
+};
+
+#define QPEDMM_OFFSET(x) offsetof(struct hipz_qpedmm, x)
+
+/* CQ Table Entry Memory Map */
+struct hipz_cqtemm {
+ u64 cqx_hcr;
+ u64 cqx_c;
+ u64 cqx_herr;
+ u64 cqx_aer;
+/* 0x20 */
+ u64 cqx_ptp;
+ u64 cqx_tp;
+ u64 cqx_fec;
+ u64 cqx_feca;
+/* 0x40 */
+ u64 cqx_ep;
+ u64 cqx_eq;
+/* 0x50 */
+ u64 reserved1;
+ u64 cqx_n0;
+/* 0x60 */
+ u64 cqx_n1;
+ u64 reserved2[(0x1000 - 0x60) / 8];
+/* 0x1000 */
+};
+
+#define CQX_FEC_CQE_CNT EHCA_BMASK_IBM(32, 63)
+#define CQX_FECADDER EHCA_BMASK_IBM(32, 63)
+#define CQX_N0_GENERATE_SOLICITED_COMP_EVENT EHCA_BMASK_IBM(0, 0)
+#define CQX_N1_GENERATE_COMP_EVENT EHCA_BMASK_IBM(0, 0)
+
+#define CQTEMM_OFFSET(x) offsetof(struct hipz_cqtemm, x)
+
+/* EQ Table Entry Memory Map */
+struct hipz_eqtemm {
+ u64 eqx_hcr;
+ u64 eqx_c;
+
+ u64 eqx_herr;
+ u64 eqx_aer;
+/* 0x20 */
+ u64 eqx_ptp;
+ u64 eqx_tp;
+ u64 eqx_ssba;
+ u64 eqx_psba;
+
+/* 0x40 */
+ u64 eqx_cec;
+ u64 eqx_meql;
+ u64 eqx_xisbi;
+ u64 eqx_xisc;
+/* 0x60 */
+ u64 eqx_it;
+
+};
+
+#define EQTEMM_OFFSET(x) offsetof(struct hipz_eqtemm, x)
+
+/* access control defines for MR/MW */
+#define HIPZ_ACCESSCTRL_L_WRITE 0x00800000
+#define HIPZ_ACCESSCTRL_R_WRITE 0x00400000
+#define HIPZ_ACCESSCTRL_R_READ 0x00200000
+#define HIPZ_ACCESSCTRL_R_ATOMIC 0x00100000
+#define HIPZ_ACCESSCTRL_MW_BIND 0x00080000
+
+/* query hca response block */
+struct hipz_query_hca {
+ u32 cur_reliable_dg;
+ u32 cur_qp;
+ u32 cur_cq;
+ u32 cur_eq;
+ u32 cur_mr;
+ u32 cur_mw;
+ u32 cur_ee_context;
+ u32 cur_mcast_grp;
+ u32 cur_qp_attached_mcast_grp;
+ u32 reserved1;
+ u32 cur_ipv6_qp;
+ u32 cur_eth_qp;
+ u32 cur_hp_mr;
+ u32 reserved2[3];
+ u32 max_rd_domain;
+ u32 max_qp;
+ u32 max_cq;
+ u32 max_eq;
+ u32 max_mr;
+ u32 max_hp_mr;
+ u32 max_mw;
+ u32 max_mrwpte;
+ u32 max_special_mrwpte;
+ u32 max_rd_ee_context;
+ u32 max_mcast_grp;
+ u32 max_total_mcast_qp_attach;
+ u32 max_mcast_qp_attach;
+ u32 max_raw_ipv6_qp;
+ u32 max_raw_ethy_qp;
+ u32 internal_clock_frequency;
+ u32 max_pd;
+ u32 max_ah;
+ u32 max_cqe;
+ u32 max_wqes_wq;
+ u32 max_partitions;
+ u32 max_rr_ee_context;
+ u32 max_rr_qp;
+ u32 max_rr_hca;
+ u32 max_act_wqs_ee_context;
+ u32 max_act_wqs_qp;
+ u32 max_sge;
+ u32 max_sge_rd;
+ u32 memory_page_size_supported;
+ u64 max_mr_size;
+ u32 local_ca_ack_delay;
+ u32 num_ports;
+ u32 vendor_id;
+ u32 vendor_part_id;
+ u32 hw_ver;
+ u64 node_guid;
+ u64 hca_cap_indicators;
+ u32 data_counter_register_size;
+ u32 max_shared_rq;
+ u32 max_isns_eq;
+ u32 max_neq;
+} __attribute__ ((packed));
+
+#define HCA_CAP_AH_PORT_NR_CHECK EHCA_BMASK_IBM( 0, 0)
+#define HCA_CAP_ATOMIC EHCA_BMASK_IBM( 1, 1)
+#define HCA_CAP_AUTO_PATH_MIG EHCA_BMASK_IBM( 2, 2)
+#define HCA_CAP_BAD_P_KEY_CTR EHCA_BMASK_IBM( 3, 3)
+#define HCA_CAP_SQD_RTS_PORT_CHANGE EHCA_BMASK_IBM( 4, 4)
+#define HCA_CAP_CUR_QP_STATE_MOD EHCA_BMASK_IBM( 5, 5)
+#define HCA_CAP_INIT_TYPE EHCA_BMASK_IBM( 6, 6)
+#define HCA_CAP_PORT_ACTIVE_EVENT EHCA_BMASK_IBM( 7, 7)
+#define HCA_CAP_Q_KEY_VIOL_CTR EHCA_BMASK_IBM( 8, 8)
+#define HCA_CAP_WQE_RESIZE EHCA_BMASK_IBM( 9, 9)
+#define HCA_CAP_RAW_PACKET_MCAST EHCA_BMASK_IBM(10, 10)
+#define HCA_CAP_SHUTDOWN_PORT EHCA_BMASK_IBM(11, 11)
+#define HCA_CAP_RC_LL_QP EHCA_BMASK_IBM(12, 12)
+#define HCA_CAP_SRQ EHCA_BMASK_IBM(13, 13)
+#define HCA_CAP_UD_LL_QP EHCA_BMASK_IBM(16, 16)
+#define HCA_CAP_RESIZE_MR EHCA_BMASK_IBM(17, 17)
+#define HCA_CAP_MINI_QP EHCA_BMASK_IBM(18, 18)
+#define HCA_CAP_H_ALLOC_RES_SYNC EHCA_BMASK_IBM(19, 19)
+
+/* query port response block */
+struct hipz_query_port {
+ u32 state;
+ u32 bad_pkey_cntr;
+ u32 lmc;
+ u32 lid;
+ u32 subnet_timeout;
+ u32 qkey_viol_cntr;
+ u32 sm_sl;
+ u32 sm_lid;
+ u32 capability_mask;
+ u32 init_type_reply;
+ u32 pkey_tbl_len;
+ u32 gid_tbl_len;
+ u64 gid_prefix;
+ u32 port_nr;
+ u16 pkey_entries[16];
+ u8 reserved1[32];
+ u32 trent_size;
+ u32 trbuf_size;
+ u64 max_msg_sz;
+ u32 max_mtu;
+ u32 vl_cap;
+ u32 phys_pstate;
+ u32 phys_state;
+ u32 phys_speed;
+ u32 phys_width;
+ u8 reserved2[1884];
+ u64 guid_entries[255];
+} __attribute__ ((packed));
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * internal queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_tools.h"
+#include "ipz_pt_fn.h"
+#include "ehca_classes.h"
+
+#define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT)
+
+struct kmem_cache *small_qp_cache;
+
+void *ipz_qpageit_get_inc(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ queue->current_q_offset += queue->pagesize;
+ if (queue->current_q_offset > queue->queue_length) {
+ queue->current_q_offset -= queue->pagesize;
+ ret = NULL;
+ }
+ if (((u64)ret) % queue->pagesize) {
+ ehca_gen_err("ERROR!! not at PAGE-Boundary");
+ return NULL;
+ }
+ return ret;
+}
+
+void *ipz_qeit_eq_get_inc(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ u64 last_entry_in_q = queue->queue_length - queue->qe_size;
+
+ queue->current_q_offset += queue->qe_size;
+ if (queue->current_q_offset > last_entry_in_q) {
+ queue->current_q_offset = 0;
+ queue->toggle_state = (~queue->toggle_state) & 1;
+ }
+
+ return ret;
+}
+
+int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset)
+{
+ int i;
+ for (i = 0; i < queue->queue_length / queue->pagesize; i++) {
+ u64 page = __pa(queue->queue_pages[i]);
+ if (addr >= page && addr < page + queue->pagesize) {
+ *q_offset = addr - page + i * queue->pagesize;
+ return 0;
+ }
+ }
+ return -EINVAL;
+}
+
+#if PAGE_SHIFT < EHCA_PAGESHIFT
+#error Kernel pages must be at least as large than eHCA pages (4K) !
+#endif
+
+/*
+ * allocate pages for queue:
+ * outer loop allocates whole kernel pages (page aligned) and
+ * inner loop divides a kernel page into smaller hca queue pages
+ */
+static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages)
+{
+ int k, f = 0;
+ u8 *kpage;
+
+ while (f < nr_of_pages) {
+ kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
+ if (!kpage)
+ goto out;
+
+ for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) {
+ queue->queue_pages[f] = (struct ipz_page *)kpage;
+ kpage += EHCA_PAGESIZE;
+ f++;
+ }
+ }
+ return 1;
+
+out:
+ for (f = 0; f < nr_of_pages && queue->queue_pages[f];
+ f += PAGES_PER_KPAGE)
+ free_page((unsigned long)(queue->queue_pages)[f]);
+ return 0;
+}
+
+static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
+{
+ int order = ilog2(queue->pagesize) - 9;
+ struct ipz_small_queue_page *page;
+ unsigned long bit;
+
+ mutex_lock(&pd->lock);
+
+ if (!list_empty(&pd->free[order]))
+ page = list_entry(pd->free[order].next,
+ struct ipz_small_queue_page, list);
+ else {
+ page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL);
+ if (!page)
+ goto out;
+
+ page->page = get_zeroed_page(GFP_KERNEL);
+ if (!page->page) {
+ kmem_cache_free(small_qp_cache, page);
+ goto out;
+ }
+
+ list_add(&page->list, &pd->free[order]);
+ }
+
+ bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order);
+ __set_bit(bit, page->bitmap);
+ page->fill++;
+
+ if (page->fill == IPZ_SPAGE_PER_KPAGE >> order)
+ list_move(&page->list, &pd->full[order]);
+
+ mutex_unlock(&pd->lock);
+
+ queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9)));
+ queue->small_page = page;
+ queue->offset = bit << (order + 9);
+ return 1;
+
+out:
+ ehca_err(pd->ib_pd.device, "failed to allocate small queue page");
+ mutex_unlock(&pd->lock);
+ return 0;
+}
+
+static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
+{
+ int order = ilog2(queue->pagesize) - 9;
+ struct ipz_small_queue_page *page = queue->small_page;
+ unsigned long bit;
+ int free_page = 0;
+
+ bit = ((unsigned long)queue->queue_pages[0] & ~PAGE_MASK)
+ >> (order + 9);
+
+ mutex_lock(&pd->lock);
+
+ __clear_bit(bit, page->bitmap);
+ page->fill--;
+
+ if (page->fill == 0) {
+ list_del(&page->list);
+ free_page = 1;
+ }
+
+ if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1)
+ /* the page was full until we freed the chunk */
+ list_move_tail(&page->list, &pd->free[order]);
+
+ mutex_unlock(&pd->lock);
+
+ if (free_page) {
+ free_page(page->page);
+ kmem_cache_free(small_qp_cache, page);
+ }
+}
+
+int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
+ const u32 nr_of_pages, const u32 pagesize,
+ const u32 qe_size, const u32 nr_of_sg,
+ int is_small)
+{
+ if (pagesize > PAGE_SIZE) {
+ ehca_gen_err("FATAL ERROR: pagesize=%x "
+ "is greater than kernel page size", pagesize);
+ return 0;
+ }
+
+ /* init queue fields */
+ queue->queue_length = nr_of_pages * pagesize;
+ queue->pagesize = pagesize;
+ queue->qe_size = qe_size;
+ queue->act_nr_of_sg = nr_of_sg;
+ queue->current_q_offset = 0;
+ queue->toggle_state = 1;
+ queue->small_page = NULL;
+
+ /* allocate queue page pointers */
+ queue->queue_pages = kzalloc(nr_of_pages * sizeof(void *),
+ GFP_KERNEL | __GFP_NOWARN);
+ if (!queue->queue_pages) {
+ queue->queue_pages = vzalloc(nr_of_pages * sizeof(void *));
+ if (!queue->queue_pages) {
+ ehca_gen_err("Couldn't allocate queue page list");
+ return 0;
+ }
+ }
+
+ /* allocate actual queue pages */
+ if (is_small) {
+ if (!alloc_small_queue_page(queue, pd))
+ goto ipz_queue_ctor_exit0;
+ } else
+ if (!alloc_queue_pages(queue, nr_of_pages))
+ goto ipz_queue_ctor_exit0;
+
+ return 1;
+
+ipz_queue_ctor_exit0:
+ ehca_gen_err("Couldn't alloc pages queue=%p "
+ "nr_of_pages=%x", queue, nr_of_pages);
+ kvfree(queue->queue_pages);
+
+ return 0;
+}
+
+int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue)
+{
+ int i, nr_pages;
+
+ if (!queue || !queue->queue_pages) {
+ ehca_gen_dbg("queue or queue_pages is NULL");
+ return 0;
+ }
+
+ if (queue->small_page)
+ free_small_queue_page(queue, pd);
+ else {
+ nr_pages = queue->queue_length / queue->pagesize;
+ for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE)
+ free_page((unsigned long)queue->queue_pages[i]);
+ }
+
+ kvfree(queue->queue_pages);
+
+ return 1;
+}
+
+int ehca_init_small_qp_cache(void)
+{
+ small_qp_cache = kmem_cache_create("ehca_cache_small_qp",
+ sizeof(struct ipz_small_queue_page),
+ 0, SLAB_HWCACHE_ALIGN, NULL);
+ if (!small_qp_cache)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void ehca_cleanup_small_qp_cache(void)
+{
+ kmem_cache_destroy(small_qp_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * internal queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __IPZ_PT_FN_H__
+#define __IPZ_PT_FN_H__
+
+#define EHCA_PAGESHIFT 12
+#define EHCA_PAGESIZE 4096UL
+#define EHCA_PAGEMASK (~(EHCA_PAGESIZE-1))
+#define EHCA_PT_ENTRIES 512UL
+
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+
+struct ehca_pd;
+struct ipz_small_queue_page;
+
+extern struct kmem_cache *small_qp_cache;
+
+/* struct generic ehca page */
+struct ipz_page {
+ u8 entries[EHCA_PAGESIZE];
+};
+
+#define IPZ_SPAGE_PER_KPAGE (PAGE_SIZE / 512)
+
+struct ipz_small_queue_page {
+ unsigned long page;
+ unsigned long bitmap[IPZ_SPAGE_PER_KPAGE / BITS_PER_LONG];
+ int fill;
+ void *mapped_addr;
+ u32 mmap_count;
+ struct list_head list;
+};
+
+/* struct generic queue in linux kernel virtual memory (kv) */
+struct ipz_queue {
+ u64 current_q_offset; /* current queue entry */
+
+ struct ipz_page **queue_pages; /* array of pages belonging to queue */
+ u32 qe_size; /* queue entry size */
+ u32 act_nr_of_sg;
+ u32 queue_length; /* queue length allocated in bytes */
+ u32 pagesize;
+ u32 toggle_state; /* toggle flag - per page */
+ u32 offset; /* save offset within page for small_qp */
+ struct ipz_small_queue_page *small_page;
+};
+
+/*
+ * return current Queue Entry for a certain q_offset
+ * returns address (kv) of Queue Entry
+ */
+static inline void *ipz_qeit_calc(struct ipz_queue *queue, u64 q_offset)
+{
+ struct ipz_page *current_page;
+ if (q_offset >= queue->queue_length)
+ return NULL;
+ current_page = (queue->queue_pages)[q_offset >> EHCA_PAGESHIFT];
+ return ¤t_page->entries[q_offset & (EHCA_PAGESIZE - 1)];
+}
+
+/*
+ * return current Queue Entry
+ * returns address (kv) of Queue Entry
+ */
+static inline void *ipz_qeit_get(struct ipz_queue *queue)
+{
+ return ipz_qeit_calc(queue, queue->current_q_offset);
+}
+
+/*
+ * return current Queue Page , increment Queue Page iterator from
+ * page to page in struct ipz_queue, last increment will return 0! and
+ * NOT wrap
+ * returns address (kv) of Queue Page
+ * warning don't use in parallel with ipz_QE_get_inc()
+ */
+void *ipz_qpageit_get_inc(struct ipz_queue *queue);
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ */
+static inline void *ipz_qeit_get_inc(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ queue->current_q_offset += queue->qe_size;
+ if (queue->current_q_offset >= queue->queue_length) {
+ queue->current_q_offset = 0;
+ /* toggle the valid flag */
+ queue->toggle_state = (~queue->toggle_state) & 1;
+ }
+
+ return ret;
+}
+
+/*
+ * return a bool indicating whether current Queue Entry is valid
+ */
+static inline int ipz_qeit_is_valid(struct ipz_queue *queue)
+{
+ struct ehca_cqe *cqe = ipz_qeit_get(queue);
+ return ((cqe->cqe_flags >> 7) == (queue->toggle_state & 1));
+}
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * returns 0 and does not increment, if wrong valid state
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ */
+static inline void *ipz_qeit_get_inc_valid(struct ipz_queue *queue)
+{
+ return ipz_qeit_is_valid(queue) ? ipz_qeit_get_inc(queue) : NULL;
+}
+
+/*
+ * returns and resets Queue Entry iterator
+ * returns address (kv) of first Queue Entry
+ */
+static inline void *ipz_qeit_reset(struct ipz_queue *queue)
+{
+ queue->current_q_offset = 0;
+ return ipz_qeit_get(queue);
+}
+
+/*
+ * return the q_offset corresponding to an absolute address
+ */
+int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset);
+
+/*
+ * return the next queue offset. don't modify the queue.
+ */
+static inline u64 ipz_queue_advance_offset(struct ipz_queue *queue, u64 offset)
+{
+ offset += queue->qe_size;
+ if (offset >= queue->queue_length) offset = 0;
+ return offset;
+}
+
+/* struct generic page table */
+struct ipz_pt {
+ u64 entries[EHCA_PT_ENTRIES];
+};
+
+/* struct page table for a queue, only to be used in pf */
+struct ipz_qpt {
+ /* queue page tables (kv), use u64 because we know the element length */
+ u64 *qpts;
+ u32 n_qpts;
+ u32 n_ptes; /* number of page table entries */
+ u64 *current_pte_addr;
+};
+
+/*
+ * constructor for a ipz_queue_t, placement new for ipz_queue_t,
+ * new for all dependent datastructors
+ * all QP Tables are the same
+ * flow:
+ * allocate+pin queue
+ * see ipz_qpt_ctor()
+ * returns true if ok, false if out of memory
+ */
+int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
+ const u32 nr_of_pages, const u32 pagesize,
+ const u32 qe_size, const u32 nr_of_sg,
+ int is_small);
+
+/*
+ * destructor for a ipz_queue_t
+ * -# free queue
+ * see ipz_queue_ctor()
+ * returns true if ok, false if queue was NULL-ptr of free failed
+ */
+int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue);
+
+/*
+ * constructor for a ipz_qpt_t,
+ * placement new for struct ipz_queue, new for all dependent datastructors
+ * all QP Tables are the same,
+ * flow:
+ * -# allocate+pin queue
+ * -# initialise ptcb
+ * -# allocate+pin PTs
+ * -# link PTs to a ring, according to HCA Arch, set bit62 id needed
+ * -# the ring must have room for exactly nr_of_PTEs
+ * see ipz_qpt_ctor()
+ */
+void ipz_qpt_ctor(struct ipz_qpt *qpt,
+ const u32 nr_of_qes,
+ const u32 pagesize,
+ const u32 qe_size,
+ const u8 lowbyte, const u8 toggle,
+ u32 * act_nr_of_QEs, u32 * act_nr_of_pages);
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ * fix EQ page problems
+ */
+void *ipz_qeit_eq_get_inc(struct ipz_queue *queue);
+
+/*
+ * return current Event Queue Entry, increment Queue Entry iterator
+ * by one step in struct ipz_queue if valid, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * returns 0 and does not increment, if wrong valid state
+ * warning don't use in parallel with ipz_queue_QPageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ */
+static inline void *ipz_eqit_eq_get_inc_valid(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ u32 qe = *(u8 *)ret;
+ if ((qe >> 7) != (queue->toggle_state & 1))
+ return NULL;
+ ipz_qeit_eq_get_inc(queue); /* this is a good one */
+ return ret;
+}
+
+static inline void *ipz_eqit_eq_peek_valid(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ u32 qe = *(u8 *)ret;
+ if ((qe >> 7) != (queue->toggle_state & 1))
+ return NULL;
+ return ret;
+}
+
+/* returns address (GX) of first queue entry */
+static inline u64 ipz_qpt_get_firstpage(struct ipz_qpt *qpt)
+{
+ return be64_to_cpu(qpt->qpts[0]);
+}
+
+/* returns address (kv) of first page of queue page table */
+static inline void *ipz_qpt_get_qpt(struct ipz_qpt *qpt)
+{
+ return qpt->qpts;
+}
+
+#endif /* __IPZ_PT_FN_H__ */
if (sleep_ok) {
mutex_lock(&ppd->hls_lock);
} else {
- while (mutex_trylock(&ppd->hls_lock) == EBUSY)
+ while (!mutex_trylock(&ppd->hls_lock))
udelay(1);
}
if (sleep_ok) {
mutex_lock(&dd->pport->hls_lock);
} else {
- while (mutex_trylock(&dd->pport->hls_lock) == EBUSY)
+ while (!mutex_trylock(&dd->pport->hls_lock))
udelay(1);
}
#include "device.h"
static struct class *class;
+static struct class *user_class;
static dev_t hfi1_dev;
int hfi1_cdev_init(int minor, const char *name,
const struct file_operations *fops,
- struct cdev *cdev, struct device **devp)
+ struct cdev *cdev, struct device **devp,
+ bool user_accessible)
{
const dev_t dev = MKDEV(MAJOR(hfi1_dev), minor);
struct device *device = NULL;
goto done;
}
- device = device_create(class, NULL, dev, NULL, "%s", name);
+ if (user_accessible)
+ device = device_create(user_class, NULL, dev, NULL, "%s", name);
+ else
+ device = device_create(class, NULL, dev, NULL, "%s", name);
+
if (!IS_ERR(device))
goto done;
ret = PTR_ERR(device);
return hfi1_class_name;
}
+static char *hfi1_devnode(struct device *dev, umode_t *mode)
+{
+ if (mode)
+ *mode = 0600;
+ return kasprintf(GFP_KERNEL, "%s", dev_name(dev));
+}
+
+static const char *hfi1_class_name_user = "hfi1_user";
+const char *class_name_user(void)
+{
+ return hfi1_class_name_user;
+}
+
+static char *hfi1_user_devnode(struct device *dev, umode_t *mode)
+{
+ if (mode)
+ *mode = 0666;
+ return kasprintf(GFP_KERNEL, "%s", dev_name(dev));
+}
+
int __init dev_init(void)
{
int ret;
ret = PTR_ERR(class);
pr_err("Could not create device class (err %d)\n", -ret);
unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);
+ goto done;
}
+ class->devnode = hfi1_devnode;
+
+ user_class = class_create(THIS_MODULE, class_name_user());
+ if (IS_ERR(user_class)) {
+ ret = PTR_ERR(user_class);
+ pr_err("Could not create device class for user accessible files (err %d)\n",
+ -ret);
+ class_destroy(class);
+ class = NULL;
+ user_class = NULL;
+ unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);
+ goto done;
+ }
+ user_class->devnode = hfi1_user_devnode;
done:
return ret;
void dev_cleanup(void)
{
- if (class) {
- class_destroy(class);
- class = NULL;
- }
+ class_destroy(class);
+ class = NULL;
+
+ class_destroy(user_class);
+ user_class = NULL;
unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);
}
int hfi1_cdev_init(int minor, const char *name,
const struct file_operations *fops,
- struct cdev *cdev, struct device **devp);
+ struct cdev *cdev, struct device **devp,
+ bool user_accessible);
void hfi1_cdev_cleanup(struct cdev *cdev, struct device **devp);
const char *class_name(void);
int __init dev_init(void);
if (atomic_inc_return(&diagpkt_count) == 1) {
ret = hfi1_cdev_init(HFI1_DIAGPKT_MINOR, name,
&diagpkt_file_ops, &diagpkt_cdev,
- &diagpkt_device);
+ &diagpkt_device, false);
}
return ret;
ret = hfi1_cdev_init(HFI1_SNOOP_CAPTURE_BASE + dd->unit, name,
&snoop_file_ops,
- &dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev);
+ &dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev,
+ false);
if (ret) {
dd_dev_err(dd, "Couldn't create %s device: %d", name, ret);
case HFI1_SNOOP_IOCSETLINKSTATE_EXTRA:
memset(&link_info, 0, sizeof(link_info));
- ret = copy_from_user(&link_info,
+ if (copy_from_user(&link_info,
(struct hfi1_link_info __user *)arg,
- sizeof(link_info));
- if (ret)
- break;
+ sizeof(link_info)))
+ ret = -EFAULT;
value = link_info.port_state;
index = link_info.port_number;
case HFI1_SNOOP_IOCGETLINKSTATE_EXTRA:
if (cmd == HFI1_SNOOP_IOCGETLINKSTATE_EXTRA) {
memset(&link_info, 0, sizeof(link_info));
- ret = copy_from_user(&link_info,
+ if (copy_from_user(&link_info,
(struct hfi1_link_info __user *)arg,
- sizeof(link_info));
+ sizeof(link_info)))
+ ret = -EFAULT;
index = link_info.port_number;
} else {
ret = __get_user(index, (int __user *) arg);
ppd->link_speed_active;
link_info.link_width_active =
ppd->link_width_active;
- ret = copy_to_user(
+ if (copy_to_user(
(struct hfi1_link_info __user *)arg,
- &link_info, sizeof(link_info));
+ &link_info, sizeof(link_info)))
+ ret = -EFAULT;
} else {
ret = __put_user(value, (int __user *)arg);
}
snoop_dbg("Setting filter");
/* just copy command structure */
argp = (unsigned long *)arg;
- ret = copy_from_user(&filter_cmd, (void __user *)argp,
- sizeof(filter_cmd));
- if (ret < 0) {
- pr_alert("Error copying filter command\n");
+ if (copy_from_user(&filter_cmd, (void __user *)argp,
+ sizeof(filter_cmd))) {
+ ret = -EFAULT;
break;
}
if (filter_cmd.opcode >= HFI1_MAX_FILTERS) {
break;
}
/* copy remaining data from userspace */
- ret = copy_from_user((u8 *)filter_value,
+ if (copy_from_user((u8 *)filter_value,
(void __user *)filter_cmd.value_ptr,
- filter_cmd.length);
- if (ret < 0) {
+ filter_cmd.length)) {
kfree(filter_value);
- pr_alert("Error copying filter data\n");
+ ret = -EFAULT;
break;
}
/* Drain packets first */
struct hfi1_filedata *fd = fp->private_data;
int ret = 0;
+ memset(&cinfo, 0, sizeof(cinfo));
ret = hfi1_get_base_kinfo(uctxt, &cinfo);
if (ret < 0)
goto done;
if (atomic_inc_return(&user_count) == 1) {
ret = hfi1_cdev_init(0, class_name(), &hfi1_file_ops,
- &wildcard_cdev, &wildcard_device);
+ &wildcard_cdev, &wildcard_device,
+ true);
if (ret)
goto done;
}
snprintf(name, sizeof(name), "%s_%d", class_name(), dd->unit);
ret = hfi1_cdev_init(dd->unit + 1, name, &hfi1_file_ops,
- &dd->user_cdev, &dd->user_device);
+ &dd->user_cdev, &dd->user_device,
+ true);
if (ret)
goto done;
snprintf(name, sizeof(name),
"%s_ui%d", class_name(), dd->unit);
ret = hfi1_cdev_init(dd->unit + UI_OFFSET, name, &ui_file_ops,
- &dd->ui_cdev, &dd->ui_device);
+ &dd->ui_cdev, &dd->ui_device,
+ false);
if (ret)
goto done;
}
psi->port_states.portphysstate_portstate =
(hfi1_ibphys_portstate(ppd) << 4) | (lstate & 0xf);
psi->link_width_downgrade_tx_active =
- ppd->link_width_downgrade_tx_active;
+ cpu_to_be16(ppd->link_width_downgrade_tx_active);
psi->link_width_downgrade_rx_active =
- ppd->link_width_downgrade_rx_active;
+ cpu_to_be16(ppd->link_width_downgrade_rx_active);
if (resp_len)
*resp_len += sizeof(struct opa_port_state_info);
*/
if (!is_power_of_2(count))
return SDMA_DESCQ_CNT;
- if (count < 64 && count > 32768)
+ if (count < 64 || count > 32768)
return SDMA_DESCQ_CNT;
return count;
}
dd_dev_err(sde->dd,
"\taidx: %u amode: %u alen: %u\n",
(u8)((desc[1] & SDMA_DESC1_HEADER_INDEX_SMASK)
- >> SDMA_DESC1_HEADER_INDEX_MASK),
+ >> SDMA_DESC1_HEADER_INDEX_SHIFT),
(u8)((desc[1] & SDMA_DESC1_HEADER_MODE_SMASK)
>> SDMA_DESC1_HEADER_MODE_SHIFT),
(u8)((desc[1] & SDMA_DESC1_HEADER_DWS_SMASK)
if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG)
seq_printf(s, "\t\tahgidx: %u ahgmode: %u\n",
(u8)((desc[1] & SDMA_DESC1_HEADER_INDEX_SMASK)
- >> SDMA_DESC1_HEADER_INDEX_MASK),
+ >> SDMA_DESC1_HEADER_INDEX_SHIFT),
(u8)((desc[1] & SDMA_DESC1_HEADER_MODE_SMASK)
>> SDMA_DESC1_HEADER_MODE_SHIFT));
head = (head + 1) & sde->sdma_mask;
/*
* Bits defined in the send DMA descriptor.
*/
-#define SDMA_DESC0_FIRST_DESC_FLAG (1ULL<<63)
-#define SDMA_DESC0_LAST_DESC_FLAG (1ULL<<62)
+#define SDMA_DESC0_FIRST_DESC_FLAG (1ULL << 63)
+#define SDMA_DESC0_LAST_DESC_FLAG (1ULL << 62)
#define SDMA_DESC0_BYTE_COUNT_SHIFT 48
#define SDMA_DESC0_BYTE_COUNT_WIDTH 14
#define SDMA_DESC0_BYTE_COUNT_MASK \
- ((1ULL<<SDMA_DESC0_BYTE_COUNT_WIDTH)-1ULL)
+ ((1ULL << SDMA_DESC0_BYTE_COUNT_WIDTH) - 1)
#define SDMA_DESC0_BYTE_COUNT_SMASK \
- (SDMA_DESC0_BYTE_COUNT_MASK<<SDMA_DESC0_BYTE_COUNT_SHIFT)
+ (SDMA_DESC0_BYTE_COUNT_MASK << SDMA_DESC0_BYTE_COUNT_SHIFT)
#define SDMA_DESC0_PHY_ADDR_SHIFT 0
#define SDMA_DESC0_PHY_ADDR_WIDTH 48
#define SDMA_DESC0_PHY_ADDR_MASK \
- ((1ULL<<SDMA_DESC0_PHY_ADDR_WIDTH)-1ULL)
+ ((1ULL << SDMA_DESC0_PHY_ADDR_WIDTH) - 1)
#define SDMA_DESC0_PHY_ADDR_SMASK \
- (SDMA_DESC0_PHY_ADDR_MASK<<SDMA_DESC0_PHY_ADDR_SHIFT)
+ (SDMA_DESC0_PHY_ADDR_MASK << SDMA_DESC0_PHY_ADDR_SHIFT)
#define SDMA_DESC1_HEADER_UPDATE1_SHIFT 32
#define SDMA_DESC1_HEADER_UPDATE1_WIDTH 32
#define SDMA_DESC1_HEADER_UPDATE1_MASK \
- ((1ULL<<SDMA_DESC1_HEADER_UPDATE1_WIDTH)-1ULL)
+ ((1ULL << SDMA_DESC1_HEADER_UPDATE1_WIDTH) - 1)
#define SDMA_DESC1_HEADER_UPDATE1_SMASK \
- (SDMA_DESC1_HEADER_UPDATE1_MASK<<SDMA_DESC1_HEADER_UPDATE1_SHIFT)
+ (SDMA_DESC1_HEADER_UPDATE1_MASK << SDMA_DESC1_HEADER_UPDATE1_SHIFT)
#define SDMA_DESC1_HEADER_MODE_SHIFT 13
#define SDMA_DESC1_HEADER_MODE_WIDTH 3
#define SDMA_DESC1_HEADER_MODE_MASK \
- ((1ULL<<SDMA_DESC1_HEADER_MODE_WIDTH)-1ULL)
+ ((1ULL << SDMA_DESC1_HEADER_MODE_WIDTH) - 1)
#define SDMA_DESC1_HEADER_MODE_SMASK \
- (SDMA_DESC1_HEADER_MODE_MASK<<SDMA_DESC1_HEADER_MODE_SHIFT)
+ (SDMA_DESC1_HEADER_MODE_MASK << SDMA_DESC1_HEADER_MODE_SHIFT)
#define SDMA_DESC1_HEADER_INDEX_SHIFT 8
#define SDMA_DESC1_HEADER_INDEX_WIDTH 5
#define SDMA_DESC1_HEADER_INDEX_MASK \
- ((1ULL<<SDMA_DESC1_HEADER_INDEX_WIDTH)-1ULL)
+ ((1ULL << SDMA_DESC1_HEADER_INDEX_WIDTH) - 1)
#define SDMA_DESC1_HEADER_INDEX_SMASK \
- (SDMA_DESC1_HEADER_INDEX_MASK<<SDMA_DESC1_HEADER_INDEX_SHIFT)
+ (SDMA_DESC1_HEADER_INDEX_MASK << SDMA_DESC1_HEADER_INDEX_SHIFT)
#define SDMA_DESC1_HEADER_DWS_SHIFT 4
#define SDMA_DESC1_HEADER_DWS_WIDTH 4
#define SDMA_DESC1_HEADER_DWS_MASK \
- ((1ULL<<SDMA_DESC1_HEADER_DWS_WIDTH)-1ULL)
+ ((1ULL << SDMA_DESC1_HEADER_DWS_WIDTH) - 1)
#define SDMA_DESC1_HEADER_DWS_SMASK \
- (SDMA_DESC1_HEADER_DWS_MASK<<SDMA_DESC1_HEADER_DWS_SHIFT)
+ (SDMA_DESC1_HEADER_DWS_MASK << SDMA_DESC1_HEADER_DWS_SHIFT)
#define SDMA_DESC1_GENERATION_SHIFT 2
#define SDMA_DESC1_GENERATION_WIDTH 2
#define SDMA_DESC1_GENERATION_MASK \
- ((1ULL<<SDMA_DESC1_GENERATION_WIDTH)-1ULL)
+ ((1ULL << SDMA_DESC1_GENERATION_WIDTH) - 1)
#define SDMA_DESC1_GENERATION_SMASK \
- (SDMA_DESC1_GENERATION_MASK<<SDMA_DESC1_GENERATION_SHIFT)
-#define SDMA_DESC1_INT_REQ_FLAG (1ULL<<1)
-#define SDMA_DESC1_HEAD_TO_HOST_FLAG (1ULL<<0)
+ (SDMA_DESC1_GENERATION_MASK << SDMA_DESC1_GENERATION_SHIFT)
+#define SDMA_DESC1_INT_REQ_FLAG (1ULL << 1)
+#define SDMA_DESC1_HEAD_TO_HOST_FLAG (1ULL << 0)
enum sdma_states {
sdma_state_s00_hw_down,
struct verbs_txreq *tx;
tx = kmem_cache_alloc(dev->verbs_txreq_cache, GFP_ATOMIC);
- if (!tx)
+ if (!tx) {
/* call slow path to get the lock */
tx = __get_txreq(dev, qp);
- if (tx)
- tx->qp = qp;
+ if (IS_ERR(tx))
+ return tx;
+ }
+ tx->qp = qp;
return tx;
}
__this_cpu_write(reporting_keystroke, true);
input_report_key(virt_keyboard, KEY_DOWN, PRESSED);
input_report_key(virt_keyboard, KEY_DOWN, RELEASED);
+ input_sync(virt_keyboard);
__this_cpu_write(reporting_keystroke, false);
/* reenable preemption */
visorbus-y += periodic_work.o
ccflags-y += -Idrivers/staging/unisys/include
-ccflags-y += -Idrivers/staging/unisys/visorutil
#define POLLJIFFIES_TESTWORK 100
#define POLLJIFFIES_NORMALCHANNEL 10
+static int busreg_rc = -ENODEV; /* stores the result from bus registration */
+
static int visorbus_uevent(struct device *xdev, struct kobj_uevent_env *env);
static int visorbus_match(struct device *xdev, struct device_driver *xdrv);
static void fix_vbus_dev_info(struct visor_device *visordev);
{
int rc = 0;
+ if (busreg_rc < 0)
+ return -ENODEV; /*can't register on a nonexistent bus*/
+
drv->driver.name = drv->name;
drv->driver.bus = &visorbus_type;
drv->driver.probe = visordriver_probe_device;
if (rc < 0)
return rc;
rc = register_driver_attributes(drv);
+ if (rc < 0)
+ driver_unregister(&drv->driver);
return rc;
}
EXPORT_SYMBOL_GPL(visorbus_register_visor_driver);
static int
create_bus_type(void)
{
- int rc = 0;
-
- rc = bus_register(&visorbus_type);
- return rc;
+ busreg_rc = bus_register(&visorbus_type);
+ return busreg_rc;
}
/** Remove the one-and-only one instance of the visor bus type (visorbus_type).
spin_lock_irqsave(&devdata->priv_lock, flags);
atomic_dec(&devdata->num_rcvbuf_in_iovm);
- /* update rcv stats - call it with priv_lock held */
- devdata->net_stats.rx_packets++;
- devdata->net_stats.rx_bytes = skb->len;
-
/* set length to how much was ACTUALLY received -
* NOTE: rcv_done_len includes actual length of data rcvd
* including ethhdr
*/
skb->len = cmdrsp->net.rcv.rcv_done_len;
+ /* update rcv stats - call it with priv_lock held */
+ devdata->net_stats.rx_packets++;
+ devdata->net_stats.rx_bytes += skb->len;
+
/* test enabled while holding lock */
if (!(devdata->enabled && devdata->enab_dis_acked)) {
/* don't process it unless we're in enable mode and until
"%s debugfs_create_dir %s failed\n",
__func__, netdev->name);
err = -ENOMEM;
- goto cleanup_xmit_cmdrsp;
+ goto cleanup_register_netdev;
}
dev_info(&dev->device, "%s success netdev=%s\n",
__func__, netdev->name);
return 0;
+cleanup_register_netdev:
+ unregister_netdev(netdev);
+
cleanup_napi_add:
del_timer_sync(&devdata->irq_poll_timer);
netif_napi_del(&devdata->napi);
if (!dev_num_pool)
goto cleanup_workqueue;
- visorbus_register_visor_driver(&visornic_driver);
- return 0;
+ err = visorbus_register_visor_driver(&visornic_driver);
+ if (!err)
+ return 0;
cleanup_workqueue:
if (visornic_timeout_reset_workqueue) {
TYPERANGE_UTF8, USE_INITIAL_ONLY);
if (!param)
goto out;
+
/*
* Extra parameters for ISER from RFC-5046
*/
} else if (!strcmp(param->name, SESSIONTYPE)) {
SET_PSTATE_NEGOTIATE(param);
} else if (!strcmp(param->name, IFMARKER)) {
- SET_PSTATE_NEGOTIATE(param);
+ SET_PSTATE_REJECT(param);
} else if (!strcmp(param->name, OFMARKER)) {
- SET_PSTATE_NEGOTIATE(param);
+ SET_PSTATE_REJECT(param);
} else if (!strcmp(param->name, IFMARKINT)) {
SET_PSTATE_REJECT(param);
} else if (!strcmp(param->name, OFMARKINT)) {
struct se_session *se_sess = se_cmd->se_sess;
struct se_node_acl *nacl = se_sess->se_node_acl;
struct se_dev_entry *deve;
+ sense_reason_t ret = TCM_NO_SENSE;
rcu_read_lock();
deve = target_nacl_find_deve(nacl, unpacked_lun);
if (deve) {
atomic_long_inc(&deve->total_cmds);
- if ((se_cmd->data_direction == DMA_TO_DEVICE) &&
- (deve->lun_flags & TRANSPORT_LUNFLAGS_READ_ONLY)) {
- pr_err("TARGET_CORE[%s]: Detected WRITE_PROTECTED LUN"
- " Access for 0x%08llx\n",
- se_cmd->se_tfo->get_fabric_name(),
- unpacked_lun);
- rcu_read_unlock();
- return TCM_WRITE_PROTECTED;
- }
-
if (se_cmd->data_direction == DMA_TO_DEVICE)
atomic_long_add(se_cmd->data_length,
&deve->write_bytes);
percpu_ref_get(&se_lun->lun_ref);
se_cmd->lun_ref_active = true;
+
+ if ((se_cmd->data_direction == DMA_TO_DEVICE) &&
+ (deve->lun_flags & TRANSPORT_LUNFLAGS_READ_ONLY)) {
+ pr_err("TARGET_CORE[%s]: Detected WRITE_PROTECTED LUN"
+ " Access for 0x%08llx\n",
+ se_cmd->se_tfo->get_fabric_name(),
+ unpacked_lun);
+ rcu_read_unlock();
+ ret = TCM_WRITE_PROTECTED;
+ goto ref_dev;
+ }
}
rcu_read_unlock();
unpacked_lun);
return TCM_NON_EXISTENT_LUN;
}
- /*
- * Force WRITE PROTECT for virtual LUN 0
- */
- if ((se_cmd->data_direction != DMA_FROM_DEVICE) &&
- (se_cmd->data_direction != DMA_NONE))
- return TCM_WRITE_PROTECTED;
se_lun = se_sess->se_tpg->tpg_virt_lun0;
se_cmd->se_lun = se_sess->se_tpg->tpg_virt_lun0;
percpu_ref_get(&se_lun->lun_ref);
se_cmd->lun_ref_active = true;
+
+ /*
+ * Force WRITE PROTECT for virtual LUN 0
+ */
+ if ((se_cmd->data_direction != DMA_FROM_DEVICE) &&
+ (se_cmd->data_direction != DMA_NONE)) {
+ ret = TCM_WRITE_PROTECTED;
+ goto ref_dev;
+ }
}
/*
* RCU reference protected by percpu se_lun->lun_ref taken above that
* pointer can be kfree_rcu() by the final se_lun->lun_group put via
* target_core_fabric_configfs.c:target_fabric_port_release
*/
+ref_dev:
se_cmd->se_dev = rcu_dereference_raw(se_lun->lun_se_dev);
atomic_long_inc(&se_cmd->se_dev->num_cmds);
atomic_long_add(se_cmd->data_length,
&se_cmd->se_dev->read_bytes);
- return 0;
+ return ret;
}
EXPORT_SYMBOL(transport_lookup_cmd_lun);
hlist_del_rcu(&orig->link);
clear_bit(DEF_PR_REG_ACTIVE, &orig->deve_flags);
- rcu_assign_pointer(orig->se_lun, NULL);
- rcu_assign_pointer(orig->se_lun_acl, NULL);
orig->lun_flags = 0;
orig->creation_time = 0;
orig->attach_count--;
kref_put(&orig->pr_kref, target_pr_kref_release);
wait_for_completion(&orig->pr_comp);
+ rcu_assign_pointer(orig->se_lun, NULL);
+ rcu_assign_pointer(orig->se_lun_acl, NULL);
+
kfree_rcu(orig, rcu_head);
core_scsi3_free_pr_reg_from_nacl(dev, nacl);
bool target_sense_desc_format(struct se_device *dev)
{
- return dev->transport->get_blocks(dev) > U32_MAX;
+ return (dev) ? dev->transport->get_blocks(dev) > U32_MAX : false;
}
mode = FMODE_READ|FMODE_EXCL;
if (!ib_dev->ibd_readonly)
mode |= FMODE_WRITE;
+ else
+ dev->dev_flags |= DF_READ_ONLY;
bd = blkdev_get_by_path(ib_dev->ibd_udev_path, mode, ib_dev);
if (IS_ERR(bd)) {
struct se_device *dev,
struct se_node_acl *nacl,
struct se_lun *lun,
- struct se_dev_entry *deve,
+ struct se_dev_entry *dest_deve,
u64 mapped_lun,
unsigned char *isid,
u64 sa_res_key,
INIT_LIST_HEAD(&pr_reg->pr_reg_atp_mem_list);
atomic_set(&pr_reg->pr_res_holders, 0);
pr_reg->pr_reg_nacl = nacl;
- pr_reg->pr_reg_deve = deve;
+ /*
+ * For destination registrations for ALL_TG_PT=1 and SPEC_I_PT=1,
+ * the se_dev_entry->pr_ref will have been already obtained by
+ * core_get_se_deve_from_rtpi() or __core_scsi3_alloc_registration().
+ *
+ * Otherwise, locate se_dev_entry now and obtain a reference until
+ * registration completes in __core_scsi3_add_registration().
+ */
+ if (dest_deve) {
+ pr_reg->pr_reg_deve = dest_deve;
+ } else {
+ rcu_read_lock();
+ pr_reg->pr_reg_deve = target_nacl_find_deve(nacl, mapped_lun);
+ if (!pr_reg->pr_reg_deve) {
+ rcu_read_unlock();
+ pr_err("Unable to locate PR deve %s mapped_lun: %llu\n",
+ nacl->initiatorname, mapped_lun);
+ kmem_cache_free(t10_pr_reg_cache, pr_reg);
+ return NULL;
+ }
+ kref_get(&pr_reg->pr_reg_deve->pr_kref);
+ rcu_read_unlock();
+ }
pr_reg->pr_res_mapped_lun = mapped_lun;
pr_reg->pr_aptpl_target_lun = lun->unpacked_lun;
pr_reg->tg_pt_sep_rtpi = lun->lun_rtpi;
!(strcmp(pr_reg->pr_tport, t_port)) &&
(pr_reg->pr_reg_tpgt == tpgt) &&
(pr_reg->pr_aptpl_target_lun == target_lun)) {
+ /*
+ * Obtain the ->pr_reg_deve pointer + reference, that
+ * is released by __core_scsi3_add_registration() below.
+ */
+ rcu_read_lock();
+ pr_reg->pr_reg_deve = target_nacl_find_deve(nacl, mapped_lun);
+ if (!pr_reg->pr_reg_deve) {
+ pr_err("Unable to locate PR APTPL %s mapped_lun:"
+ " %llu\n", nacl->initiatorname, mapped_lun);
+ rcu_read_unlock();
+ continue;
+ }
+ kref_get(&pr_reg->pr_reg_deve->pr_kref);
+ rcu_read_unlock();
pr_reg->pr_reg_nacl = nacl;
pr_reg->tg_pt_sep_rtpi = lun->lun_rtpi;
-
list_del(&pr_reg->pr_reg_aptpl_list);
spin_unlock(&pr_tmpl->aptpl_reg_lock);
/*
* At this point all of the pointers in *pr_reg will
* be setup, so go ahead and add the registration.
*/
-
__core_scsi3_add_registration(dev, nacl, pr_reg, 0, 0);
/*
* If this registration is the reservation holder,
__core_scsi3_dump_registration(tfo, dev, nacl, pr_reg, register_type);
spin_unlock(&pr_tmpl->registration_lock);
-
- rcu_read_lock();
- deve = pr_reg->pr_reg_deve;
- if (deve)
- set_bit(DEF_PR_REG_ACTIVE, &deve->deve_flags);
- rcu_read_unlock();
-
/*
* Skip extra processing for ALL_TG_PT=0 or REGISTER_AND_MOVE.
*/
if (!pr_reg->pr_reg_all_tg_pt || register_move)
- return;
+ goto out;
/*
* Walk pr_reg->pr_reg_atp_list and add registrations for ALL_TG_PT=1
* allocated in __core_scsi3_alloc_registration()
__core_scsi3_dump_registration(tfo, dev, nacl_tmp, pr_reg_tmp,
register_type);
spin_unlock(&pr_tmpl->registration_lock);
-
+ /*
+ * Drop configfs group dependency reference and deve->pr_kref
+ * obtained from __core_scsi3_alloc_registration() code.
+ */
rcu_read_lock();
deve = pr_reg_tmp->pr_reg_deve;
- if (deve)
+ if (deve) {
set_bit(DEF_PR_REG_ACTIVE, &deve->deve_flags);
+ core_scsi3_lunacl_undepend_item(deve);
+ pr_reg_tmp->pr_reg_deve = NULL;
+ }
rcu_read_unlock();
-
- /*
- * Drop configfs group dependency reference from
- * __core_scsi3_alloc_registration()
- */
- core_scsi3_lunacl_undepend_item(pr_reg_tmp->pr_reg_deve);
}
+out:
+ /*
+ * Drop deve->pr_kref obtained in __core_scsi3_do_alloc_registration()
+ */
+ rcu_read_lock();
+ deve = pr_reg->pr_reg_deve;
+ if (deve) {
+ set_bit(DEF_PR_REG_ACTIVE, &deve->deve_flags);
+ kref_put(&deve->pr_kref, target_pr_kref_release);
+ pr_reg->pr_reg_deve = NULL;
+ }
+ rcu_read_unlock();
}
static int core_scsi3_alloc_registration(
dest_node_acl->initiatorname, i_buf, (dest_se_deve) ?
dest_se_deve->mapped_lun : 0);
- if (!dest_se_deve)
+ if (!dest_se_deve) {
+ kref_put(&local_pr_reg->pr_reg_deve->pr_kref,
+ target_pr_kref_release);
continue;
-
+ }
core_scsi3_lunacl_undepend_item(dest_se_deve);
core_scsi3_nodeacl_undepend_item(dest_node_acl);
core_scsi3_tpg_undepend_item(dest_tpg);
kmem_cache_free(t10_pr_reg_cache, dest_pr_reg);
- if (!dest_se_deve)
+ if (!dest_se_deve) {
+ kref_put(&local_pr_reg->pr_reg_deve->pr_kref,
+ target_pr_kref_release);
continue;
-
+ }
core_scsi3_lunacl_undepend_item(dest_se_deve);
core_scsi3_nodeacl_undepend_item(dest_node_acl);
core_scsi3_tpg_undepend_item(dest_tpg);
list_add_tail(&lun->lun_dev_link, &dev->dev_sep_list);
spin_unlock(&dev->se_port_lock);
- lun->lun_access = lun_access;
+ if (dev->dev_flags & DF_READ_ONLY)
+ lun->lun_access = TRANSPORT_LUNFLAGS_READ_ONLY;
+ else
+ lun->lun_access = lun_access;
if (!(dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE))
hlist_add_head_rcu(&lun->link, &tpg->tpg_lun_hlist);
mutex_unlock(&tpg->tpg_lun_mutex);
config HISI_THERMAL
tristate "Hisilicon thermal driver"
- depends on ARCH_HISI && CPU_THERMAL && OF
+ depends on (ARCH_HISI && CPU_THERMAL && OF) || COMPILE_TEST
help
Enable this to plug hisilicon's thermal sensor driver into the Linux
thermal framework. cpufreq is used as the cooling device to throttle
config SPEAR_THERMAL
bool "SPEAr thermal sensor driver"
- depends on PLAT_SPEAR
+ depends on PLAT_SPEAR || COMPILE_TEST
depends on OF
help
Enable this to plug the SPEAr thermal sensor driver into the Linux
config ROCKCHIP_THERMAL
tristate "Rockchip thermal driver"
- depends on ARCH_ROCKCHIP
+ depends on ARCH_ROCKCHIP || COMPILE_TEST
depends on RESET_CONTROLLER
help
Rockchip thermal driver provides support for Temperature sensor
config KIRKWOOD_THERMAL
tristate "Temperature sensor on Marvell Kirkwood SoCs"
- depends on MACH_KIRKWOOD
+ depends on MACH_KIRKWOOD || COMPILE_TEST
depends on OF
help
Support for the Kirkwood thermal sensor driver into the Linux thermal
config DOVE_THERMAL
tristate "Temperature sensor on Marvell Dove SoCs"
- depends on ARCH_DOVE || MACH_DOVE
+ depends on ARCH_DOVE || MACH_DOVE || COMPILE_TEST
depends on OF
help
Support for the Dove thermal sensor driver in the Linux thermal
config ARMADA_THERMAL
tristate "Armada 370/XP thermal management"
- depends on ARCH_MVEBU
+ depends on ARCH_MVEBU || COMPILE_TEST
depends on OF
help
Enable this option if you want to have support for thermal management
programmable trip points and other information.
menu "Texas Instruments thermal drivers"
+depends on ARCH_HAS_BANDGAP || COMPILE_TEST
source "drivers/thermal/ti-soc-thermal/Kconfig"
endmenu
menu "Samsung thermal drivers"
-depends on ARCH_EXYNOS
+depends on ARCH_EXYNOS || COMPILE_TEST
source "drivers/thermal/samsung/Kconfig"
endmenu
config QCOM_SPMI_TEMP_ALARM
tristate "Qualcomm SPMI PMIC Temperature Alarm"
- depends on OF && SPMI && IIO
+ depends on OF && (SPMI || COMPILE_TEST) && IIO
select REGMAP_SPMI
help
This enables a thermal sysfs driver for Qualcomm plug-and-play (QPNP)
* efficiently. Power is stored in mW, frequency in KHz. The
* resulting table is in ascending order.
*
- * Return: 0 on success, -E* on error.
+ * Return: 0 on success, -EINVAL if there are no OPPs for any CPUs,
+ * -ENOMEM if we run out of memory or -EAGAIN if an OPP was
+ * added/enabled while the function was executing.
*/
static int build_dyn_power_table(struct cpufreq_cooling_device *cpufreq_device,
u32 capacitance)
int num_opps = 0, cpu, i, ret = 0;
unsigned long freq;
- rcu_read_lock();
-
for_each_cpu(cpu, &cpufreq_device->allowed_cpus) {
dev = get_cpu_device(cpu);
if (!dev) {
}
num_opps = dev_pm_opp_get_opp_count(dev);
- if (num_opps > 0) {
+ if (num_opps > 0)
break;
- } else if (num_opps < 0) {
- ret = num_opps;
- goto unlock;
- }
+ else if (num_opps < 0)
+ return num_opps;
}
- if (num_opps == 0) {
- ret = -EINVAL;
- goto unlock;
- }
+ if (num_opps == 0)
+ return -EINVAL;
power_table = kcalloc(num_opps, sizeof(*power_table), GFP_KERNEL);
- if (!power_table) {
- ret = -ENOMEM;
- goto unlock;
- }
+ if (!power_table)
+ return -ENOMEM;
+
+ rcu_read_lock();
for (freq = 0, i = 0;
opp = dev_pm_opp_find_freq_ceil(dev, &freq), !IS_ERR(opp);
u32 freq_mhz, voltage_mv;
u64 power;
+ if (i >= num_opps) {
+ rcu_read_unlock();
+ ret = -EAGAIN;
+ goto free_power_table;
+ }
+
freq_mhz = freq / 1000000;
voltage_mv = dev_pm_opp_get_voltage(opp) / 1000;
power_table[i].power = power;
}
- if (i == 0) {
+ rcu_read_unlock();
+
+ if (i != num_opps) {
ret = PTR_ERR(opp);
- goto unlock;
+ goto free_power_table;
}
cpufreq_device->cpu_dev = dev;
cpufreq_device->dyn_power_table = power_table;
cpufreq_device->dyn_power_table_entries = i;
-unlock:
- rcu_read_unlock();
+ return 0;
+
+free_power_table:
+ kfree(power_table);
+
return ret;
}
ret = get_idr(&cpufreq_idr, &cpufreq_dev->id);
if (ret) {
cool_dev = ERR_PTR(ret);
- goto free_table;
+ goto free_power_table;
}
snprintf(dev_name, sizeof(dev_name), "thermal-cpufreq-%d",
remove_idr:
release_idr(&cpufreq_idr, cpufreq_dev->id);
+free_power_table:
+ kfree(cpufreq_dev->dyn_power_table);
free_table:
kfree(cpufreq_dev->freq_table);
free_time_in_idle_timestamp:
thermal_cooling_device_unregister(cpufreq_dev->cool_dev);
release_idr(&cpufreq_idr, cpufreq_dev->id);
+ kfree(cpufreq_dev->dyn_power_table);
kfree(cpufreq_dev->time_in_idle_timestamp);
kfree(cpufreq_dev->time_in_idle);
kfree(cpufreq_dev->freq_table);
{ .compatible = "stericsson,db8500-cpufreq-cooling" },
{},
};
+MODULE_DEVICE_TABLE(of, db8500_cpufreq_cooling_match);
#endif
static struct platform_driver db8500_cpufreq_cooling_driver = {
#include "thermal_core.h"
+#define INVALID_TRIP -1
+
#define FRAC_BITS 10
#define int_to_frac(x) ((x) << FRAC_BITS)
#define frac_to_int(x) ((x) >> FRAC_BITS)
/**
* struct power_allocator_params - parameters for the power allocator governor
+ * @allocated_tzp: whether we have allocated tzp for this thermal zone and
+ * it needs to be freed on unbind
* @err_integral: accumulated error in the PID controller.
* @prev_err: error in the previous iteration of the PID controller.
* Used to calculate the derivative term.
* @trip_switch_on: first passive trip point of the thermal zone. The
* governor switches on when this trip point is crossed.
+ * If the thermal zone only has one passive trip point,
+ * @trip_switch_on should be INVALID_TRIP.
* @trip_max_desired_temperature: last passive trip point of the thermal
* zone. The temperature we are
* controlling for.
*/
struct power_allocator_params {
+ bool allocated_tzp;
s64 err_integral;
s32 prev_err;
int trip_switch_on;
int trip_max_desired_temperature;
};
+/**
+ * estimate_sustainable_power() - Estimate the sustainable power of a thermal zone
+ * @tz: thermal zone we are operating in
+ *
+ * For thermal zones that don't provide a sustainable_power in their
+ * thermal_zone_params, estimate one. Calculate it using the minimum
+ * power of all the cooling devices as that gives a valid value that
+ * can give some degree of functionality. For optimal performance of
+ * this governor, provide a sustainable_power in the thermal zone's
+ * thermal_zone_params.
+ */
+static u32 estimate_sustainable_power(struct thermal_zone_device *tz)
+{
+ u32 sustainable_power = 0;
+ struct thermal_instance *instance;
+ struct power_allocator_params *params = tz->governor_data;
+
+ list_for_each_entry(instance, &tz->thermal_instances, tz_node) {
+ struct thermal_cooling_device *cdev = instance->cdev;
+ u32 min_power;
+
+ if (instance->trip != params->trip_max_desired_temperature)
+ continue;
+
+ if (power_actor_get_min_power(cdev, tz, &min_power))
+ continue;
+
+ sustainable_power += min_power;
+ }
+
+ return sustainable_power;
+}
+
+/**
+ * estimate_pid_constants() - Estimate the constants for the PID controller
+ * @tz: thermal zone for which to estimate the constants
+ * @sustainable_power: sustainable power for the thermal zone
+ * @trip_switch_on: trip point number for the switch on temperature
+ * @control_temp: target temperature for the power allocator governor
+ * @force: whether to force the update of the constants
+ *
+ * This function is used to update the estimation of the PID
+ * controller constants in struct thermal_zone_parameters.
+ * Sustainable power is provided in case it was estimated. The
+ * estimated sustainable_power should not be stored in the
+ * thermal_zone_parameters so it has to be passed explicitly to this
+ * function.
+ *
+ * If @force is not set, the values in the thermal zone's parameters
+ * are preserved if they are not zero. If @force is set, the values
+ * in thermal zone's parameters are overwritten.
+ */
+static void estimate_pid_constants(struct thermal_zone_device *tz,
+ u32 sustainable_power, int trip_switch_on,
+ int control_temp, bool force)
+{
+ int ret;
+ int switch_on_temp;
+ u32 temperature_threshold;
+
+ ret = tz->ops->get_trip_temp(tz, trip_switch_on, &switch_on_temp);
+ if (ret)
+ switch_on_temp = 0;
+
+ temperature_threshold = control_temp - switch_on_temp;
+ /*
+ * estimate_pid_constants() tries to find appropriate default
+ * values for thermal zones that don't provide them. If a
+ * system integrator has configured a thermal zone with two
+ * passive trip points at the same temperature, that person
+ * hasn't put any effort to set up the thermal zone properly
+ * so just give up.
+ */
+ if (!temperature_threshold)
+ return;
+
+ if (!tz->tzp->k_po || force)
+ tz->tzp->k_po = int_to_frac(sustainable_power) /
+ temperature_threshold;
+
+ if (!tz->tzp->k_pu || force)
+ tz->tzp->k_pu = int_to_frac(2 * sustainable_power) /
+ temperature_threshold;
+
+ if (!tz->tzp->k_i || force)
+ tz->tzp->k_i = int_to_frac(10) / 1000;
+ /*
+ * The default for k_d and integral_cutoff is 0, so we can
+ * leave them as they are.
+ */
+}
+
/**
* pid_controller() - PID controller
* @tz: thermal zone we are operating in
{
s64 p, i, d, power_range;
s32 err, max_power_frac;
+ u32 sustainable_power;
struct power_allocator_params *params = tz->governor_data;
max_power_frac = int_to_frac(max_allocatable_power);
+ if (tz->tzp->sustainable_power) {
+ sustainable_power = tz->tzp->sustainable_power;
+ } else {
+ sustainable_power = estimate_sustainable_power(tz);
+ estimate_pid_constants(tz, sustainable_power,
+ params->trip_switch_on, control_temp,
+ true);
+ }
+
err = control_temp - current_temp;
err = int_to_frac(err);
power_range = p + i + d;
/* feed-forward the known sustainable dissipatable power */
- power_range = tz->tzp->sustainable_power + frac_to_int(power_range);
+ power_range = sustainable_power + frac_to_int(power_range);
power_range = clamp(power_range, (s64)0, (s64)max_allocatable_power);
}
}
+ if (!num_actors) {
+ ret = -ENODEV;
+ goto unlock;
+ }
+
/*
* We need to allocate five arrays of the same size:
* req_power, max_power, granted_power, extra_actor_power and
return ret;
}
-static int get_governor_trips(struct thermal_zone_device *tz,
- struct power_allocator_params *params)
+/**
+ * get_governor_trips() - get the number of the two trip points that are key for this governor
+ * @tz: thermal zone to operate on
+ * @params: pointer to private data for this governor
+ *
+ * The power allocator governor works optimally with two trips points:
+ * a "switch on" trip point and a "maximum desired temperature". These
+ * are defined as the first and last passive trip points.
+ *
+ * If there is only one trip point, then that's considered to be the
+ * "maximum desired temperature" trip point and the governor is always
+ * on. If there are no passive or active trip points, then the
+ * governor won't do anything. In fact, its throttle function
+ * won't be called at all.
+ */
+static void get_governor_trips(struct thermal_zone_device *tz,
+ struct power_allocator_params *params)
{
- int i, ret, last_passive;
+ int i, last_active, last_passive;
bool found_first_passive;
found_first_passive = false;
- last_passive = -1;
- ret = -EINVAL;
+ last_active = INVALID_TRIP;
+ last_passive = INVALID_TRIP;
for (i = 0; i < tz->trips; i++) {
enum thermal_trip_type type;
+ int ret;
ret = tz->ops->get_trip_type(tz, i, &type);
- if (ret)
- return ret;
+ if (ret) {
+ dev_warn(&tz->device,
+ "Failed to get trip point %d type: %d\n", i,
+ ret);
+ continue;
+ }
- if (!found_first_passive) {
- if (type == THERMAL_TRIP_PASSIVE) {
+ if (type == THERMAL_TRIP_PASSIVE) {
+ if (!found_first_passive) {
params->trip_switch_on = i;
found_first_passive = true;
+ } else {
+ last_passive = i;
}
- } else if (type == THERMAL_TRIP_PASSIVE) {
- last_passive = i;
+ } else if (type == THERMAL_TRIP_ACTIVE) {
+ last_active = i;
} else {
break;
}
}
- if (last_passive != -1) {
+ if (last_passive != INVALID_TRIP) {
params->trip_max_desired_temperature = last_passive;
- ret = 0;
+ } else if (found_first_passive) {
+ params->trip_max_desired_temperature = params->trip_switch_on;
+ params->trip_switch_on = INVALID_TRIP;
} else {
- ret = -EINVAL;
+ params->trip_switch_on = INVALID_TRIP;
+ params->trip_max_desired_temperature = last_active;
}
-
- return ret;
}
static void reset_pid_controller(struct power_allocator_params *params)
* power_allocator_bind() - bind the power_allocator governor to a thermal zone
* @tz: thermal zone to bind it to
*
- * Check that the thermal zone is valid for this governor, that is, it
- * has two thermal trips. If so, initialize the PID controller
- * parameters and bind it to the thermal zone.
+ * Initialize the PID controller parameters and bind it to the thermal
+ * zone.
*
- * Return: 0 on success, -EINVAL if the trips were invalid or -ENOMEM
- * if we ran out of memory.
+ * Return: 0 on success, or -ENOMEM if we ran out of memory.
*/
static int power_allocator_bind(struct thermal_zone_device *tz)
{
int ret;
struct power_allocator_params *params;
- int switch_on_temp, control_temp;
- u32 temperature_threshold;
-
- if (!tz->tzp || !tz->tzp->sustainable_power) {
- dev_err(&tz->device,
- "power_allocator: missing sustainable_power\n");
- return -EINVAL;
- }
+ int control_temp;
params = kzalloc(sizeof(*params), GFP_KERNEL);
if (!params)
return -ENOMEM;
- ret = get_governor_trips(tz, params);
- if (ret) {
- dev_err(&tz->device,
- "thermal zone %s has wrong trip setup for power allocator\n",
- tz->type);
- goto free;
- }
+ if (!tz->tzp) {
+ tz->tzp = kzalloc(sizeof(*tz->tzp), GFP_KERNEL);
+ if (!tz->tzp) {
+ ret = -ENOMEM;
+ goto free_params;
+ }
- ret = tz->ops->get_trip_temp(tz, params->trip_switch_on,
- &switch_on_temp);
- if (ret)
- goto free;
+ params->allocated_tzp = true;
+ }
- ret = tz->ops->get_trip_temp(tz, params->trip_max_desired_temperature,
- &control_temp);
- if (ret)
- goto free;
+ if (!tz->tzp->sustainable_power)
+ dev_warn(&tz->device, "power_allocator: sustainable_power will be estimated\n");
- temperature_threshold = control_temp - switch_on_temp;
+ get_governor_trips(tz, params);
- tz->tzp->k_po = tz->tzp->k_po ?:
- int_to_frac(tz->tzp->sustainable_power) / temperature_threshold;
- tz->tzp->k_pu = tz->tzp->k_pu ?:
- int_to_frac(2 * tz->tzp->sustainable_power) /
- temperature_threshold;
- tz->tzp->k_i = tz->tzp->k_i ?: int_to_frac(10) / 1000;
- /*
- * The default for k_d and integral_cutoff is 0, so we can
- * leave them as they are.
- */
+ if (tz->trips > 0) {
+ ret = tz->ops->get_trip_temp(tz,
+ params->trip_max_desired_temperature,
+ &control_temp);
+ if (!ret)
+ estimate_pid_constants(tz, tz->tzp->sustainable_power,
+ params->trip_switch_on,
+ control_temp, false);
+ }
reset_pid_controller(params);
return 0;
-free:
+free_params:
kfree(params);
+
return ret;
}
static void power_allocator_unbind(struct thermal_zone_device *tz)
{
+ struct power_allocator_params *params = tz->governor_data;
+
dev_dbg(&tz->device, "Unbinding from thermal zone %d\n", tz->id);
+
+ if (params->allocated_tzp) {
+ kfree(tz->tzp);
+ tz->tzp = NULL;
+ }
+
kfree(tz->governor_data);
tz->governor_data = NULL;
}
ret = tz->ops->get_trip_temp(tz, params->trip_switch_on,
&switch_on_temp);
- if (ret) {
- dev_warn(&tz->device,
- "Failed to get switch on temperature: %d\n", ret);
- return ret;
- }
-
- if (current_temp < switch_on_temp) {
+ if (!ret && (current_temp < switch_on_temp)) {
tz->passive = 0;
reset_pid_controller(params);
allow_maximum_power(tz);
return cdev->ops->state2power(cdev, tz, 0, max_power);
}
+/**
+ * power_actor_get_min_power() - get the mainimum power that a cdev can consume
+ * @cdev: pointer to &thermal_cooling_device
+ * @tz: a valid thermal zone device pointer
+ * @min_power: pointer in which to store the minimum power
+ *
+ * Calculate the minimum power consumption in milliwatts that the
+ * cooling device can currently consume and store it in @min_power.
+ *
+ * Return: 0 on success, -EINVAL if @cdev doesn't support the
+ * power_actor API or -E* on other error.
+ */
+int power_actor_get_min_power(struct thermal_cooling_device *cdev,
+ struct thermal_zone_device *tz, u32 *min_power)
+{
+ unsigned long max_state;
+ int ret;
+
+ if (!cdev_is_power_actor(cdev))
+ return -EINVAL;
+
+ ret = cdev->ops->get_max_state(cdev, &max_state);
+ if (ret)
+ return ret;
+
+ return cdev->ops->state2power(cdev, tz, max_state, min_power);
+}
+
/**
* power_actor_set_power() - limit the maximum power that a cooling device can consume
* @cdev: pointer to &thermal_cooling_device
config TI_SOC_THERMAL
tristate "Texas Instruments SoCs temperature sensor driver"
- depends on THERMAL
- depends on ARCH_HAS_BANDGAP
help
If you say yes here you get support for the Texas Instruments
OMAP4460+ on die bandgap temperature sensor support. The register
config OMAP4_THERMAL
bool "Texas Instruments OMAP4 thermal support"
depends on TI_SOC_THERMAL
- depends on ARCH_OMAP4
+ depends on ARCH_OMAP4 || COMPILE_TEST
help
If you say yes here you get thermal support for the Texas Instruments
OMAP4 SoC family. The current chip supported are:
config OMAP5_THERMAL
bool "Texas Instruments OMAP5 thermal support"
depends on TI_SOC_THERMAL
- depends on SOC_OMAP5
+ depends on SOC_OMAP5 || COMPILE_TEST
help
If you say yes here you get thermal support for the Texas Instruments
OMAP5 SoC family. The current chip supported are:
config DRA752_THERMAL
bool "Texas Instruments DRA752 thermal support"
depends on TI_SOC_THERMAL
- depends on SOC_DRA7XX
+ depends on SOC_DRA7XX || COMPILE_TEST
help
If you say yes here you get thermal support for the Texas Instruments
DRA752 SoC family. The current chip supported are:
{
.class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0,
.vendor = PCI_VENDOR_ID_INTEL, .device = 0x156c,
- .subvendor = 0x2222, .subdevice = 0x1111,
+ .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID,
},
{ 0,}
};
spin_lock_irqsave(&tty->ctrl_lock, flags);
tty->ctrl_status |= TIOCPKT_FLUSHREAD;
spin_unlock_irqrestore(&tty->ctrl_lock, flags);
- if (waitqueue_active(&tty->link->read_wait))
- wake_up_interruptible(&tty->link->read_wait);
+ wake_up_interruptible(&tty->link->read_wait);
}
}
put_tty_queue(c, ldata);
smp_store_release(&ldata->canon_head, ldata->read_head);
kill_fasync(&tty->fasync, SIGIO, POLL_IN);
- if (waitqueue_active(&tty->read_wait))
- wake_up_interruptible_poll(&tty->read_wait, POLLIN);
+ wake_up_interruptible_poll(&tty->read_wait, POLLIN);
return 0;
}
}
if ((read_cnt(ldata) >= ldata->minimum_to_wake) || L_EXTPROC(tty)) {
kill_fasync(&tty->fasync, SIGIO, POLL_IN);
- if (waitqueue_active(&tty->read_wait))
- wake_up_interruptible_poll(&tty->read_wait, POLLIN);
+ wake_up_interruptible_poll(&tty->read_wait, POLLIN);
}
}
}
/* The termios change make the tty ready for I/O */
- if (waitqueue_active(&tty->write_wait))
- wake_up_interruptible(&tty->write_wait);
- if (waitqueue_active(&tty->read_wait))
- wake_up_interruptible(&tty->read_wait);
+ wake_up_interruptible(&tty->write_wait);
+ wake_up_interruptible(&tty->read_wait);
}
/**
UART_FCR7_64BYTE,
.flags = UART_CAP_FIFO,
},
+ [PORT_RT2880] = {
+ .name = "Palmchip BK-3103",
+ .fifo_size = 16,
+ .tx_loadsz = 16,
+ .fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
+ .rxtrig_bytes = {1, 4, 8, 14},
+ .flags = UART_CAP_FIFO,
+ },
};
/* Uart divisor latch read */
}
#endif /* CONFIG_SERIAL_8250_CONSOLE */
+
+MODULE_LICENSE("GPL");
ret = atmel_init_gpios(port, &pdev->dev);
if (ret < 0) {
dev_err(&pdev->dev, "Failed to initialize GPIOs.");
- goto err;
+ goto err_clear_bit;
}
ret = atmel_init_port(port, pdev);
int locked = 1;
int retval;
- retval = clk_prepare_enable(sport->clk_per);
+ retval = clk_enable(sport->clk_per);
if (retval)
return;
- retval = clk_prepare_enable(sport->clk_ipg);
+ retval = clk_enable(sport->clk_ipg);
if (retval) {
- clk_disable_unprepare(sport->clk_per);
+ clk_disable(sport->clk_per);
return;
}
if (locked)
spin_unlock_irqrestore(&sport->port.lock, flags);
- clk_disable_unprepare(sport->clk_ipg);
- clk_disable_unprepare(sport->clk_per);
+ clk_disable(sport->clk_ipg);
+ clk_disable(sport->clk_per);
}
/*
retval = uart_set_options(&sport->port, co, baud, parity, bits, flow);
- clk_disable_unprepare(sport->clk_ipg);
+ clk_disable(sport->clk_ipg);
+ if (retval) {
+ clk_unprepare(sport->clk_ipg);
+ goto error_console;
+ }
+
+ retval = clk_prepare(sport->clk_per);
+ if (retval)
+ clk_disable_unprepare(sport->clk_ipg);
error_console:
return retval;
atomic_inc(&buf->priority);
mutex_lock(&buf->lock);
- while ((next = buf->head->next) != NULL) {
+ /* paired w/ release in __tty_buffer_request_room; ensures there are
+ * no pending memory accesses to the freed buffer
+ */
+ while ((next = smp_load_acquire(&buf->head->next)) != NULL) {
tty_buffer_free(port, buf->head);
buf->head = next;
}
if (n != NULL) {
n->flags = flags;
buf->tail = n;
- b->commit = b->used;
+ /* paired w/ acquire in flush_to_ldisc(); ensures
+ * flush_to_ldisc() sees buffer data.
+ */
+ smp_store_release(&b->commit, b->used);
/* paired w/ acquire in flush_to_ldisc(); ensures the
* latest commit value can be read before the head is
* advanced to the next buffer
{
struct tty_bufhead *buf = &port->buf;
- buf->tail->commit = buf->tail->used;
+ /* paired w/ acquire in flush_to_ldisc(); ensures
+ * flush_to_ldisc() sees buffer data.
+ */
+ smp_store_release(&buf->tail->commit, buf->tail->used);
schedule_work(&buf->work);
}
EXPORT_SYMBOL(tty_schedule_flip);
struct tty_struct *tty;
struct tty_ldisc *disc;
- tty = port->itty;
+ tty = READ_ONCE(port->itty);
if (tty == NULL)
return;
* is advancing to the next buffer
*/
next = smp_load_acquire(&head->next);
- count = head->commit - head->read;
+ /* paired w/ release in __tty_buffer_request_room() or in
+ * tty_buffer_flush(); ensures we see the committed buffer data
+ */
+ count = smp_load_acquire(&head->commit) - head->read;
if (!count) {
if (next == NULL) {
check_other_closed(tty);
if (!noctty &&
current->signal->leader &&
!current->signal->tty &&
- tty->session == NULL)
- __proc_set_tty(tty);
+ tty->session == NULL) {
+ /*
+ * Don't let a process that only has write access to the tty
+ * obtain the privileges associated with having a tty as
+ * controlling terminal (being able to reopen it with full
+ * access through /dev/tty, being able to perform pushback).
+ * Many distributions set the group of all ttys to "tty" and
+ * grant write-only access to all terminals for setgid tty
+ * binaries, which should not imply full privileges on all ttys.
+ *
+ * This could theoretically break old code that performs open()
+ * on a write-only file descriptor. In that case, it might be
+ * necessary to also permit this if
+ * inode_permission(inode, MAY_READ) == 0.
+ */
+ if (filp->f_mode & FMODE_READ)
+ __proc_set_tty(tty);
+ }
spin_unlock_irq(¤t->sighand->siglock);
read_unlock(&tasklist_lock);
tty_unlock(tty);
* Takes ->siglock() when updating signal->tty
*/
-static int tiocsctty(struct tty_struct *tty, int arg)
+static int tiocsctty(struct tty_struct *tty, struct file *file, int arg)
{
int ret = 0;
goto unlock;
}
}
+
+ /* See the comment in tty_open(). */
+ if ((file->f_mode & FMODE_READ) == 0 && !capable(CAP_SYS_ADMIN)) {
+ ret = -EPERM;
+ goto unlock;
+ }
+
proc_set_tty(tty);
unlock:
read_unlock(&tasklist_lock);
no_tty();
return 0;
case TIOCSCTTY:
- return tiocsctty(tty, arg);
+ return tiocsctty(tty, file, arg);
case TIOCGPGRP:
return tiocgpgrp(tty, real_tty, p);
case TIOCSPGRP:
static int tty_cdev_add(struct tty_driver *driver, dev_t dev,
unsigned int index, unsigned int count)
{
+ int err;
+
/* init here, since reused cdevs cause crashes */
driver->cdevs[index] = cdev_alloc();
if (!driver->cdevs[index])
return -ENOMEM;
- cdev_init(driver->cdevs[index], &tty_fops);
+ driver->cdevs[index]->ops = &tty_fops;
driver->cdevs[index]->owner = driver->owner;
- return cdev_add(driver->cdevs[index], dev, count);
+ err = cdev_add(driver->cdevs[index], dev, count);
+ if (err)
+ kobject_put(&driver->cdevs[index]->kobj);
+ return err;
}
/**
{ .compatible = "fsl,imx27-usb", .data = &imx27_usb_data},
{ .compatible = "fsl,imx6q-usb", .data = &imx6q_usb_data},
{ .compatible = "fsl,imx6sl-usb", .data = &imx6sl_usb_data},
- { .compatible = "fsl,imx6sx-usb", .data = &imx6sl_usb_data},
+ { .compatible = "fsl,imx6sx-usb", .data = &imx6sx_usb_data},
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, ci_hdrc_imx_dt_ids);
#include <linux/dma-mapping.h>
#include <linux/module.h>
#include <linux/of.h>
+#include <linux/of_platform.h>
#include <linux/phy/phy.h>
#include <linux/platform_device.h>
#include <linux/usb/chipidea.h>
.flags = CI_HDRC_DISABLE_STREAMING,
};
+static struct ci_hdrc_platform_data ci_zynq_pdata = {
+ .capoffset = DEF_CAPOFFSET,
+};
+
+static const struct of_device_id ci_hdrc_usb2_of_match[] = {
+ { .compatible = "chipidea,usb2"},
+ { .compatible = "xlnx,zynq-usb-2.20a", .data = &ci_zynq_pdata},
+ { }
+};
+MODULE_DEVICE_TABLE(of, ci_hdrc_usb2_of_match);
+
static int ci_hdrc_usb2_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct ci_hdrc_usb2_priv *priv;
struct ci_hdrc_platform_data *ci_pdata = dev_get_platdata(dev);
int ret;
+ const struct of_device_id *match;
if (!ci_pdata) {
ci_pdata = devm_kmalloc(dev, sizeof(*ci_pdata), GFP_KERNEL);
*ci_pdata = ci_default_pdata; /* struct copy */
}
+ match = of_match_device(ci_hdrc_usb2_of_match, &pdev->dev);
+ if (match && match->data) {
+ /* struct copy */
+ *ci_pdata = *(struct ci_hdrc_platform_data *)match->data;
+ }
+
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
return 0;
}
-static const struct of_device_id ci_hdrc_usb2_of_match[] = {
- { .compatible = "chipidea,usb2" },
- { }
-};
-MODULE_DEVICE_TABLE(of, ci_hdrc_usb2_of_match);
-
static struct platform_driver ci_hdrc_usb2_driver = {
.probe = ci_hdrc_usb2_probe,
.remove = ci_hdrc_usb2_remove,
return 0;
}
+static int _ep_set_halt(struct usb_ep *ep, int value, bool check_transfer)
+{
+ struct ci_hw_ep *hwep = container_of(ep, struct ci_hw_ep, ep);
+ int direction, retval = 0;
+ unsigned long flags;
+
+ if (ep == NULL || hwep->ep.desc == NULL)
+ return -EINVAL;
+
+ if (usb_endpoint_xfer_isoc(hwep->ep.desc))
+ return -EOPNOTSUPP;
+
+ spin_lock_irqsave(hwep->lock, flags);
+
+ if (value && hwep->dir == TX && check_transfer &&
+ !list_empty(&hwep->qh.queue) &&
+ !usb_endpoint_xfer_control(hwep->ep.desc)) {
+ spin_unlock_irqrestore(hwep->lock, flags);
+ return -EAGAIN;
+ }
+
+ direction = hwep->dir;
+ do {
+ retval |= hw_ep_set_halt(hwep->ci, hwep->num, hwep->dir, value);
+
+ if (!value)
+ hwep->wedge = 0;
+
+ if (hwep->type == USB_ENDPOINT_XFER_CONTROL)
+ hwep->dir = (hwep->dir == TX) ? RX : TX;
+
+ } while (hwep->dir != direction);
+
+ spin_unlock_irqrestore(hwep->lock, flags);
+ return retval;
+}
+
+
/**
* _gadget_stop_activity: stops all USB activity, flushes & disables all endpts
* @gadget: gadget
num += ci->hw_ep_max / 2;
spin_unlock(&ci->lock);
- err = usb_ep_set_halt(&ci->ci_hw_ep[num].ep);
+ err = _ep_set_halt(&ci->ci_hw_ep[num].ep, 1, false);
spin_lock(&ci->lock);
if (!err)
isr_setup_status_phase(ci);
if (err < 0) {
spin_unlock(&ci->lock);
- if (usb_ep_set_halt(&hwep->ep))
- dev_err(ci->dev, "error: ep_set_halt\n");
+ if (_ep_set_halt(&hwep->ep, 1, false))
+ dev_err(ci->dev, "error: _ep_set_halt\n");
spin_lock(&ci->lock);
}
}
err = isr_setup_status_phase(ci);
if (err < 0) {
spin_unlock(&ci->lock);
- if (usb_ep_set_halt(&hwep->ep))
+ if (_ep_set_halt(&hwep->ep, 1, false))
dev_err(ci->dev,
- "error: ep_set_halt\n");
+ "error: _ep_set_halt\n");
spin_lock(&ci->lock);
}
}
*/
static int ep_set_halt(struct usb_ep *ep, int value)
{
- struct ci_hw_ep *hwep = container_of(ep, struct ci_hw_ep, ep);
- int direction, retval = 0;
- unsigned long flags;
-
- if (ep == NULL || hwep->ep.desc == NULL)
- return -EINVAL;
-
- if (usb_endpoint_xfer_isoc(hwep->ep.desc))
- return -EOPNOTSUPP;
-
- spin_lock_irqsave(hwep->lock, flags);
-
-#ifndef STALL_IN
- /* g_file_storage MS compliant but g_zero fails chapter 9 compliance */
- if (value && hwep->type == USB_ENDPOINT_XFER_BULK && hwep->dir == TX &&
- !list_empty(&hwep->qh.queue)) {
- spin_unlock_irqrestore(hwep->lock, flags);
- return -EAGAIN;
- }
-#endif
-
- direction = hwep->dir;
- do {
- retval |= hw_ep_set_halt(hwep->ci, hwep->num, hwep->dir, value);
-
- if (!value)
- hwep->wedge = 0;
-
- if (hwep->type == USB_ENDPOINT_XFER_CONTROL)
- hwep->dir = (hwep->dir == TX) ? RX : TX;
-
- } while (hwep->dir != direction);
-
- spin_unlock_irqrestore(hwep->lock, flags);
- return retval;
+ return _ep_set_halt(ep, value, true);
}
/**
cfgno, inum, asnum, ep->desc.bEndpointAddress);
ep->ss_ep_comp.bmAttributes = 16;
} else if (usb_endpoint_xfer_isoc(&ep->desc) &&
- desc->bmAttributes > 2) {
+ USB_SS_MULT(desc->bmAttributes) > 3) {
dev_warn(ddev, "Isoc endpoint has Mult of %d in "
"config %d interface %d altsetting %d ep %d: "
"setting to 3\n", desc->bmAttributes + 1,
}
if (usb_endpoint_xfer_isoc(&ep->desc))
- max_tx = (desc->bMaxBurst + 1) * (desc->bmAttributes + 1) *
+ max_tx = (desc->bMaxBurst + 1) *
+ (USB_SS_MULT(desc->bmAttributes)) *
usb_endpoint_maxp(&ep->desc);
else if (usb_endpoint_xfer_int(&ep->desc))
max_tx = usb_endpoint_maxp(&ep->desc) *
{ USB_DEVICE(0x046d, 0x082d), .driver_info = USB_QUIRK_DELAY_INIT },
{ USB_DEVICE(0x046d, 0x0843), .driver_info = USB_QUIRK_DELAY_INIT },
+ /* Logitech ConferenceCam CC3000e */
+ { USB_DEVICE(0x046d, 0x0847), .driver_info = USB_QUIRK_DELAY_INIT },
+ { USB_DEVICE(0x046d, 0x0848), .driver_info = USB_QUIRK_DELAY_INIT },
+
+ /* Logitech PTZ Pro Camera */
+ { USB_DEVICE(0x046d, 0x0853), .driver_info = USB_QUIRK_DELAY_INIT },
+
/* Logitech Quickcam Fusion */
{ USB_DEVICE(0x046d, 0x08c1), .driver_info = USB_QUIRK_RESET_RESUME },
/* Philips PSC805 audio device */
{ USB_DEVICE(0x0471, 0x0155), .driver_info = USB_QUIRK_RESET_RESUME },
+ /* Plantronic Audio 655 DSP */
+ { USB_DEVICE(0x047f, 0xc008), .driver_info = USB_QUIRK_RESET_RESUME },
+
+ /* Plantronic Audio 648 USB */
+ { USB_DEVICE(0x047f, 0xc013), .driver_info = USB_QUIRK_RESET_RESUME },
+
/* Artisman Watchdog Dongle */
{ USB_DEVICE(0x04b4, 0x0526), .driver_info =
USB_QUIRK_CONFIG_INTF_STRINGS },
goto err1;
}
- dwc3_omap_enable_irqs(omap);
-
ret = dwc3_omap_extcon_register(omap);
if (ret < 0)
goto err2;
goto err3;
}
+ dwc3_omap_enable_irqs(omap);
+
return 0;
err3:
int i;
irqreturn_t ret = IRQ_NONE;
- spin_lock(&dwc->lock);
-
for (i = 0; i < dwc->num_event_buffers; i++) {
irqreturn_t status;
ret = status;
}
- spin_unlock(&dwc->lock);
-
return ret;
}
list_for_each_entry (ep, &gadget->ep_list, ep_list) {
ep->claimed = false;
+ ep->driver_data = NULL;
}
gadget->in_epnum = 0;
gadget->out_epnum = 0;
writel(AMD_BIT(UDC_DEVCFG_SOFTRESET), &dev->regs->cfg);
if (dev->irq_registered)
free_irq(pdev->irq, dev);
- if (dev->regs)
- iounmap(dev->regs);
+ if (dev->virt_addr)
+ iounmap(dev->virt_addr);
if (dev->mem_region)
release_mem_region(pci_resource_start(pdev, 0),
pci_resource_len(pdev, 0));
/* init */
dev = kzalloc(sizeof(struct udc), GFP_KERNEL);
- if (!dev) {
- retval = -ENOMEM;
- goto finished;
- }
+ if (!dev)
+ return -ENOMEM;
/* pci setup */
if (pci_enable_device(pdev) < 0) {
- kfree(dev);
- dev = NULL;
retval = -ENODEV;
- goto finished;
+ goto err_pcidev;
}
dev->active = 1;
if (!request_mem_region(resource, len, name)) {
dev_dbg(&pdev->dev, "pci device used already\n");
- kfree(dev);
- dev = NULL;
retval = -EBUSY;
- goto finished;
+ goto err_memreg;
}
dev->mem_region = 1;
dev->virt_addr = ioremap_nocache(resource, len);
if (dev->virt_addr == NULL) {
dev_dbg(&pdev->dev, "start address cannot be mapped\n");
- kfree(dev);
- dev = NULL;
retval = -EFAULT;
- goto finished;
+ goto err_ioremap;
}
if (!pdev->irq) {
dev_err(&pdev->dev, "irq not set\n");
- kfree(dev);
- dev = NULL;
retval = -ENODEV;
- goto finished;
+ goto err_irq;
}
spin_lock_init(&dev->lock);
if (request_irq(pdev->irq, udc_irq, IRQF_SHARED, name, dev) != 0) {
dev_dbg(&pdev->dev, "request_irq(%d) fail\n", pdev->irq);
- kfree(dev);
- dev = NULL;
retval = -EBUSY;
- goto finished;
+ goto err_irq;
}
dev->irq_registered = 1;
return 0;
finished:
- if (dev)
- udc_pci_remove(pdev);
+ udc_pci_remove(pdev);
+ return retval;
+
+err_irq:
+ iounmap(dev->virt_addr);
+err_ioremap:
+ release_mem_region(resource, len);
+err_memreg:
+ pci_disable_device(pdev);
+err_pcidev:
+ kfree(dev);
return retval;
}
ep->udc = udc;
INIT_LIST_HEAD(&ep->queue);
+ if (ep->index == 0) {
+ ep->ep.caps.type_control = true;
+ } else {
+ ep->ep.caps.type_iso = ep->can_isoc;
+ ep->ep.caps.type_bulk = true;
+ ep->ep.caps.type_int = true;
+ }
+
+ ep->ep.caps.dir_in = true;
+ ep->ep.caps.dir_out = true;
+
if (i)
list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
bdc->scratchpad.buff, bdc->scratchpad.sp_dma);
/* Destroy the dma pools */
- if (bdc->bd_table_pool)
- dma_pool_destroy(bdc->bd_table_pool);
+ dma_pool_destroy(bdc->bd_table_pool);
/* Free the bdc_ep array */
kfree(bdc->bdc_ep_array);
bd_table->start_bd = dma_pool_alloc(bdc->bd_table_pool,
GFP_ATOMIC,
&dma);
- if (!bd_table->start_bd)
+ if (!bd_table->start_bd) {
+ kfree(bd_table);
goto fail;
+ }
bd_table->dma = dma;
{
struct dummy *dum = dum_hcd->dum;
struct dummy_request *req;
+ int sent = 0;
top:
/* if there's no request queued, the device is NAKing; return */
if (len == 0)
break;
- /* use an extra pass for the final short packet */
- if (len > ep->ep.maxpacket) {
- rescan = 1;
- len -= (len % ep->ep.maxpacket);
+ /* send multiple of maxpacket first, then remainder */
+ if (len >= ep->ep.maxpacket) {
+ is_short = 0;
+ if (len % ep->ep.maxpacket)
+ rescan = 1;
+ len -= len % ep->ep.maxpacket;
+ } else {
+ is_short = 1;
}
- is_short = (len % ep->ep.maxpacket) != 0;
len = dummy_perform_transfer(urb, req, len);
req->req.status = len;
} else {
limit -= len;
+ sent += len;
urb->actual_length += len;
req->req.actual += len;
}
*status = -EOVERFLOW;
else
*status = 0;
- } else if (!to_host) {
+ } else {
*status = 0;
if (host_len > dev_len)
req->req.status = -EOVERFLOW;
req->req.status = 0;
}
- /* many requests terminate without a short packet */
+ /*
+ * many requests terminate without a short packet.
+ * send a zlp if demanded by flags.
+ */
} else {
- if (req->req.length == req->req.actual
- && !req->req.zero)
- req->req.status = 0;
- if (urb->transfer_buffer_length == urb->actual_length
- && !(urb->transfer_flags
- & URB_ZERO_PACKET))
- *status = 0;
+ if (req->req.length == req->req.actual) {
+ if (req->req.zero && to_host)
+ rescan = 1;
+ else
+ req->req.status = 0;
+ }
+ if (urb->transfer_buffer_length == urb->actual_length) {
+ if (urb->transfer_flags & URB_ZERO_PACKET &&
+ !to_host)
+ rescan = 1;
+ else
+ *status = 0;
+ }
}
/* device side completion --> continuable */
if (rescan)
goto top;
}
- return limit;
+ return sent;
}
static int periodic_bytes(struct dummy *dum, struct dummy_ep *ep)
default:
treat_control_like_bulk:
ep->last_io = jiffies;
- total = transfer(dum_hcd, urb, ep, limit, &status);
+ total -= transfer(dum_hcd, urb, ep, limit, &status);
break;
}
return -EBUSY;
gr_dfs_delete(dev);
- if (dev->desc_pool)
- dma_pool_destroy(dev->desc_pool);
+ dma_pool_destroy(dev->desc_pool);
platform_set_drvdata(pdev, NULL);
gr_free_request(&dev->epi[0].ep, &dev->ep0reqi->req);
usb_del_gadget_udc(&u3d->gadget);
/* free memory allocated in probe */
- if (u3d->trb_pool)
- dma_pool_destroy(u3d->trb_pool);
+ dma_pool_destroy(u3d->trb_pool);
if (u3d->ep_context)
dma_free_coherent(&dev->dev, u3d->ep_context_size,
}
/* free memory allocated in probe */
- if (udc->dtd_pool)
- dma_pool_destroy(udc->dtd_pool);
+ dma_pool_destroy(udc->dtd_pool);
if (udc->ep_dqh)
dma_free_coherent(&pdev->dev, udc->ep_dqh_size,
* use Event Data TRBs, and we don't chain in a link TRB on short
* transfers, we're basically dividing by 1.
*
- * xHCI 1.0 specification indicates that the Average TRB Length should
- * be set to 8 for control endpoints.
+ * xHCI 1.0 and 1.1 specification indicates that the Average TRB Length
+ * should be set to 8 for control endpoints.
*/
- if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version == 0x100)
+ if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100)
ep_ctx->tx_info |= cpu_to_le32(AVG_TRB_LENGTH_FOR_EP(8));
else
ep_ctx->tx_info |=
int size;
int i, j, num_ports;
- if (timer_pending(&xhci->cmd_timer))
- del_timer_sync(&xhci->cmd_timer);
+ del_timer_sync(&xhci->cmd_timer);
/* Free the Event Ring Segment Table and the actual Event Ring */
size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries);
INIT_LIST_HEAD(&xhci->cmd_list);
+ /* init command timeout timer */
+ setup_timer(&xhci->cmd_timer, xhci_handle_command_timeout,
+ (unsigned long)xhci);
+
page_size = readl(&xhci->op_regs->page_size);
xhci_dbg_trace(xhci, trace_xhci_dbg_init,
"Supported page size register = 0x%x", page_size);
"Wrote ERST address to ir_set 0.");
xhci_print_ir_set(xhci, 0);
- /* init command timeout timer */
- setup_timer(&xhci->cmd_timer, xhci_handle_command_timeout,
- (unsigned long)xhci);
-
/*
* XXX: Might need to set the Interrupter Moderation Register to
* something other than the default (~1ms minimum between interrupts).
"QUIRK: Resetting on resume");
}
-/*
- * In some Intel xHCI controllers, in order to get D3 working,
- * through a vendor specific SSIC CONFIG register at offset 0x883c,
- * SSIC PORT need to be marked as "unused" before putting xHCI
- * into D3. After D3 exit, the SSIC port need to be marked as "used".
- * Without this change, xHCI might not enter D3 state.
- * Make sure PME works on some Intel xHCI controllers by writing 1 to clear
- * the Internal PME flag bit in vendor specific PMCTRL register at offset 0x80a4
- */
-static void xhci_pme_quirk(struct usb_hcd *hcd, bool suspend)
-{
- struct xhci_hcd *xhci = hcd_to_xhci(hcd);
- struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
- u32 val;
- void __iomem *reg;
-
- if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
- pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI) {
-
- reg = (void __iomem *) xhci->cap_regs + PORT2_SSIC_CONFIG_REG2;
-
- /* Notify SSIC that SSIC profile programming is not done */
- val = readl(reg) & ~PROG_DONE;
- writel(val, reg);
-
- /* Mark SSIC port as unused(suspend) or used(resume) */
- val = readl(reg);
- if (suspend)
- val |= SSIC_PORT_UNUSED;
- else
- val &= ~SSIC_PORT_UNUSED;
- writel(val, reg);
-
- /* Notify SSIC that SSIC profile programming is done */
- val = readl(reg) | PROG_DONE;
- writel(val, reg);
- readl(reg);
- }
-
- reg = (void __iomem *) xhci->cap_regs + 0x80a4;
- val = readl(reg);
- writel(val | BIT(28), reg);
- readl(reg);
-}
-
#ifdef CONFIG_ACPI
static void xhci_pme_acpi_rtd3_enable(struct pci_dev *dev)
{
}
#ifdef CONFIG_PM
+/*
+ * In some Intel xHCI controllers, in order to get D3 working,
+ * through a vendor specific SSIC CONFIG register at offset 0x883c,
+ * SSIC PORT need to be marked as "unused" before putting xHCI
+ * into D3. After D3 exit, the SSIC port need to be marked as "used".
+ * Without this change, xHCI might not enter D3 state.
+ * Make sure PME works on some Intel xHCI controllers by writing 1 to clear
+ * the Internal PME flag bit in vendor specific PMCTRL register at offset 0x80a4
+ */
+static void xhci_pme_quirk(struct usb_hcd *hcd, bool suspend)
+{
+ struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+ struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
+ u32 val;
+ void __iomem *reg;
+
+ if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
+ pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI) {
+
+ reg = (void __iomem *) xhci->cap_regs + PORT2_SSIC_CONFIG_REG2;
+
+ /* Notify SSIC that SSIC profile programming is not done */
+ val = readl(reg) & ~PROG_DONE;
+ writel(val, reg);
+
+ /* Mark SSIC port as unused(suspend) or used(resume) */
+ val = readl(reg);
+ if (suspend)
+ val |= SSIC_PORT_UNUSED;
+ else
+ val &= ~SSIC_PORT_UNUSED;
+ writel(val, reg);
+
+ /* Notify SSIC that SSIC profile programming is done */
+ val = readl(reg) | PROG_DONE;
+ writel(val, reg);
+ readl(reg);
+ }
+
+ reg = (void __iomem *) xhci->cap_regs + 0x80a4;
+ val = readl(reg);
+ writel(val | BIT(28), reg);
+ readl(reg);
+}
+
static int xhci_pci_suspend(struct usb_hcd *hcd, bool do_wakeup)
{
struct xhci_hcd *xhci = hcd_to_xhci(hcd);
ret = xhci_handshake(&xhci->op_regs->cmd_ring,
CMD_RING_RUNNING, 0, 5 * 1000 * 1000);
if (ret < 0) {
+ /* we are about to kill xhci, give it one more chance */
+ xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
+ &xhci->op_regs->cmd_ring);
+ udelay(1000);
+ ret = xhci_handshake(&xhci->op_regs->cmd_ring,
+ CMD_RING_RUNNING, 0, 3 * 1000 * 1000);
+ if (ret == 0)
+ return 0;
+
xhci_err(xhci, "Stopped the command ring failed, "
"maybe the host is dead\n");
xhci->xhc_state |= XHCI_STATE_DYING;
if (start_cycle == 0)
field |= 0x1;
- /* xHCI 1.0 6.4.1.2.1: Transfer Type field */
- if (xhci->hci_version == 0x100) {
+ /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */
+ if (xhci->hci_version >= 0x100) {
if (urb->transfer_buffer_length > 0) {
if (setup->bRequestType & USB_DIR_IN)
field |= TRB_TX_TYPE(TRB_DATA_IN);
"waited %u microseconds.\n",
XHCI_MAX_HALT_USEC);
if (!ret)
- xhci->xhc_state &= ~XHCI_STATE_HALTED;
+ xhci->xhc_state &= ~(XHCI_STATE_HALTED | XHCI_STATE_DYING);
+
return ret;
}
}
EXPORT_SYMBOL_GPL(xhci_run);
-static void xhci_only_stop_hcd(struct usb_hcd *hcd)
-{
- struct xhci_hcd *xhci = hcd_to_xhci(hcd);
-
- spin_lock_irq(&xhci->lock);
- xhci_halt(xhci);
- spin_unlock_irq(&xhci->lock);
-}
-
/*
* Stop xHCI driver.
*
u32 temp;
struct xhci_hcd *xhci = hcd_to_xhci(hcd);
- if (!usb_hcd_is_primary_hcd(hcd)) {
- xhci_only_stop_hcd(xhci->shared_hcd);
+ if (xhci->xhc_state & XHCI_STATE_HALTED)
return;
- }
+ mutex_lock(&xhci->mutex);
spin_lock_irq(&xhci->lock);
+ xhci->xhc_state |= XHCI_STATE_HALTED;
+ xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
+
/* Make sure the xHC is halted for a USB3 roothub
* (xhci_stop() could be called as part of failed init).
*/
xhci_dbg_trace(xhci, trace_xhci_dbg_init,
"xhci_stop completed - status = %x",
readl(&xhci->op_regs->status));
+ mutex_unlock(&xhci->mutex);
}
/*
mutex_lock(&xhci->mutex);
+ if (xhci->xhc_state) /* dying or halted */
+ goto out;
+
if (!udev->slot_id) {
xhci_dbg_trace(xhci, trace_xhci_dbg_address,
"Bad Slot ID %d", udev->slot_id);
if (this_time > max)
this_time = max;
- memcpy(data, dev->buf, this_time);
+ memcpy(data, dev->buf + dev->used, this_time);
dev->used += this_time;
* (c) peripheral initiates, using SRP
*/
if (musb->port_mode != MUSB_PORT_MODE_HOST &&
+ musb->xceiv->otg->state != OTG_STATE_A_WAIT_BCON &&
(devctl & MUSB_DEVCTL_VBUS) == MUSB_DEVCTL_VBUS) {
musb->is_active = 1;
} else {
struct musb *musb = dev_to_musb(dev);
unsigned long flags;
+ musb_platform_disable(musb);
+ musb_generic_disable(musb);
+
spin_lock_irqsave(&musb->lock, flags);
if (is_peripheral_active(musb)) {
pm_runtime_disable(dev);
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
+
+ musb_start(musb);
+
return 0;
}
} else {
cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);
+ /* delay to drain to cppi dma pipeline for isoch */
+ udelay(250);
+
csr = musb_readw(epio, MUSB_RXCSR);
csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB);
musb_writew(epio, MUSB_RXCSR, csr);
dsps_writel(reg_base, wrp->epintr_set, epmask);
dsps_writel(reg_base, wrp->coreintr_set, coremask);
- /* start polling for ID change. */
- mod_timer(&glue->timer, jiffies + msecs_to_jiffies(wrp->poll_timeout));
+ /* start polling for ID change in dual-role idle mode */
+ if (musb->xceiv->otg->state == OTG_STATE_B_IDLE &&
+ musb->port_mode == MUSB_PORT_MODE_DUAL_ROLE)
+ mod_timer(&glue->timer, jiffies +
+ msecs_to_jiffies(wrp->poll_timeout));
dsps_musb_try_idle(musb, 0);
}
{}
};
+MODULE_DEVICE_TABLE(of, ux500_match);
+
static struct platform_driver ux500_driver = {
.probe = ux500_probe,
.remove = ux500_remove,
config USB_QCOM_8X16_PHY
tristate "Qualcomm APQ8016/MSM8916 on-chip USB PHY controller support"
depends on ARCH_QCOM || COMPILE_TEST
- depends on RESET_CONTROLLER
+ depends on RESET_CONTROLLER && EXTCON
select USB_PHY
select USB_ULPI_VIEWPORT
help
clk_rate = pdata->clk_rate;
needs_vcc = pdata->needs_vcc;
if (gpio_is_valid(pdata->gpio_reset)) {
- err = devm_gpio_request_one(dev, pdata->gpio_reset, 0,
+ err = devm_gpio_request_one(dev, pdata->gpio_reset,
+ GPIOF_ACTIVE_LOW,
dev_name(dev));
if (!err)
nop->gpiod_reset =
{ "isp1301", 0 },
{ }
};
+MODULE_DEVICE_TABLE(i2c, isp1301_id);
static struct i2c_client *isp1301_i2c_client;
.compatible = "renesas,usbhs-r8a7794",
.data = (void *)USBHS_TYPE_RCAR_GEN2,
},
+ {
+ /* Gen3 is compatible with Gen2 */
+ .compatible = "renesas,usbhs-r8a7795",
+ .data = (void *)USBHS_TYPE_RCAR_GEN2,
+ },
{ },
};
MODULE_DEVICE_TABLE(of, usbhs_of_match);
return NULL;
dparam = &info->driver_param;
- dparam->type = of_id ? (u32)of_id->data : 0;
+ dparam->type = of_id ? (uintptr_t)of_id->data : 0;
if (!of_property_read_u32(dev->of_node, "renesas,buswait", &tmp))
dparam->buswait_bwait = tmp;
gpio = of_get_named_gpio_flags(dev->of_node, "renesas,enable-gpio", 0,
#define ZTE_PRODUCT_MF622 0x0001
#define ZTE_PRODUCT_MF628 0x0015
#define ZTE_PRODUCT_MF626 0x0031
+#define ZTE_PRODUCT_ZM8620_X 0x0396
+#define ZTE_PRODUCT_ME3620_MBIM 0x0426
+#define ZTE_PRODUCT_ME3620_X 0x1432
+#define ZTE_PRODUCT_ME3620_L 0x1433
#define ZTE_PRODUCT_AC2726 0xfff1
#define ZTE_PRODUCT_MG880 0xfffd
#define ZTE_PRODUCT_CDMA_TECH 0xfffe
.sendsetup = BIT(1) | BIT(2) | BIT(3),
};
+static const struct option_blacklist_info zte_me3620_mbim_blacklist = {
+ .reserved = BIT(2) | BIT(3) | BIT(4),
+};
+
+static const struct option_blacklist_info zte_me3620_xl_blacklist = {
+ .reserved = BIT(3) | BIT(4) | BIT(5),
+};
+
+static const struct option_blacklist_info zte_zm8620_x_blacklist = {
+ .reserved = BIT(3) | BIT(4) | BIT(5),
+};
+
static const struct option_blacklist_info huawei_cdc12_blacklist = {
.reserved = BIT(1) | BIT(2),
};
.driver_info = (kernel_ulong_t)&zte_ad3812_z_blacklist },
{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, ZTE_PRODUCT_MC2716, 0xff, 0xff, 0xff),
.driver_info = (kernel_ulong_t)&zte_mc2716_z_blacklist },
+ { USB_DEVICE(ZTE_VENDOR_ID, ZTE_PRODUCT_ME3620_L),
+ .driver_info = (kernel_ulong_t)&zte_me3620_xl_blacklist },
+ { USB_DEVICE(ZTE_VENDOR_ID, ZTE_PRODUCT_ME3620_MBIM),
+ .driver_info = (kernel_ulong_t)&zte_me3620_mbim_blacklist },
+ { USB_DEVICE(ZTE_VENDOR_ID, ZTE_PRODUCT_ME3620_X),
+ .driver_info = (kernel_ulong_t)&zte_me3620_xl_blacklist },
+ { USB_DEVICE(ZTE_VENDOR_ID, ZTE_PRODUCT_ZM8620_X),
+ .driver_info = (kernel_ulong_t)&zte_zm8620_x_blacklist },
{ USB_VENDOR_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0xff, 0x02, 0x01) },
{ USB_VENDOR_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0xff, 0x02, 0x05) },
{ USB_VENDOR_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0xff, 0x86, 0x10) },
static int whiteheat_firmware_attach(struct usb_serial *serial);
/* function prototypes for the Connect Tech WhiteHEAT serial converter */
+static int whiteheat_probe(struct usb_serial *serial,
+ const struct usb_device_id *id);
static int whiteheat_attach(struct usb_serial *serial);
static void whiteheat_release(struct usb_serial *serial);
static int whiteheat_port_probe(struct usb_serial_port *port);
.description = "Connect Tech - WhiteHEAT",
.id_table = id_table_std,
.num_ports = 4,
+ .probe = whiteheat_probe,
.attach = whiteheat_attach,
.release = whiteheat_release,
.port_probe = whiteheat_port_probe,
/*****************************************************************************
* Connect Tech's White Heat serial driver functions
*****************************************************************************/
+
+static int whiteheat_probe(struct usb_serial *serial,
+ const struct usb_device_id *id)
+{
+ struct usb_host_interface *iface_desc;
+ struct usb_endpoint_descriptor *endpoint;
+ size_t num_bulk_in = 0;
+ size_t num_bulk_out = 0;
+ size_t min_num_bulk;
+ unsigned int i;
+
+ iface_desc = serial->interface->cur_altsetting;
+
+ for (i = 0; i < iface_desc->desc.bNumEndpoints; i++) {
+ endpoint = &iface_desc->endpoint[i].desc;
+ if (usb_endpoint_is_bulk_in(endpoint))
+ ++num_bulk_in;
+ if (usb_endpoint_is_bulk_out(endpoint))
+ ++num_bulk_out;
+ }
+
+ min_num_bulk = COMMAND_PORT + 1;
+ if (num_bulk_in < min_num_bulk || num_bulk_out < min_num_bulk)
+ return -ENODEV;
+
+ return 0;
+}
+
static int whiteheat_attach(struct usb_serial *serial)
{
struct usb_serial_port *command_port;
enum {
VHOST_NET_FEATURES = VHOST_FEATURES |
(1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
- (1ULL << VIRTIO_NET_F_MRG_RXBUF) |
- (1ULL << VIRTIO_F_VERSION_1),
+ (1ULL << VIRTIO_NET_F_MRG_RXBUF)
};
enum {
/* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
enum {
VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
- (1ULL << VIRTIO_SCSI_F_T10_PI) |
- (1ULL << VIRTIO_F_ANY_LAYOUT) |
- (1ULL << VIRTIO_F_VERSION_1)
+ (1ULL << VIRTIO_SCSI_F_T10_PI)
};
#define VHOST_SCSI_MAX_TARGET 256
return -EFAULT;
return 0;
case VHOST_SET_FEATURES:
+ printk(KERN_ERR "1\n");
if (copy_from_user(&features, featurep, sizeof features))
return -EFAULT;
+ printk(KERN_ERR "2\n");
if (features & ~VHOST_FEATURES)
return -EOPNOTSUPP;
+ printk(KERN_ERR "3\n");
return vhost_test_set_features(n, features);
case VHOST_RESET_OWNER:
return vhost_test_reset_owner(n);
VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) |
(1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
(1ULL << VIRTIO_RING_F_EVENT_IDX) |
- (1ULL << VHOST_F_LOG_ALL),
+ (1ULL << VHOST_F_LOG_ALL) |
+ (1ULL << VIRTIO_F_ANY_LAYOUT) |
+ (1ULL << VIRTIO_F_VERSION_1)
};
static inline bool vhost_has_feature(struct vhost_virtqueue *vq, int bit)
if ((fw_entry->size < 8*1024) || (fw_entry->size > 64*1024)) {
dev_err(dev, "Invalid waveform\n");
err = -EINVAL;
- goto err_failed;
+ goto err_fw;
}
mutex_lock(&(par->io_lock));
mutex_unlock(&(par->io_lock));
if (err < 0) {
dev_err(dev, "Failed to store broadsheet waveform\n");
- goto err_failed;
+ goto err_fw;
}
dev_info(dev, "Stored broadsheet waveform, size %zd\n", fw_entry->size);
- return len;
+ err = len;
+err_fw:
+ release_firmware(fw_entry);
err_failed:
return err;
}
static int fsl_diu_resume(struct platform_device *ofdev)
{
struct fsl_diu_data *data;
+ unsigned int i;
data = dev_get_drvdata(&ofdev->dev);
- enable_lcdc(data->fsl_diu_info);
+
+ fsl_diu_enable_interrupts(data);
+ update_lcdc(data->fsl_diu_info);
+ for (i = 0; i < NUM_AOIS; i++) {
+ if (data->mfb[i].count)
+ fsl_diu_enable_panel(&data->fsl_diu_info[i]);
+ }
return 0;
}
{ .compatible = "fujitsu,coral", },
{ /* end */ }
};
+MODULE_DEVICE_TABLE(of, of_platform_mb862xx_tbl);
static struct platform_driver of_platform_mb862xxfb_driver = {
.driver = {
adapter_node = of_parse_phandle(node, "ddc-i2c-bus", 0);
if (adapter_node) {
- adapter = of_find_i2c_adapter_by_node(adapter_node);
+ adapter = of_get_i2c_adapter_by_node(adapter_node);
if (adapter == NULL) {
dev_err(&pdev->dev, "failed to parse ddc-i2c-bus\n");
omap_dss_put_device(ddata->in);
{ .compatible = "omapdss,sony,acx565akm", },
{},
};
+MODULE_DEVICE_TABLE(of, acx565akm_of_match);
static struct spi_driver acx565akm_driver = {
.driver = {
writemmr(par, DST1, point(x, y));
writemmr(par, DST2, point(x + w - 1, y + h - 1));
- memcpy(par->io_virt + 0x10000, data, 4 * size);
+ iowrite32_rep(par->io_virt + 0x10000, data, size);
}
static void blade_copy_rect(struct tridentfb_par *par,
static inline void set_lwidth(struct tridentfb_par *par, int width)
{
write3X4(par, VGA_CRTC_OFFSET, width & 0xFF);
- write3X4(par, AddColReg,
- (read3X4(par, AddColReg) & 0xCF) | ((width & 0x300) >> 4));
+ /* chips older than TGUI9660 have only 1 width bit in AddColReg */
+ /* touching the other one breaks I2C/DDC */
+ if (par->chip_id == TGUI9440 || par->chip_id == CYBER9320)
+ write3X4(par, AddColReg,
+ (read3X4(par, AddColReg) & 0xEF) | ((width & 0x100) >> 4));
+ else
+ write3X4(par, AddColReg,
+ (read3X4(par, AddColReg) & 0xCF) | ((width & 0x300) >> 4));
}
/* For resolutions smaller than FP resolution stretch */
*/
pr_err("%s: error in timing %d\n",
of_node_full_name(np), disp->num_timings + 1);
+ kfree(dt);
goto timingfail;
}
tristate "Intel TCO Timer/Watchdog"
depends on (X86 || IA64) && PCI
select WATCHDOG_CORE
+ depends on I2C || I2C=n
select LPC_ICH if !EXPERT
- select I2C_I801 if !EXPERT
+ select I2C_I801 if !EXPERT && I2C
---help---
Hardware driver for the intel TCO timer based watchdog devices.
These drivers are included in the Intel 82801 I/O Controller
#define PM_RSTC_WRCFG_FULL_RESET 0x00000020
#define PM_RSTC_RESET 0x00000102
+/*
+ * The Raspberry Pi firmware uses the RSTS register to know which partiton
+ * to boot from. The partiton value is spread into bits 0, 2, 4, 6, 8, 10.
+ * Partiton 63 is a special partition used by the firmware to indicate halt.
+ */
+#define PM_RSTS_RASPBERRYPI_HALT 0x555
+
#define SECS_TO_WDOG_TICKS(x) ((x) << 16)
#define WDOG_TICKS_TO_SECS(x) ((x) >> 16)
* hard reset.
*/
val = readl_relaxed(wdt->base + PM_RSTS);
- val &= PM_RSTC_WRCFG_CLR;
- val |= PM_PASSWORD | PM_RSTS_HADWRH_SET;
+ val |= PM_PASSWORD | PM_RSTS_RASPBERRYPI_HALT;
writel_relaxed(val, wdt->base + PM_RSTS);
/* Continue with normal reset mechanism */
},
{},
};
+MODULE_DEVICE_TABLE(of, gef_wdt_ids);
static struct platform_driver gef_wdt_driver = {
.driver = {
{ .compatible = "men,a021-wdt" },
{ },
};
+MODULE_DEVICE_TABLE(of, a21_wdt_ids);
static struct platform_driver a21_wdt_driver = {
.probe = a21_wdt_probe,
{ .compatible = "moxa,moxart-watchdog" },
{ },
};
+MODULE_DEVICE_TABLE(of, moxart_watchdog_match);
static struct platform_driver moxart_wdt_driver = {
.probe = moxart_wdt_probe,
goto out_clear;
}
bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9);
+ /*
+ * If the partition is not aligned on a page
+ * boundary, we can't do dax I/O to it.
+ */
+ if ((bdev->bd_part->start_sect % (PAGE_SIZE / 512)) ||
+ (bdev->bd_part->nr_sects % (PAGE_SIZE / 512)))
+ bdev->bd_inode->i_flags &= ~S_DAX;
}
} else {
if (bdev->bd_contains == bdev) {
#define BTRFS_INODE_IN_DELALLOC_LIST 9
#define BTRFS_INODE_READDIO_NEED_LOCK 10
#define BTRFS_INODE_HAS_PROPS 11
-/* DIO is ready to submit */
-#define BTRFS_INODE_DIO_READY 12
/*
* The following 3 bits are meant only for the btree inode.
* When any of them is set, it means an error happened while writing an
!extent_buffer_uptodate(chunk_root->node)) {
printk(KERN_ERR "BTRFS: failed to read chunk root on %s\n",
sb->s_id);
+ if (!IS_ERR(chunk_root->node))
+ free_extent_buffer(chunk_root->node);
chunk_root->node = NULL;
goto fail_tree_roots;
}
!extent_buffer_uptodate(tree_root->node)) {
printk(KERN_WARNING "BTRFS: failed to read tree root on %s\n",
sb->s_id);
+ if (!IS_ERR(tree_root->node))
+ free_extent_buffer(tree_root->node);
tree_root->node = NULL;
goto recovery_tree_root;
}
* block groups queued for removal, the deletion will be
* skipped when we quit the cleaner thread.
*/
- mutex_lock(&root->fs_info->cleaner_mutex);
btrfs_delete_unused_bgs(root->fs_info);
- mutex_unlock(&root->fs_info->cleaner_mutex);
ret = btrfs_commit_super(root);
if (ret)
u32 generation;
if (fh_type == FILEID_BTRFS_WITH_PARENT) {
- if (fh_len != BTRFS_FID_SIZE_CONNECTABLE)
+ if (fh_len < BTRFS_FID_SIZE_CONNECTABLE)
return NULL;
root_objectid = fid->root_objectid;
} else if (fh_type == FILEID_BTRFS_WITH_PARENT_ROOT) {
- if (fh_len != BTRFS_FID_SIZE_CONNECTABLE_ROOT)
+ if (fh_len < BTRFS_FID_SIZE_CONNECTABLE_ROOT)
return NULL;
root_objectid = fid->parent_root_objectid;
} else
u32 generation;
if ((fh_type != FILEID_BTRFS_WITH_PARENT ||
- fh_len != BTRFS_FID_SIZE_CONNECTABLE) &&
+ fh_len < BTRFS_FID_SIZE_CONNECTABLE) &&
(fh_type != FILEID_BTRFS_WITH_PARENT_ROOT ||
- fh_len != BTRFS_FID_SIZE_CONNECTABLE_ROOT) &&
+ fh_len < BTRFS_FID_SIZE_CONNECTABLE_ROOT) &&
(fh_type != FILEID_BTRFS_WITHOUT_PARENT ||
- fh_len != BTRFS_FID_SIZE_NON_CONNECTABLE))
+ fh_len < BTRFS_FID_SIZE_NON_CONNECTABLE))
return NULL;
objectid = fid->objectid;
struct btrfs_delayed_ref_head *head;
int ret;
int run_all = count == (unsigned long)-1;
+ bool can_flush_pending_bgs = trans->can_flush_pending_bgs;
/* We'll clean this up in btrfs_cleanup_transaction */
if (trans->aborted)
#ifdef SCRAMBLE_DELAYED_REFS
delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
#endif
+ trans->can_flush_pending_bgs = false;
ret = __btrfs_run_delayed_refs(trans, root, count);
if (ret < 0) {
btrfs_abort_transaction(trans, root, ret);
}
out:
assert_qgroups_uptodate(trans);
+ trans->can_flush_pending_bgs = can_flush_pending_bgs;
return 0;
}
found->bytes_reserved = 0;
found->bytes_readonly = 0;
found->bytes_may_use = 0;
- if (total_bytes > 0)
- found->full = 0;
- else
- found->full = 1;
+ found->full = 0;
found->force_alloc = CHUNK_ALLOC_NO_FORCE;
found->chunk_alloc = 0;
found->flush = 0;
* the block groups that were made dirty during the lifetime of the
* transaction.
*/
- if (trans->chunk_bytes_reserved >= (2 * 1024 * 1024ull)) {
+ if (trans->can_flush_pending_bgs &&
+ trans->chunk_bytes_reserved >= (2 * 1024 * 1024ull)) {
btrfs_create_pending_block_groups(trans, trans->root);
btrfs_trans_release_chunk_metadata(trans);
}
}
if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) {
- btrfs_drop_and_free_fs_root(tree_root->fs_info, root);
+ btrfs_add_dropped_root(trans, root);
} else {
free_extent_buffer(root->node);
free_extent_buffer(root->commit_root);
struct btrfs_block_group_item item;
struct btrfs_key key;
int ret = 0;
+ bool can_flush_pending_bgs = trans->can_flush_pending_bgs;
+ trans->can_flush_pending_bgs = false;
list_for_each_entry_safe(block_group, tmp, &trans->new_bgs, bg_list) {
if (ret)
goto next;
next:
list_del_init(&block_group->bg_list);
}
+ trans->can_flush_pending_bgs = can_flush_pending_bgs;
}
int btrfs_make_block_group(struct btrfs_trans_handle *trans,
bio_end_io_t end_io_func,
int mirror_num,
unsigned long prev_bio_flags,
- unsigned long bio_flags)
+ unsigned long bio_flags,
+ bool force_bio_submit)
{
int ret = 0;
struct bio *bio;
contig = bio_end_sector(bio) == sector;
if (prev_bio_flags != bio_flags || !contig ||
+ force_bio_submit ||
merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
bio_add_page(bio, page, page_size, offset) < page_size) {
ret = submit_one_bio(rw, bio, mirror_num,
get_extent_t *get_extent,
struct extent_map **em_cached,
struct bio **bio, int mirror_num,
- unsigned long *bio_flags, int rw)
+ unsigned long *bio_flags, int rw,
+ u64 *prev_em_start)
{
struct inode *inode = page->mapping->host;
u64 start = page_offset(page);
}
while (cur <= end) {
unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
+ bool force_bio_submit = false;
if (cur >= last_byte) {
char *userpage;
block_start = em->block_start;
if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
block_start = EXTENT_MAP_HOLE;
+
+ /*
+ * If we have a file range that points to a compressed extent
+ * and it's followed by a consecutive file range that points to
+ * to the same compressed extent (possibly with a different
+ * offset and/or length, so it either points to the whole extent
+ * or only part of it), we must make sure we do not submit a
+ * single bio to populate the pages for the 2 ranges because
+ * this makes the compressed extent read zero out the pages
+ * belonging to the 2nd range. Imagine the following scenario:
+ *
+ * File layout
+ * [0 - 8K] [8K - 24K]
+ * | |
+ * | |
+ * points to extent X, points to extent X,
+ * offset 4K, length of 8K offset 0, length 16K
+ *
+ * [extent X, compressed length = 4K uncompressed length = 16K]
+ *
+ * If the bio to read the compressed extent covers both ranges,
+ * it will decompress extent X into the pages belonging to the
+ * first range and then it will stop, zeroing out the remaining
+ * pages that belong to the other range that points to extent X.
+ * So here we make sure we submit 2 bios, one for the first
+ * range and another one for the third range. Both will target
+ * the same physical extent from disk, but we can't currently
+ * make the compressed bio endio callback populate the pages
+ * for both ranges because each compressed bio is tightly
+ * coupled with a single extent map, and each range can have
+ * an extent map with a different offset value relative to the
+ * uncompressed data of our extent and different lengths. This
+ * is a corner case so we prioritize correctness over
+ * non-optimal behavior (submitting 2 bios for the same extent).
+ */
+ if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
+ prev_em_start && *prev_em_start != (u64)-1 &&
+ *prev_em_start != em->orig_start)
+ force_bio_submit = true;
+
+ if (prev_em_start)
+ *prev_em_start = em->orig_start;
+
free_extent_map(em);
em = NULL;
bdev, bio, pnr,
end_bio_extent_readpage, mirror_num,
*bio_flags,
- this_bio_flag);
+ this_bio_flag,
+ force_bio_submit);
if (!ret) {
nr++;
*bio_flags = this_bio_flag;
get_extent_t *get_extent,
struct extent_map **em_cached,
struct bio **bio, int mirror_num,
- unsigned long *bio_flags, int rw)
+ unsigned long *bio_flags, int rw,
+ u64 *prev_em_start)
{
struct inode *inode;
struct btrfs_ordered_extent *ordered;
for (index = 0; index < nr_pages; index++) {
__do_readpage(tree, pages[index], get_extent, em_cached, bio,
- mirror_num, bio_flags, rw);
+ mirror_num, bio_flags, rw, prev_em_start);
page_cache_release(pages[index]);
}
}
int nr_pages, get_extent_t *get_extent,
struct extent_map **em_cached,
struct bio **bio, int mirror_num,
- unsigned long *bio_flags, int rw)
+ unsigned long *bio_flags, int rw,
+ u64 *prev_em_start)
{
u64 start = 0;
u64 end = 0;
index - first_index, start,
end, get_extent, em_cached,
bio, mirror_num, bio_flags,
- rw);
+ rw, prev_em_start);
start = page_start;
end = start + PAGE_CACHE_SIZE - 1;
first_index = index;
__do_contiguous_readpages(tree, &pages[first_index],
index - first_index, start,
end, get_extent, em_cached, bio,
- mirror_num, bio_flags, rw);
+ mirror_num, bio_flags, rw,
+ prev_em_start);
}
static int __extent_read_full_page(struct extent_io_tree *tree,
}
ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
- bio_flags, rw);
+ bio_flags, rw, NULL);
return ret;
}
int ret;
ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
- &bio_flags, READ);
+ &bio_flags, READ, NULL);
if (bio)
ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
return ret;
sector, iosize, pg_offset,
bdev, &epd->bio, max_nr,
end_bio_extent_writepage,
- 0, 0, 0);
+ 0, 0, 0, false);
if (ret)
SetPageError(page);
}
ret = submit_extent_page(rw, tree, wbc, p, offset >> 9,
PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
-1, end_bio_extent_buffer_writepage,
- 0, epd->bio_flags, bio_flags);
+ 0, epd->bio_flags, bio_flags, false);
epd->bio_flags = bio_flags;
if (ret) {
set_btree_ioerr(p);
struct page *page;
struct extent_map *em_cached = NULL;
int nr = 0;
+ u64 prev_em_start = (u64)-1;
for (page_idx = 0; page_idx < nr_pages; page_idx++) {
page = list_entry(pages->prev, struct page, lru);
if (nr < ARRAY_SIZE(pagepool))
continue;
__extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
- &bio, 0, &bio_flags, READ);
+ &bio, 0, &bio_flags, READ, &prev_em_start);
nr = 0;
}
if (nr)
__extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
- &bio, 0, &bio_flags, READ);
+ &bio, 0, &bio_flags, READ, &prev_em_start);
if (em_cached)
free_extent_map(em_cached);
goto no_delete;
}
/* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
- btrfs_wait_ordered_range(inode, 0, (u64)-1);
+ if (!special_file(inode->i_mode))
+ btrfs_wait_ordered_range(inode, 0, (u64)-1);
btrfs_free_io_failure_record(inode, 0, (u64)-1);
return em;
}
+struct btrfs_dio_data {
+ u64 outstanding_extents;
+ u64 reserve;
+};
static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
struct buffer_head *bh_result, int create)
struct extent_map *em;
struct btrfs_root *root = BTRFS_I(inode)->root;
struct extent_state *cached_state = NULL;
+ struct btrfs_dio_data *dio_data = NULL;
u64 start = iblock << inode->i_blkbits;
u64 lockstart, lockend;
u64 len = bh_result->b_size;
- u64 *outstanding_extents = NULL;
int unlock_bits = EXTENT_LOCKED;
int ret = 0;
* that anything that needs to check if there's a transction doesn't get
* confused.
*/
- outstanding_extents = current->journal_info;
+ dio_data = current->journal_info;
current->journal_info = NULL;
}
* within our reservation, otherwise we need to adjust our inode
* counter appropriately.
*/
- if (*outstanding_extents) {
- (*outstanding_extents)--;
+ if (dio_data->outstanding_extents) {
+ (dio_data->outstanding_extents)--;
} else {
spin_lock(&BTRFS_I(inode)->lock);
BTRFS_I(inode)->outstanding_extents++;
spin_unlock(&BTRFS_I(inode)->lock);
}
- current->journal_info = outstanding_extents;
btrfs_free_reserved_data_space(inode, len);
- set_bit(BTRFS_INODE_DIO_READY, &BTRFS_I(inode)->runtime_flags);
+ WARN_ON(dio_data->reserve < len);
+ dio_data->reserve -= len;
+ current->journal_info = dio_data;
}
/*
unlock_err:
clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
unlock_bits, 1, 0, &cached_state, GFP_NOFS);
- if (outstanding_extents)
- current->journal_info = outstanding_extents;
+ if (dio_data)
+ current->journal_info = dio_data;
return ret;
}
{
struct file *file = iocb->ki_filp;
struct inode *inode = file->f_mapping->host;
- u64 outstanding_extents = 0;
+ struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_dio_data dio_data = { 0 };
size_t count = 0;
int flags = 0;
bool wakeup = true;
ret = btrfs_delalloc_reserve_space(inode, count);
if (ret)
goto out;
- outstanding_extents = div64_u64(count +
+ dio_data.outstanding_extents = div64_u64(count +
BTRFS_MAX_EXTENT_SIZE - 1,
BTRFS_MAX_EXTENT_SIZE);
* do the accounting properly if we go over the number we
* originally calculated. Abuse current->journal_info for this.
*/
- current->journal_info = &outstanding_extents;
+ dio_data.reserve = round_up(count, root->sectorsize);
+ current->journal_info = &dio_data;
} else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
&BTRFS_I(inode)->runtime_flags)) {
inode_dio_end(inode);
if (iov_iter_rw(iter) == WRITE) {
current->journal_info = NULL;
if (ret < 0 && ret != -EIOCBQUEUED) {
- /*
- * If the error comes from submitting stage,
- * btrfs_get_blocsk_direct() has free'd data space,
- * and metadata space will be handled by
- * finish_ordered_fn, don't do that again to make
- * sure bytes_may_use is correct.
- */
- if (!test_and_clear_bit(BTRFS_INODE_DIO_READY,
- &BTRFS_I(inode)->runtime_flags))
- btrfs_delalloc_release_space(inode, count);
+ if (dio_data.reserve)
+ btrfs_delalloc_release_space(inode,
+ dio_data.reserve);
} else if (ret >= 0 && (size_t)ret < count)
btrfs_delalloc_release_space(inode,
count - (size_t)ret);
/*
* We know that it is or will be overwritten. Check this now.
* The current inode being processed might have been the one that caused
- * inode 'ino' to be orphanized, therefore ow_inode can actually be the
- * same as sctx->send_progress.
+ * inode 'ino' to be orphanized, therefore check if ow_inode matches
+ * the current inode being processed.
*/
- if (ow_inode <= sctx->send_progress)
+ if ((ow_inode < sctx->send_progress) ||
+ (ino != sctx->cur_ino && ow_inode == sctx->cur_ino &&
+ gen == sctx->cur_inode_gen))
ret = 1;
else
ret = 0;
* groups on disk until we're mounted read-write again
* unless we clean them up here.
*/
- mutex_lock(&root->fs_info->cleaner_mutex);
btrfs_delete_unused_bgs(fs_info);
- mutex_unlock(&root->fs_info->cleaner_mutex);
btrfs_dev_replace_suspend_for_unmount(fs_info);
btrfs_scrub_cancel(fs_info);
btrfs_unpin_free_ino(root);
clear_btree_io_tree(&root->dirty_log_pages);
}
+
+ /* We can free old roots now. */
+ spin_lock(&trans->dropped_roots_lock);
+ while (!list_empty(&trans->dropped_roots)) {
+ root = list_first_entry(&trans->dropped_roots,
+ struct btrfs_root, root_list);
+ list_del_init(&root->root_list);
+ spin_unlock(&trans->dropped_roots_lock);
+ btrfs_drop_and_free_fs_root(fs_info, root);
+ spin_lock(&trans->dropped_roots_lock);
+ }
+ spin_unlock(&trans->dropped_roots_lock);
up_write(&fs_info->commit_root_sem);
}
INIT_LIST_HEAD(&cur_trans->pending_ordered);
INIT_LIST_HEAD(&cur_trans->dirty_bgs);
INIT_LIST_HEAD(&cur_trans->io_bgs);
+ INIT_LIST_HEAD(&cur_trans->dropped_roots);
mutex_init(&cur_trans->cache_write_mutex);
cur_trans->num_dirty_bgs = 0;
spin_lock_init(&cur_trans->dirty_bgs_lock);
INIT_LIST_HEAD(&cur_trans->deleted_bgs);
spin_lock_init(&cur_trans->deleted_bgs_lock);
+ spin_lock_init(&cur_trans->dropped_roots_lock);
list_add_tail(&cur_trans->list, &fs_info->trans_list);
extent_io_tree_init(&cur_trans->dirty_pages,
fs_info->btree_inode->i_mapping);
}
+void btrfs_add_dropped_root(struct btrfs_trans_handle *trans,
+ struct btrfs_root *root)
+{
+ struct btrfs_transaction *cur_trans = trans->transaction;
+
+ /* Add ourselves to the transaction dropped list */
+ spin_lock(&cur_trans->dropped_roots_lock);
+ list_add_tail(&root->root_list, &cur_trans->dropped_roots);
+ spin_unlock(&cur_trans->dropped_roots_lock);
+
+ /* Make sure we don't try to update the root at commit time */
+ spin_lock(&root->fs_info->fs_roots_radix_lock);
+ radix_tree_tag_clear(&root->fs_info->fs_roots_radix,
+ (unsigned long)root->root_key.objectid,
+ BTRFS_ROOT_TRANS_TAG);
+ spin_unlock(&root->fs_info->fs_roots_radix_lock);
+}
+
int btrfs_record_root_in_trans(struct btrfs_trans_handle *trans,
struct btrfs_root *root)
{
h->delayed_ref_elem.seq = 0;
h->type = type;
h->allocating_chunk = false;
+ h->can_flush_pending_bgs = true;
h->reloc_reserved = false;
h->sync = false;
INIT_LIST_HEAD(&h->qgroup_ref_list);
struct list_head switch_commits;
struct list_head dirty_bgs;
struct list_head io_bgs;
+ struct list_head dropped_roots;
u64 num_dirty_bgs;
/*
spinlock_t dirty_bgs_lock;
struct list_head deleted_bgs;
spinlock_t deleted_bgs_lock;
+ spinlock_t dropped_roots_lock;
struct btrfs_delayed_ref_root delayed_refs;
int aborted;
int dirty_bg_run;
short aborted;
short adding_csums;
bool allocating_chunk;
+ bool can_flush_pending_bgs;
bool reloc_reserved;
bool sync;
unsigned int type;
int btrfs_transaction_in_commit(struct btrfs_fs_info *info);
void btrfs_put_transaction(struct btrfs_transaction *transaction);
void btrfs_apply_pending_changes(struct btrfs_fs_info *fs_info);
-
+void btrfs_add_dropped_root(struct btrfs_trans_handle *trans,
+ struct btrfs_root *root);
#endif
return 0;
}
+/* Server has provided av pairs/target info in the type 2 challenge
+ * packet and we have plucked it and stored within smb session.
+ * We parse that blob here to find the server given timestamp
+ * as part of ntlmv2 authentication (or local current time as
+ * default in case of failure)
+ */
+static __le64
+find_timestamp(struct cifs_ses *ses)
+{
+ unsigned int attrsize;
+ unsigned int type;
+ unsigned int onesize = sizeof(struct ntlmssp2_name);
+ unsigned char *blobptr;
+ unsigned char *blobend;
+ struct ntlmssp2_name *attrptr;
+
+ if (!ses->auth_key.len || !ses->auth_key.response)
+ return 0;
+
+ blobptr = ses->auth_key.response;
+ blobend = blobptr + ses->auth_key.len;
+
+ while (blobptr + onesize < blobend) {
+ attrptr = (struct ntlmssp2_name *) blobptr;
+ type = le16_to_cpu(attrptr->type);
+ if (type == NTLMSSP_AV_EOL)
+ break;
+ blobptr += 2; /* advance attr type */
+ attrsize = le16_to_cpu(attrptr->length);
+ blobptr += 2; /* advance attr size */
+ if (blobptr + attrsize > blobend)
+ break;
+ if (type == NTLMSSP_AV_TIMESTAMP) {
+ if (attrsize == sizeof(u64))
+ return *((__le64 *)blobptr);
+ }
+ blobptr += attrsize; /* advance attr value */
+ }
+
+ return cpu_to_le64(cifs_UnixTimeToNT(CURRENT_TIME));
+}
+
static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash,
const struct nls_table *nls_cp)
{
struct ntlmv2_resp *ntlmv2;
char ntlmv2_hash[16];
unsigned char *tiblob = NULL; /* target info blob */
+ __le64 rsp_timestamp;
if (ses->server->negflavor == CIFS_NEGFLAVOR_EXTENDED) {
if (!ses->domainName) {
}
}
+ /* Must be within 5 minutes of the server (or in range +/-2h
+ * in case of Mac OS X), so simply carry over server timestamp
+ * (as Windows 7 does)
+ */
+ rsp_timestamp = find_timestamp(ses);
+
baselen = CIFS_SESS_KEY_SIZE + sizeof(struct ntlmv2_resp);
tilen = ses->auth_key.len;
tiblob = ses->auth_key.response;
(ses->auth_key.response + CIFS_SESS_KEY_SIZE);
ntlmv2->blob_signature = cpu_to_le32(0x00000101);
ntlmv2->reserved = 0;
- /* Must be within 5 minutes of the server */
- ntlmv2->time = cpu_to_le64(cifs_UnixTimeToNT(CURRENT_TIME));
+ ntlmv2->time = rsp_timestamp;
+
get_random_bytes(&ntlmv2->client_chal, sizeof(ntlmv2->client_chal));
ntlmv2->reserved2 = 0;
static void
cifs_show_security(struct seq_file *s, struct cifs_ses *ses)
{
- if (ses->sectype == Unspecified)
+ if (ses->sectype == Unspecified) {
+ if (ses->user_name == NULL)
+ seq_puts(s, ",sec=none");
return;
+ }
seq_puts(s, ",sec=");
extern const struct export_operations cifs_export_ops;
#endif /* CONFIG_CIFS_NFSD_EXPORT */
-#define CIFS_VERSION "2.07"
+#define CIFS_VERSION "2.08"
#endif /* _CIFSFS_H */
struct tcon_link *tlink = NULL;
struct cifs_tcon *tcon = NULL;
struct TCP_Server_Info *server;
- struct cifs_io_parms io_parms;
/*
* To avoid spurious oplock breaks from server, in the case of
rc = -ENOSYS;
cifsFileInfo_put(open_file);
cifs_dbg(FYI, "SetFSize for attrs rc = %d\n", rc);
- if ((rc == -EINVAL) || (rc == -EOPNOTSUPP)) {
- unsigned int bytes_written;
-
- io_parms.netfid = open_file->fid.netfid;
- io_parms.pid = open_file->pid;
- io_parms.tcon = tcon;
- io_parms.offset = 0;
- io_parms.length = attrs->ia_size;
- rc = CIFSSMBWrite(xid, &io_parms, &bytes_written,
- NULL, NULL, 1);
- cifs_dbg(FYI, "Wrt seteof rc %d\n", rc);
- }
} else
rc = -EINVAL;
else
rc = -ENOSYS;
cifs_dbg(FYI, "SetEOF by path (setattrs) rc = %d\n", rc);
- if ((rc == -EINVAL) || (rc == -EOPNOTSUPP)) {
- __u16 netfid;
- int oplock = 0;
- rc = SMBLegacyOpen(xid, tcon, full_path, FILE_OPEN,
- GENERIC_WRITE, CREATE_NOT_DIR, &netfid,
- &oplock, NULL, cifs_sb->local_nls,
- cifs_remap(cifs_sb));
- if (rc == 0) {
- unsigned int bytes_written;
-
- io_parms.netfid = netfid;
- io_parms.pid = current->tgid;
- io_parms.tcon = tcon;
- io_parms.offset = 0;
- io_parms.length = attrs->ia_size;
- rc = CIFSSMBWrite(xid, &io_parms, &bytes_written, NULL,
- NULL, 1);
- cifs_dbg(FYI, "wrt seteof rc %d\n", rc);
- CIFSSMBClose(xid, tcon, netfid);
- }
- }
if (tlink)
cifs_put_tlink(tlink);
goto out_drop_write;
}
+ if (src_file.file->f_op->unlocked_ioctl != cifs_ioctl) {
+ rc = -EBADF;
+ cifs_dbg(VFS, "src file seems to be from a different filesystem type\n");
+ goto out_fput;
+ }
+
if ((!src_file.file->private_data) || (!dst_file->private_data)) {
rc = -EBADF;
cifs_dbg(VFS, "missing cifsFileInfo on copy range src file\n");
break;
default:
server->echoes = true;
- server->oplocks = true;
+ if (enable_oplocks) {
+ server->oplocks = true;
+ server->oplock_credits = 1;
+ } else
+ server->oplocks = false;
+
server->echo_credits = 1;
- server->oplock_credits = 1;
}
server->credits -= server->echo_credits + server->oplock_credits;
return 0;
#include "smb2status.h"
#include "smb2glob.h"
#include "cifspdu.h"
+#include "cifs_spnego.h"
/*
* The following table defines the expected "StructureSize" of SMB2 requests
cifs_dbg(FYI, "missing security blob on negprot\n");
rc = cifs_enable_signing(server, ses->sign);
-#ifdef CONFIG_SMB2_ASN1 /* BB REMOVEME when updated asn1.c ready */
if (rc)
goto neg_exit;
- if (blob_length)
+ if (blob_length) {
rc = decode_negTokenInit(security_blob, blob_length, server);
- if (rc == 1)
- rc = 0;
- else if (rc == 0) {
- rc = -EIO;
- goto neg_exit;
+ if (rc == 1)
+ rc = 0;
+ else if (rc == 0)
+ rc = -EIO;
}
-#endif
-
neg_exit:
free_rsp_buf(resp_buftype, rsp);
return rc;
__le32 phase = NtLmNegotiate; /* NTLMSSP, if needed, is multistage */
struct TCP_Server_Info *server = ses->server;
u16 blob_length = 0;
- char *security_blob;
+ struct key *spnego_key = NULL;
+ char *security_blob = NULL;
char *ntlmssp_blob = NULL;
bool use_spnego = false; /* else use raw ntlmssp */
ses->ntlmssp->sesskey_per_smbsess = true;
/* FIXME: allow for other auth types besides NTLMSSP (e.g. krb5) */
- ses->sectype = RawNTLMSSP;
+ if (ses->sectype != Kerberos && ses->sectype != RawNTLMSSP)
+ ses->sectype = RawNTLMSSP;
ssetup_ntlmssp_authenticate:
if (phase == NtLmChallenge)
iov[0].iov_base = (char *)req;
/* 4 for rfc1002 length field and 1 for pad */
iov[0].iov_len = get_rfc1002_length(req) + 4 - 1;
- if (phase == NtLmNegotiate) {
+
+ if (ses->sectype == Kerberos) {
+#ifdef CONFIG_CIFS_UPCALL
+ struct cifs_spnego_msg *msg;
+
+ spnego_key = cifs_get_spnego_key(ses);
+ if (IS_ERR(spnego_key)) {
+ rc = PTR_ERR(spnego_key);
+ spnego_key = NULL;
+ goto ssetup_exit;
+ }
+
+ msg = spnego_key->payload.data;
+ /*
+ * check version field to make sure that cifs.upcall is
+ * sending us a response in an expected form
+ */
+ if (msg->version != CIFS_SPNEGO_UPCALL_VERSION) {
+ cifs_dbg(VFS,
+ "bad cifs.upcall version. Expected %d got %d",
+ CIFS_SPNEGO_UPCALL_VERSION, msg->version);
+ rc = -EKEYREJECTED;
+ goto ssetup_exit;
+ }
+ ses->auth_key.response = kmemdup(msg->data, msg->sesskey_len,
+ GFP_KERNEL);
+ if (!ses->auth_key.response) {
+ cifs_dbg(VFS,
+ "Kerberos can't allocate (%u bytes) memory",
+ msg->sesskey_len);
+ rc = -ENOMEM;
+ goto ssetup_exit;
+ }
+ ses->auth_key.len = msg->sesskey_len;
+ blob_length = msg->secblob_len;
+ iov[1].iov_base = msg->data + msg->sesskey_len;
+ iov[1].iov_len = blob_length;
+#else
+ rc = -EOPNOTSUPP;
+ goto ssetup_exit;
+#endif /* CONFIG_CIFS_UPCALL */
+ } else if (phase == NtLmNegotiate) { /* if not krb5 must be ntlmssp */
ntlmssp_blob = kmalloc(sizeof(struct _NEGOTIATE_MESSAGE),
GFP_KERNEL);
if (ntlmssp_blob == NULL) {
/* with raw NTLMSSP we don't encapsulate in SPNEGO */
security_blob = ntlmssp_blob;
}
+ iov[1].iov_base = security_blob;
+ iov[1].iov_len = blob_length;
} else if (phase == NtLmAuthenticate) {
req->hdr.SessionId = ses->Suid;
ntlmssp_blob = kzalloc(sizeof(struct _NEGOTIATE_MESSAGE) + 500,
} else {
security_blob = ntlmssp_blob;
}
+ iov[1].iov_base = security_blob;
+ iov[1].iov_len = blob_length;
} else {
cifs_dbg(VFS, "illegal ntlmssp phase\n");
rc = -EIO;
cpu_to_le16(sizeof(struct smb2_sess_setup_req) -
1 /* pad */ - 4 /* rfc1001 len */);
req->SecurityBufferLength = cpu_to_le16(blob_length);
- iov[1].iov_base = security_blob;
- iov[1].iov_len = blob_length;
inc_rfc1001_len(req, blob_length - 1 /* pad */);
kfree(security_blob);
rsp = (struct smb2_sess_setup_rsp *)iov[0].iov_base;
+ ses->Suid = rsp->hdr.SessionId;
if (resp_buftype != CIFS_NO_BUFFER &&
rsp->hdr.Status == STATUS_MORE_PROCESSING_REQUIRED) {
if (phase != NtLmNegotiate) {
/* NTLMSSP Negotiate sent now processing challenge (response) */
phase = NtLmChallenge; /* process ntlmssp challenge */
rc = 0; /* MORE_PROCESSING is not an error here but expected */
- ses->Suid = rsp->hdr.SessionId;
rc = decode_ntlmssp_challenge(rsp->Buffer,
le16_to_cpu(rsp->SecurityBufferLength), ses);
}
kfree(ses->auth_key.response);
ses->auth_key.response = NULL;
}
+ if (spnego_key) {
+ key_invalidate(spnego_key);
+ key_put(spnego_key);
+ }
kfree(ses->ntlmssp);
return rc;
if (tcon && tcon->bad_network_name)
return -ENOENT;
+ if ((tcon && tcon->seal) &&
+ ((ses->server->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION) == 0)) {
+ cifs_dbg(VFS, "encryption requested but no server support");
+ return -EOPNOTSUPP;
+ }
+
unc_path = kmalloc(MAX_SHARENAME_LENGTH * 2, GFP_KERNEL);
if (unc_path == NULL)
return -ENOMEM;
((tcon->share_flags & SHI1005_FLAGS_DFS) == 0))
cifs_dbg(VFS, "DFS capability contradicts DFS flag\n");
init_copy_chunk_defaults(tcon);
+ if (tcon->share_flags & SHI1005_FLAGS_ENCRYPT_DATA)
+ cifs_dbg(VFS, "Encrypted shares not supported");
if (tcon->ses->server->ops->validate_negotiate)
rc = tcon->ses->server->ops->validate_negotiate(xid, tcon);
tcon_exit:
size_t len;
if (pos == max) {
unsigned blkbits = inode->i_blkbits;
- sector_t block = pos >> blkbits;
+ long page = pos >> PAGE_SHIFT;
+ sector_t block = page << (PAGE_SHIFT - blkbits);
unsigned first = pos - (block << blkbits);
long size;
if (!buffer_size_valid(&bh) || bh.b_size < PMD_SIZE)
goto fallback;
+ sector = bh.b_blocknr << (blkbits - 9);
+
if (buffer_unwritten(&bh) || buffer_new(&bh)) {
int i;
+
+ length = bdev_direct_access(bh.b_bdev, sector, &kaddr, &pfn,
+ bh.b_size);
+ if (length < 0) {
+ result = VM_FAULT_SIGBUS;
+ goto out;
+ }
+ if ((length < PMD_SIZE) || (pfn & PG_PMD_COLOUR))
+ goto fallback;
+
for (i = 0; i < PTRS_PER_PMD; i++)
clear_pmem(kaddr + i * PAGE_SIZE, PAGE_SIZE);
wmb_pmem();
result = VM_FAULT_NOPAGE;
spin_unlock(ptl);
} else {
- sector = bh.b_blocknr << (blkbits - 9);
length = bdev_direct_access(bh.b_bdev, sector, &kaddr, &pfn,
bh.b_size);
if (length < 0) {
wbc_detach_inode(&wbc);
work->nr_pages -= write_chunk - wbc.nr_to_write;
wrote += write_chunk - wbc.nr_to_write;
+
+ if (need_resched()) {
+ /*
+ * We're trying to balance between building up a nice
+ * long list of IOs to improve our merge rate, and
+ * getting those IOs out quickly for anyone throttling
+ * in balance_dirty_pages(). cond_resched() doesn't
+ * unplug, so get our IOs out the door before we
+ * give up the CPU.
+ */
+ blk_flush_plug(current);
+ cond_resched();
+ }
+
+
spin_lock(&wb->list_lock);
spin_lock(&inode->i_lock);
if (!(inode->i_state & I_DIRTY_ALL))
requeue_inode(inode, wb, &wbc);
inode_sync_complete(inode);
spin_unlock(&inode->i_lock);
- cond_resched_lock(&wb->list_lock);
+
/*
* bail out to wb_writeback() often enough to check
* background threshold and other termination conditions.
negative = d_is_negative(dentry);
if (read_seqcount_retry(&dentry->d_seq, seq))
return -ECHILD;
- if (negative)
- return -ENOENT;
/*
* This sequence count validates that the parent had no
goto unlazy;
}
}
+ /*
+ * Note: do negative dentry check after revalidation in
+ * case that drops it.
+ */
+ if (negative)
+ return -ENOENT;
path->mnt = mnt;
path->dentry = dentry;
if (likely(__follow_mount_rcu(nd, path, inode, seqp)))
return status;
}
-static int nfs_delegation_claim_opens(struct inode *inode, const nfs4_stateid *stateid)
+static int nfs_delegation_claim_opens(struct inode *inode,
+ const nfs4_stateid *stateid, fmode_t type)
{
struct nfs_inode *nfsi = NFS_I(inode);
struct nfs_open_context *ctx;
/* Block nfs4_proc_unlck */
mutex_lock(&sp->so_delegreturn_mutex);
seq = raw_seqcount_begin(&sp->so_reclaim_seqcount);
- err = nfs4_open_delegation_recall(ctx, state, stateid);
+ err = nfs4_open_delegation_recall(ctx, state, stateid, type);
if (!err)
err = nfs_delegation_claim_locks(ctx, state, stateid);
if (!err && read_seqcount_retry(&sp->so_reclaim_seqcount, seq))
do {
if (test_bit(NFS_DELEGATION_REVOKED, &delegation->flags))
break;
- err = nfs_delegation_claim_opens(inode, &delegation->stateid);
+ err = nfs_delegation_claim_opens(inode, &delegation->stateid,
+ delegation->type);
if (!issync || err != -EAGAIN)
break;
/*
/* NFSv4 delegation-related procedures */
int nfs4_proc_delegreturn(struct inode *inode, struct rpc_cred *cred, const nfs4_stateid *stateid, int issync);
-int nfs4_open_delegation_recall(struct nfs_open_context *ctx, struct nfs4_state *state, const nfs4_stateid *stateid);
+int nfs4_open_delegation_recall(struct nfs_open_context *ctx, struct nfs4_state *state, const nfs4_stateid *stateid, fmode_t type);
int nfs4_lock_delegation_recall(struct file_lock *fl, struct nfs4_state *state, const nfs4_stateid *stateid);
bool nfs4_copy_delegation_stateid(nfs4_stateid *dst, struct inode *inode, fmode_t flags);
struct nfs_writeverf *verfp = &dreq->verf;
#ifdef CONFIG_NFS_V4_1
- if (ds_clp) {
- /* pNFS is in use, use the DS verf */
+ /*
+ * pNFS is in use, use the DS verf except commit_through_mds is set
+ * for layout segment where nbuckets is zero.
+ */
+ if (ds_clp && dreq->ds_cinfo.nbuckets > 0) {
if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)
verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;
else
goto out;
}
-static void filelayout_free_fh_array(struct nfs4_filelayout_segment *fl)
+static void _filelayout_free_lseg(struct nfs4_filelayout_segment *fl)
{
int i;
- for (i = 0; i < fl->num_fh; i++) {
- if (!fl->fh_array[i])
- break;
- kfree(fl->fh_array[i]);
+ if (fl->fh_array) {
+ for (i = 0; i < fl->num_fh; i++) {
+ if (!fl->fh_array[i])
+ break;
+ kfree(fl->fh_array[i]);
+ }
+ kfree(fl->fh_array);
}
- kfree(fl->fh_array);
- fl->fh_array = NULL;
-}
-
-static void
-_filelayout_free_lseg(struct nfs4_filelayout_segment *fl)
-{
- filelayout_free_fh_array(fl);
kfree(fl);
}
/* Do we want to use a mempool here? */
fl->fh_array[i] = kmalloc(sizeof(struct nfs_fh), gfp_flags);
if (!fl->fh_array[i])
- goto out_err_free;
+ goto out_err;
p = xdr_inline_decode(&stream, 4);
if (unlikely(!p))
- goto out_err_free;
+ goto out_err;
fl->fh_array[i]->size = be32_to_cpup(p++);
if (sizeof(struct nfs_fh) < fl->fh_array[i]->size) {
printk(KERN_ERR "NFS: Too big fh %d received %d\n",
i, fl->fh_array[i]->size);
- goto out_err_free;
+ goto out_err;
}
p = xdr_inline_decode(&stream, fl->fh_array[i]->size);
if (unlikely(!p))
- goto out_err_free;
+ goto out_err;
memcpy(fl->fh_array[i]->data, p, fl->fh_array[i]->size);
dprintk("DEBUG: %s: fh len %d\n", __func__,
fl->fh_array[i]->size);
__free_page(scratch);
return 0;
-out_err_free:
- filelayout_free_fh_array(fl);
out_err:
__free_page(scratch);
return -EIO;
{
struct nfs_server *server = NFS_SERVER(file_inode(filep));
struct nfs4_exception exception = { };
- int err;
+ loff_t err;
do {
err = _nfs42_proc_llseek(filep, offset, whence);
+ if (err >= 0)
+ break;
if (err == -ENOTSUPP)
return -EOPNOTSUPP;
err = nfs4_handle_exception(server, err, &exception);
return ret;
}
+static bool nfs4_mode_match_open_stateid(struct nfs4_state *state,
+ fmode_t fmode)
+{
+ switch(fmode & (FMODE_READ|FMODE_WRITE)) {
+ case FMODE_READ|FMODE_WRITE:
+ return state->n_rdwr != 0;
+ case FMODE_WRITE:
+ return state->n_wronly != 0;
+ case FMODE_READ:
+ return state->n_rdonly != 0;
+ }
+ WARN_ON_ONCE(1);
+ return false;
+}
+
static int can_open_cached(struct nfs4_state *state, fmode_t mode, int open_mode)
{
int ret = 0;
if (delegation)
delegation_flags = delegation->flags;
rcu_read_unlock();
- if (data->o_arg.claim == NFS4_OPEN_CLAIM_DELEGATE_CUR) {
+ switch (data->o_arg.claim) {
+ default:
+ break;
+ case NFS4_OPEN_CLAIM_DELEGATE_CUR:
+ case NFS4_OPEN_CLAIM_DELEG_CUR_FH:
pr_err_ratelimited("NFS: Broken NFSv4 server %s is "
"returning a delegation for "
"OPEN(CLAIM_DELEGATE_CUR)\n",
clp->cl_hostname);
- } else if ((delegation_flags & 1UL<<NFS_DELEGATION_NEED_RECLAIM) == 0)
+ return;
+ }
+ if ((delegation_flags & 1UL<<NFS_DELEGATION_NEED_RECLAIM) == 0)
nfs_inode_set_delegation(state->inode,
data->owner->so_cred,
&data->o_res);
return opendata;
}
-static int nfs4_open_recover_helper(struct nfs4_opendata *opendata, fmode_t fmode, struct nfs4_state **res)
+static int nfs4_open_recover_helper(struct nfs4_opendata *opendata,
+ fmode_t fmode)
{
struct nfs4_state *newstate;
int ret;
- if ((opendata->o_arg.claim == NFS4_OPEN_CLAIM_DELEGATE_CUR ||
- opendata->o_arg.claim == NFS4_OPEN_CLAIM_DELEG_CUR_FH) &&
- (opendata->o_arg.u.delegation_type & fmode) != fmode)
- /* This mode can't have been delegated, so we must have
- * a valid open_stateid to cover it - not need to reclaim.
- */
+ if (!nfs4_mode_match_open_stateid(opendata->state, fmode))
return 0;
opendata->o_arg.open_flags = 0;
opendata->o_arg.fmode = fmode;
newstate = nfs4_opendata_to_nfs4_state(opendata);
if (IS_ERR(newstate))
return PTR_ERR(newstate);
+ if (newstate != opendata->state)
+ ret = -ESTALE;
nfs4_close_state(newstate, fmode);
- *res = newstate;
- return 0;
+ return ret;
}
static int nfs4_open_recover(struct nfs4_opendata *opendata, struct nfs4_state *state)
{
- struct nfs4_state *newstate;
int ret;
/* Don't trigger recovery in nfs_test_and_clear_all_open_stateid */
clear_bit(NFS_DELEGATED_STATE, &state->flags);
clear_bit(NFS_OPEN_STATE, &state->flags);
smp_rmb();
- if (state->n_rdwr != 0) {
- ret = nfs4_open_recover_helper(opendata, FMODE_READ|FMODE_WRITE, &newstate);
- if (ret != 0)
- return ret;
- if (newstate != state)
- return -ESTALE;
- }
- if (state->n_wronly != 0) {
- ret = nfs4_open_recover_helper(opendata, FMODE_WRITE, &newstate);
- if (ret != 0)
- return ret;
- if (newstate != state)
- return -ESTALE;
- }
- if (state->n_rdonly != 0) {
- ret = nfs4_open_recover_helper(opendata, FMODE_READ, &newstate);
- if (ret != 0)
- return ret;
- if (newstate != state)
- return -ESTALE;
- }
+ ret = nfs4_open_recover_helper(opendata, FMODE_READ|FMODE_WRITE);
+ if (ret != 0)
+ return ret;
+ ret = nfs4_open_recover_helper(opendata, FMODE_WRITE);
+ if (ret != 0)
+ return ret;
+ ret = nfs4_open_recover_helper(opendata, FMODE_READ);
+ if (ret != 0)
+ return ret;
/*
* We may have performed cached opens for all three recoveries.
* Check if we need to update the current stateid.
return err;
}
-int nfs4_open_delegation_recall(struct nfs_open_context *ctx, struct nfs4_state *state, const nfs4_stateid *stateid)
+int nfs4_open_delegation_recall(struct nfs_open_context *ctx,
+ struct nfs4_state *state, const nfs4_stateid *stateid,
+ fmode_t type)
{
struct nfs_server *server = NFS_SERVER(state->inode);
struct nfs4_opendata *opendata;
- int err;
+ int err = 0;
opendata = nfs4_open_recoverdata_alloc(ctx, state,
NFS4_OPEN_CLAIM_DELEG_CUR_FH);
if (IS_ERR(opendata))
return PTR_ERR(opendata);
nfs4_stateid_copy(&opendata->o_arg.u.delegation, stateid);
- err = nfs4_open_recover(opendata, state);
+ write_seqlock(&state->seqlock);
+ nfs4_stateid_copy(&state->stateid, &state->open_stateid);
+ write_sequnlock(&state->seqlock);
+ clear_bit(NFS_DELEGATED_STATE, &state->flags);
+ switch (type & (FMODE_READ|FMODE_WRITE)) {
+ case FMODE_READ|FMODE_WRITE:
+ case FMODE_WRITE:
+ err = nfs4_open_recover_helper(opendata, FMODE_READ|FMODE_WRITE);
+ if (err)
+ break;
+ err = nfs4_open_recover_helper(opendata, FMODE_WRITE);
+ if (err)
+ break;
+ case FMODE_READ:
+ err = nfs4_open_recover_helper(opendata, FMODE_READ);
+ }
nfs4_opendata_put(opendata);
return nfs4_handle_delegation_recall_error(server, state, stateid, err);
}
data->rpc_done = 0;
data->rpc_status = 0;
data->timestamp = jiffies;
+ if (data->is_recover)
+ nfs4_set_sequence_privileged(&data->c_arg.seq_args);
task = rpc_run_task(&task_setup_data);
if (IS_ERR(task))
return PTR_ERR(task);
return err;
}
+static bool
+nfs4_wait_on_layoutreturn(struct inode *inode, struct rpc_task *task)
+{
+ if (inode == NULL || !nfs_have_layout(inode))
+ return false;
+
+ return pnfs_wait_on_layoutreturn(inode, task);
+}
+
struct nfs4_closedata {
struct inode *inode;
struct nfs4_state *state;
goto out_no_action;
}
+ if (nfs4_wait_on_layoutreturn(inode, task)) {
+ nfs_release_seqid(calldata->arg.seqid);
+ goto out_wait;
+ }
+
if (calldata->arg.fmode == 0)
task->tk_msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLOSE];
if (calldata->roc)
d_data = (struct nfs4_delegreturndata *)data;
+ if (nfs4_wait_on_layoutreturn(d_data->inode, task))
+ return;
+
if (d_data->roc)
pnfs_roc_get_barrier(d_data->inode, &d_data->roc_barrier);
dprintk("%s: NFS4ERR_RECALLCONFLICT waiting %lu\n",
__func__, delay);
rpc_delay(task, delay);
- task->tk_status = 0;
- rpc_restart_call_prepare(task);
- goto out; /* Do not call nfs4_async_handle_error() */
+ /* Do not call nfs4_async_handle_error() */
+ goto out_restart;
}
break;
case -NFS4ERR_EXPIRED:
case -NFS4ERR_BAD_STATEID:
spin_lock(&inode->i_lock);
- lo = NFS_I(inode)->layout;
- if (!lo || list_empty(&lo->plh_segs)) {
+ if (nfs4_stateid_match(&lgp->args.stateid,
+ &lgp->args.ctx->state->stateid)) {
spin_unlock(&inode->i_lock);
/* If the open stateid was bad, then recover it. */
state = lgp->args.ctx->state;
- } else {
+ break;
+ }
+ lo = NFS_I(inode)->layout;
+ if (lo && nfs4_stateid_match(&lgp->args.stateid,
+ &lo->plh_stateid)) {
LIST_HEAD(head);
/*
* Mark the bad layout state as invalid, then retry
* with the current stateid.
*/
+ set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
pnfs_mark_matching_lsegs_invalid(lo, &head, NULL);
spin_unlock(&inode->i_lock);
pnfs_free_lseg_list(&head);
-
- task->tk_status = 0;
- rpc_restart_call_prepare(task);
- }
+ } else
+ spin_unlock(&inode->i_lock);
+ goto out_restart;
}
if (nfs4_async_handle_error(task, server, state, NULL) == -EAGAIN)
- rpc_restart_call_prepare(task);
+ goto out_restart;
out:
dprintk("<-- %s\n", __func__);
return;
+out_restart:
+ task->tk_status = 0;
+ rpc_restart_call_prepare(task);
+ return;
out_overflow:
task->tk_status = -EOVERFLOW;
goto out;
spin_unlock(&state->state_lock);
}
nfs4_put_open_state(state);
- clear_bit(NFS4CLNT_RECLAIM_NOGRACE,
+ clear_bit(NFS_STATE_RECLAIM_NOGRACE,
&state->flags);
spin_lock(&sp->so_lock);
goto restart;
if (!test_and_clear_bit(ops->owner_flag_bit,
&sp->so_flags))
continue;
- atomic_inc(&sp->so_count);
+ if (!atomic_inc_not_zero(&sp->so_count))
+ continue;
spin_unlock(&clp->cl_lock);
rcu_read_unlock();
__entry->flags = flags;
__entry->fmode = (__force unsigned int)ctx->mode;
__entry->dev = ctx->dentry->d_sb->s_dev;
- if (!IS_ERR(state))
+ if (!IS_ERR_OR_NULL(state))
inode = state->inode;
if (inode != NULL) {
__entry->fileid = NFS_FILEID(inode);
* for it without upsetting the slab allocator.
*/
if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
- sizeof(struct page) > PAGE_SIZE)
+ sizeof(struct page *) > PAGE_SIZE)
return 0;
return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
mark_lseg_invalid(lseg, &tmp_list);
found = true;
}
- /* pnfs_prepare_layoutreturn() grabs lo ref and it will be put
- * in pnfs_roc_release(). We don't really send a layoutreturn but
- * still want others to view us like we are sending one!
- *
- * If pnfs_prepare_layoutreturn() fails, it means someone else is doing
- * LAYOUTRETURN, so we proceed like there are no layouts to return.
- *
- * ROC in three conditions:
+ /* ROC in two conditions:
* 1. there are ROC lsegs
* 2. we don't send layoutreturn
- * 3. no others are sending layoutreturn
*/
- if (found && !layoutreturn && pnfs_prepare_layoutreturn(lo))
+ if (found && !layoutreturn) {
+ /* lo ref dropped in pnfs_roc_release() */
+ pnfs_get_layout_hdr(lo);
roc = true;
+ }
out_noroc:
spin_unlock(&ino->i_lock);
spin_unlock(&ino->i_lock);
}
+bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
+{
+ struct nfs_inode *nfsi = NFS_I(ino);
+ struct pnfs_layout_hdr *lo;
+ bool sleep = false;
+
+ /* we might not have grabbed lo reference. so need to check under
+ * i_lock */
+ spin_lock(&ino->i_lock);
+ lo = nfsi->layout;
+ if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
+ sleep = true;
+ spin_unlock(&ino->i_lock);
+
+ if (sleep)
+ rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
+
+ return sleep;
+}
+
/*
* Compare two layout segments for sorting into layout cache.
* We want to preferentially return RW over RO layouts, so ensure those
void pnfs_roc_release(struct inode *ino);
void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier);
+bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task);
void pnfs_set_layoutcommit(struct inode *, struct pnfs_layout_segment *, loff_t);
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data);
int pnfs_layoutcommit_inode(struct inode *inode, bool sync);
{
}
+static inline bool
+pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
+{
+ return false;
+}
+
static inline void set_pnfs_layoutdriver(struct nfs_server *s,
const struct nfs_fh *mntfh, u32 id)
{
{
struct nfs_pgio_mirror *mirror;
+ if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
+ pgio->pg_ops->pg_cleanup(pgio);
+
pgio->pg_ops = &nfs_pgio_rw_ops;
/* read path should never have more than one mirror */
if (!nfs_pageio_add_request(pgio, req)) {
nfs_redirty_request(req);
ret = pgio->pg_error;
- }
+ } else
+ nfs_add_stats(page_file_mapping(page)->host,
+ NFSIOS_WRITEPAGES, 1);
out:
return ret;
}
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
{
- struct inode *inode = page_file_mapping(page)->host;
int ret;
- nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
- nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
-
nfs_pageio_cond_complete(pgio, page_file_index(page));
ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE);
if (ret == -EAGAIN) {
static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc)
{
struct nfs_pageio_descriptor pgio;
+ struct inode *inode = page_file_mapping(page)->host;
int err;
- nfs_pageio_init_write(&pgio, page->mapping->host, wb_priority(wbc),
+ nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
+ nfs_pageio_init_write(&pgio, inode, wb_priority(wbc),
false, &nfs_async_write_completion_ops);
err = nfs_do_writepage(page, wbc, &pgio);
nfs_pageio_complete(&pgio);
return 1;
if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
list_empty_careful(&flctx->flc_posix)))
- return 0;
+ return 1;
/* Check to see if there are whole file write locks */
ret = 0;
{
struct nfs_pgio_mirror *mirror;
+ if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
+ pgio->pg_ops->pg_cleanup(pgio);
+
pgio->pg_ops = &nfs_pgio_rw_ops;
nfs_pageio_stop_mirroring(pgio);
int found, ret;
int set_maybe;
int dispatch_assert = 0;
+ int dispatched = 0;
if (!dlm_grab(dlm))
return DLM_MASTER_RESP_NO;
mlog(ML_ERROR, "failed to dispatch assert master work\n");
response = DLM_MASTER_RESP_ERROR;
dlm_lockres_put(res);
- } else
+ } else {
+ dispatched = 1;
__dlm_lockres_grab_inflight_worker(dlm, res);
+ }
spin_unlock(&res->spinlock);
} else {
if (res)
dlm_lockres_put(res);
}
- dlm_put(dlm);
+ if (!dispatched)
+ dlm_put(dlm);
return response;
}
/* queue up work for dlm_assert_master_worker */
- dlm_grab(dlm); /* get an extra ref for the work item */
dlm_init_work_item(dlm, item, dlm_assert_master_worker, NULL);
item->u.am.lockres = res; /* already have a ref */
/* can optionally ignore node numbers higher than this node */
unsigned int hash;
int master = DLM_LOCK_RES_OWNER_UNKNOWN;
u32 flags = DLM_ASSERT_MASTER_REQUERY;
+ int dispatched = 0;
if (!dlm_grab(dlm)) {
/* since the domain has gone away on this
dlm_put(dlm);
/* sender will take care of this and retry */
return ret;
- } else
+ } else {
+ dispatched = 1;
__dlm_lockres_grab_inflight_worker(dlm, res);
+ }
spin_unlock(&res->spinlock);
} else {
/* put.. incase we are not the master */
}
spin_unlock(&dlm->spinlock);
- dlm_put(dlm);
+ if (!dispatched)
+ dlm_put(dlm);
return master;
}
{
int err;
- mutex_lock(&inode->i_mutex);
err = security_inode_init_security(inode, dentry, qstr,
&init_xattrs, 0);
- mutex_unlock(&inode->i_mutex);
-
if (err) {
struct ubifs_info *c = dentry->i_sb->s_fs_info;
ubifs_err(c, "cannot initialize security for inode %lu, error %d",
* the fault_*wqh.
*/
spin_lock(&ctx->fault_pending_wqh.lock);
- __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, 0, &range);
- __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, &range);
+ __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
+ __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, &range);
spin_unlock(&ctx->fault_pending_wqh.lock);
wake_up_poll(&ctx->fd_wqh, POLLHUP);
spin_lock(&ctx->fault_pending_wqh.lock);
/* wake all in the range and autoremove */
if (waitqueue_active(&ctx->fault_pending_wqh))
- __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, 0,
+ __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
range);
if (waitqueue_active(&ctx->fault_wqh))
- __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, range);
+ __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, range);
spin_unlock(&ctx->fault_pending_wqh.lock);
}
file = anon_inode_getfile("[userfaultfd]", &userfaultfd_fops, ctx,
O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS));
- if (IS_ERR(file))
+ if (IS_ERR(file)) {
+ mmput(ctx->mm);
kmem_cache_free(userfaultfd_ctx_cachep, ctx);
+ }
out:
return file;
}
#include <linux/notifier.h>
-#if defined(CONFIG_ACPI_BUTTON) || defined(CONFIG_ACPI_BUTTON_MODULE)
+#if IS_ENABLED(CONFIG_ACPI_BUTTON)
extern int acpi_lid_notifier_register(struct notifier_block *nb);
extern int acpi_lid_notifier_unregister(struct notifier_block *nb);
extern int acpi_lid_open(void);
{
return 1;
}
-#endif /* defined(CONFIG_ACPI_BUTTON) || defined(CONFIG_ACPI_BUTTON_MODULE) */
+#endif /* IS_ENABLED(CONFIG_ACPI_BUTTON) */
#endif /* ACPI_BUTTON_H */
acpi_backlight_native,
};
-#if (defined CONFIG_ACPI_VIDEO || defined CONFIG_ACPI_VIDEO_MODULE)
+#if IS_ENABLED(CONFIG_ACPI_VIDEO)
extern int acpi_video_register(void);
extern void acpi_video_unregister(void);
extern int acpi_video_get_edid(struct acpi_device *device, int type,
* Convert a physical address to a Page Frame Number and back
*/
#define __phys_to_pfn(paddr) ((unsigned long)((paddr) >> PAGE_SHIFT))
-#define __pfn_to_phys(pfn) ((pfn) << PAGE_SHIFT)
+#define __pfn_to_phys(pfn) PFN_PHYS(pfn)
#define page_to_pfn __page_to_pfn
#define pfn_to_page __pfn_to_page
cpu_relax();
}
-#ifndef virt_queued_spin_lock
-static __always_inline bool virt_queued_spin_lock(struct qspinlock *lock)
+#ifndef virt_spin_lock
+static __always_inline bool virt_spin_lock(struct qspinlock *lock)
{
return false;
}
#ifndef _ASM_WORD_AT_A_TIME_H
#define _ASM_WORD_AT_A_TIME_H
-/*
- * This says "generic", but it's actually big-endian only.
- * Little-endian can use more efficient versions of these
- * interfaces, see for example
- * arch/x86/include/asm/word-at-a-time.h
- * for those.
- */
-
#include <linux/kernel.h>
+#include <asm/byteorder.h>
+
+#ifdef __BIG_ENDIAN
struct word_at_a_time {
const unsigned long high_bits, low_bits;
#define zero_bytemask(mask) (~1ul << __fls(mask))
#endif
+#else
+
+/*
+ * The optimal byte mask counting is probably going to be something
+ * that is architecture-specific. If you have a reliably fast
+ * bit count instruction, that might be better than the multiply
+ * and shift, for example.
+ */
+struct word_at_a_time {
+ const unsigned long one_bits, high_bits;
+};
+
+#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
+
+#ifdef CONFIG_64BIT
+
+/*
+ * Jan Achrenius on G+: microoptimized version of
+ * the simpler "(mask & ONEBYTES) * ONEBYTES >> 56"
+ * that works for the bytemasks without having to
+ * mask them first.
+ */
+static inline long count_masked_bytes(unsigned long mask)
+{
+ return mask*0x0001020304050608ul >> 56;
+}
+
+#else /* 32-bit case */
+
+/* Carl Chatfield / Jan Achrenius G+ version for 32-bit */
+static inline long count_masked_bytes(long mask)
+{
+ /* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */
+ long a = (0x0ff0001+mask) >> 23;
+ /* Fix the 1 for 00 case */
+ return a & mask;
+}
+
+#endif
+
+/* Return nonzero if it has a zero */
+static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c)
+{
+ unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
+ *bits = mask;
+ return mask;
+}
+
+static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c)
+{
+ return bits;
+}
+
+static inline unsigned long create_zero_mask(unsigned long bits)
+{
+ bits = (bits - 1) & ~bits;
+ return bits >> 7;
+}
+
+/* The mask we created is directly usable as a bytemask */
+#define zero_bytemask(mask) (mask)
+
+static inline unsigned long find_zero(unsigned long mask)
+{
+ return count_masked_bytes(mask);
+}
+
+#endif /* __BIG_ENDIAN */
+
#endif /* _ASM_WORD_AT_A_TIME_H */
* ATOMIC: used in the atomic code.
* This is the category used by the DRM_DEBUG_ATOMIC() macro.
*
+ * VBL: used for verbose debug message in the vblank code
+ * This is the category used by the DRM_DEBUG_VBL() macro.
+ *
* Enabling verbose debug messages is done through the drm.debug parameter,
* each category being enabled by a bit.
*
* drm.debug=0x2 will enable DRIVER messages
* drm.debug=0x3 will enable CORE and DRIVER messages
* ...
- * drm.debug=0xf will enable all messages
+ * drm.debug=0x3f will enable all messages
*
* An interesting feature is that it's possible to enable verbose logging at
* run-time by echoing the debug value in its sysfs node:
#define DRM_UT_KMS 0x04
#define DRM_UT_PRIME 0x08
#define DRM_UT_ATOMIC 0x10
+#define DRM_UT_VBL 0x20
extern __printf(2, 3)
void drm_ut_debug_printk(const char *function_name,
if (unlikely(drm_debug & DRM_UT_ATOMIC)) \
drm_ut_debug_printk(__func__, fmt, ##args); \
} while (0)
+#define DRM_DEBUG_VBL(fmt, args...) \
+ do { \
+ if (unlikely(drm_debug & DRM_UT_VBL)) \
+ drm_ut_debug_printk(__func__, fmt, ##args); \
+ } while (0)
/*@}*/
/**
* get_vblank_counter - get raw hardware vblank counter
* @dev: DRM device
- * @crtc: counter to fetch
+ * @pipe: counter to fetch
*
* Driver callback for fetching a raw hardware vblank counter for @crtc.
* If a device doesn't have a hardware counter, the driver can simply
* RETURNS
* Raw vblank counter value.
*/
- u32 (*get_vblank_counter) (struct drm_device *dev, int crtc);
+ u32 (*get_vblank_counter) (struct drm_device *dev, unsigned int pipe);
/**
* enable_vblank - enable vblank interrupt events
* @dev: DRM device
- * @crtc: which irq to enable
+ * @pipe: which irq to enable
*
* Enable vblank interrupts for @crtc. If the device doesn't have
* a hardware vblank counter, this routine should be a no-op, since
* Zero on success, appropriate errno if the given @crtc's vblank
* interrupt cannot be enabled.
*/
- int (*enable_vblank) (struct drm_device *dev, int crtc);
+ int (*enable_vblank) (struct drm_device *dev, unsigned int pipe);
/**
* disable_vblank - disable vblank interrupt events
* @dev: DRM device
- * @crtc: which irq to enable
+ * @pipe: which irq to enable
*
* Disable vblank interrupts for @crtc. If the device doesn't have
* a hardware vblank counter, this routine should be a no-op, since
* interrupts will have to stay on to keep the count accurate.
*/
- void (*disable_vblank) (struct drm_device *dev, int crtc);
+ void (*disable_vblank) (struct drm_device *dev, unsigned int pipe);
/**
* Called by \c drm_device_is_agp. Typically used to determine if a
* optional accurate ktime_get timestamp of when position was measured.
*
* \param dev DRM device.
- * \param crtc Id of the crtc to query.
+ * \param pipe Id of the crtc to query.
* \param flags Flags from the caller (DRM_CALLED_FROM_VBLIRQ or 0).
* \param *vpos Target location for current vertical scanout position.
* \param *hpos Target location for current horizontal scanout position.
* scanout position query. Can be NULL to skip timestamp.
* \param *etime Target location for timestamp taken immediately after
* scanout position query. Can be NULL to skip timestamp.
+ * \param mode Current display timings.
*
* Returns vpos as a positive number while in active scanout area.
* Returns vpos as a negative number inside vblank, counting the number
* but unknown small number of scanlines wrt. real scanout position.
*
*/
- int (*get_scanout_position) (struct drm_device *dev, int crtc,
- unsigned int flags,
- int *vpos, int *hpos, ktime_t *stime,
- ktime_t *etime);
+ int (*get_scanout_position) (struct drm_device *dev, unsigned int pipe,
+ unsigned int flags, int *vpos, int *hpos,
+ ktime_t *stime, ktime_t *etime,
+ const struct drm_display_mode *mode);
/**
* Called by \c drm_get_last_vbltimestamp. Should return a precise
* to the OpenML OML_sync_control extension specification.
*
* \param dev dev DRM device handle.
- * \param crtc crtc for which timestamp should be returned.
+ * \param pipe crtc for which timestamp should be returned.
* \param *max_error Maximum allowable timestamp error in nanoseconds.
* Implementation should strive to provide timestamp
* with an error of at most *max_error nanoseconds.
* negative number on failure. A positive status code on success,
* which describes how the vblank_time timestamp was computed.
*/
- int (*get_vblank_timestamp) (struct drm_device *dev, int crtc,
+ int (*get_vblank_timestamp) (struct drm_device *dev, unsigned int pipe,
int *max_error,
struct timeval *vblank_time,
unsigned flags);
u32 last_wait; /* Last vblank seqno waited per CRTC */
unsigned int inmodeset; /* Display driver is setting mode */
unsigned int pipe; /* crtc index */
+ int framedur_ns; /* frame/field duration in ns */
+ int linedur_ns; /* line duration in ns */
bool enabled; /* so we don't call enable more than
once per disable */
};
/* Misc. IOCTL support (drm_ioctl.c) */
int drm_noop(struct drm_device *dev, void *data,
struct drm_file *file_priv);
+int drm_invalid_op(struct drm_device *dev, void *data,
+ struct drm_file *file_priv);
/* Cache management (drm_cache.c) */
void drm_clflush_pages(struct page *pages[], unsigned long num_pages);
extern int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs);
extern int drm_wait_vblank(struct drm_device *dev, void *data,
struct drm_file *filp);
-extern u32 drm_vblank_count(struct drm_device *dev, int pipe);
+extern u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe);
extern u32 drm_crtc_vblank_count(struct drm_crtc *crtc);
extern u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
struct timeval *vblanktime);
+extern u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
+ struct timeval *vblanktime);
extern void drm_send_vblank_event(struct drm_device *dev, unsigned int pipe,
struct drm_pending_vblank_event *e);
extern void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
extern void drm_crtc_vblank_reset(struct drm_crtc *crtc);
extern void drm_crtc_vblank_on(struct drm_crtc *crtc);
extern void drm_vblank_cleanup(struct drm_device *dev);
+extern u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe);
extern int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
unsigned int pipe, int *max_error,
struct timeval *vblank_time,
unsigned flags,
- const struct drm_crtc *refcrtc,
const struct drm_display_mode *mode);
extern void drm_calc_timestamping_constants(struct drm_crtc *crtc,
const struct drm_display_mode *mode);
struct drm_device;
struct drm_file;
-#define __OS_HAS_AGP (defined(CONFIG_AGP) || (defined(CONFIG_AGP_MODULE) && \
- defined(MODULE)))
-
struct drm_agp_head {
struct agp_kern_info agp_info;
struct list_head memory;
unsigned long page_mask;
};
-#if __OS_HAS_AGP
+#if IS_ENABLED(CONFIG_AGP)
void drm_free_agp(struct agp_memory * handle, int pages);
int drm_bind_agp(struct agp_memory * handle, unsigned int start);
int drm_agp_bind_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
-#else /* __OS_HAS_AGP */
+#else /* CONFIG_AGP */
static inline void drm_free_agp(struct agp_memory * handle, int pages)
{
return -ENODEV;
}
-static inline int drm_agp_acquire_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
static inline int drm_agp_release(struct drm_device *dev)
{
return -ENODEV;
}
-static inline int drm_agp_release_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
static inline int drm_agp_enable(struct drm_device *dev,
struct drm_agp_mode mode)
{
return -ENODEV;
}
-static inline int drm_agp_enable_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
static inline int drm_agp_info(struct drm_device *dev,
struct drm_agp_info *info)
{
return -ENODEV;
}
-static inline int drm_agp_info_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
static inline int drm_agp_alloc(struct drm_device *dev,
struct drm_agp_buffer *request)
{
return -ENODEV;
}
-static inline int drm_agp_alloc_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
static inline int drm_agp_free(struct drm_device *dev,
struct drm_agp_buffer *request)
{
return -ENODEV;
}
-static inline int drm_agp_free_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
static inline int drm_agp_unbind(struct drm_device *dev,
struct drm_agp_binding *request)
{
return -ENODEV;
}
-static inline int drm_agp_unbind_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
static inline int drm_agp_bind(struct drm_device *dev,
struct drm_agp_binding *request)
{
return -ENODEV;
}
-static inline int drm_agp_bind_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- return -ENODEV;
-}
-
-#endif /* __OS_HAS_AGP */
+#endif /* CONFIG_AGP */
#endif /* _DRM_AGPSUPPORT_H_ */
#include <drm/drm_crtc.h>
+struct drm_atomic_state;
+
int drm_atomic_helper_check_modeset(struct drm_device *dev,
struct drm_atomic_state *state);
int drm_atomic_helper_check_planes(struct drm_device *dev,
int drm_atomic_helper_prepare_planes(struct drm_device *dev,
struct drm_atomic_state *state);
void drm_atomic_helper_commit_planes(struct drm_device *dev,
- struct drm_atomic_state *state);
+ struct drm_atomic_state *state,
+ bool active_only);
void drm_atomic_helper_cleanup_planes(struct drm_device *dev,
struct drm_atomic_state *old_state);
void drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state);
uint32_t src_x, uint32_t src_y,
uint32_t src_w, uint32_t src_h);
int drm_atomic_helper_disable_plane(struct drm_plane *plane);
+int __drm_atomic_helper_disable_plane(struct drm_plane *plane,
+ struct drm_plane_state *plane_state);
int drm_atomic_helper_set_config(struct drm_mode_set *set);
+int __drm_atomic_helper_set_config(struct drm_mode_set *set,
+ struct drm_atomic_state *state);
int drm_atomic_helper_crtc_set_property(struct drm_crtc *crtc,
struct drm_property *property,
struct drm_connector_state *state);
struct drm_connector_state *
drm_atomic_helper_connector_duplicate_state(struct drm_connector *connector);
+struct drm_atomic_state *
+drm_atomic_helper_duplicate_state(struct drm_device *dev,
+ struct drm_modeset_acquire_ctx *ctx);
void
__drm_atomic_helper_connector_destroy_state(struct drm_connector *connector,
struct drm_connector_state *state);
}
/* rotation property bits */
+#define DRM_ROTATE_MASK 0x0f
#define DRM_ROTATE_0 0
#define DRM_ROTATE_90 1
#define DRM_ROTATE_180 2
#define DRM_ROTATE_270 3
+#define DRM_REFLECT_MASK (~DRM_ROTATE_MASK)
#define DRM_REFLECT_X 4
#define DRM_REFLECT_Y 5
int flags;
uint32_t pixel_format; /* fourcc format */
struct list_head filp_head;
- /* if you are using the helper */
- void *helper_private;
};
struct drm_property_blob {
* @enabled: is this CRTC enabled?
* @mode: current mode timings
* @hwmode: mode timings as programmed to hw regs
- * @invert_dimensions: for purposes of error checking crtc vs fb sizes,
- * invert the width/height of the crtc. This is used if the driver
- * is performing 90 or 270 degree rotated scanout
* @x: x position on screen
* @y: y position on screen
* @funcs: CRTC control functions
* @gamma_size: size of gamma ramp
* @gamma_store: gamma ramp values
- * @framedur_ns: precise frame timing
- * @linedur_ns: precise line timing
- * @pixeldur_ns: precise pixel timing
* @helper_private: mid-layer private data
* @properties: property tracking for this CRTC
* @state: current atomic state for this CRTC
*/
struct drm_display_mode hwmode;
- bool invert_dimensions;
-
int x, y;
const struct drm_crtc_funcs *funcs;
uint32_t gamma_size;
uint16_t *gamma_store;
- /* Constants needed for precise vblank and swap timestamping. */
- int framedur_ns, linedur_ns, pixeldur_ns;
-
/* if you are using the helper */
const void *helper_private;
* @next: the next bridge in the encoder chain
* @of_node: device node pointer to the bridge
* @list: to keep track of all added bridges
- * @base: base mode object
* @funcs: control functions
* @driver_private: pointer to the bridge driver's internal context
*/
extern int drm_mode_create_dvi_i_properties(struct drm_device *dev);
extern int drm_mode_create_tv_properties(struct drm_device *dev,
unsigned int num_modes,
- char *modes[]);
+ const char * const modes[]);
extern int drm_mode_create_scaling_mode_property(struct drm_device *dev);
extern int drm_mode_create_aspect_ratio_property(struct drm_device *dev);
extern int drm_mode_create_dirty_info_property(struct drm_device *dev);
extern void drm_kms_helper_poll_disable(struct drm_device *dev);
extern void drm_kms_helper_poll_enable(struct drm_device *dev);
+extern void drm_kms_helper_poll_enable_locked(struct drm_device *dev);
#endif
#define DP_AUX_I2C_WRITE 0x0
#define DP_AUX_I2C_READ 0x1
-#define DP_AUX_I2C_STATUS 0x2
+#define DP_AUX_I2C_WRITE_STATUS_UPDATE 0x2
#define DP_AUX_I2C_MOT 0x4
#define DP_AUX_NATIVE_WRITE 0x8
#define DP_AUX_NATIVE_READ 0x9
#define MODE_I2C_READ 4
#define MODE_I2C_STOP 8
+/* DP 1.2 MST PORTs - Section 2.5.1 v1.2a spec */
+#define DP_MST_PHYSICAL_PORT_0 0
+#define DP_MST_LOGICAL_PORT_0 8
+
#define DP_LINK_STATUS_SIZE 6
bool drm_dp_channel_eq_ok(const u8 link_status[DP_LINK_STATUS_SIZE],
int lane_count);
(dpcd[DP_MAX_LANE_COUNT] & DP_ENHANCED_FRAME_CAP);
}
+static inline bool
+drm_dp_tps3_supported(const u8 dpcd[DP_RECEIVER_CAP_SIZE])
+{
+ return dpcd[DP_DPCD_REV] >= 0x12 &&
+ dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED;
+}
+
/*
* DisplayPort AUX channel
*/
struct drm_dp_mst_topology_cbs {
/* create a connector for a port */
struct drm_connector *(*add_connector)(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port, const char *path);
+ void (*register_connector)(struct drm_connector *connector);
void (*destroy_connector)(struct drm_dp_mst_topology_mgr *mgr,
struct drm_connector *connector);
void (*hotplug)(struct drm_dp_mst_topology_mgr *mgr);
int drm_edid_to_sad(struct edid *edid, struct cea_sad **sads);
int drm_edid_to_speaker_allocation(struct edid *edid, u8 **sadb);
int drm_av_sync_delay(struct drm_connector *connector,
- struct drm_display_mode *mode);
-struct drm_connector *drm_select_eld(struct drm_encoder *encoder,
- struct drm_display_mode *mode);
+ const struct drm_display_mode *mode);
+struct drm_connector *drm_select_eld(struct drm_encoder *encoder);
int drm_load_edid_firmware(struct drm_connector *connector);
int
struct drm_connector *connector;
};
+/**
+ * struct drm_fb_helper - helper to emulate fbdev on top of kms
+ * @fb: Scanout framebuffer object
+ * @dev: DRM device
+ * @crtc_count: number of possible CRTCs
+ * @crtc_info: per-CRTC helper state (mode, x/y offset, etc)
+ * @connector_count: number of connected connectors
+ * @connector_info_alloc_count: size of connector_info
+ * @funcs: driver callbacks for fb helper
+ * @fbdev: emulated fbdev device info struct
+ * @pseudo_palette: fake palette of 16 colors
+ * @kernel_fb_list: list_head in kernel_fb_helper_list
+ * @delayed_hotplug: was there a hotplug while kms master active?
+ */
struct drm_fb_helper {
struct drm_framebuffer *fb;
struct drm_device *dev;
/* we got a hotplug but fbdev wasn't running the console
delay until next set_par */
bool delayed_hotplug;
+
+ /**
+ * @atomic:
+ *
+ * Use atomic updates for restore_fbdev_mode(), etc. This defaults to
+ * true if driver has DRIVER_ATOMIC feature flag, but drivers can
+ * override it to true after drm_fb_helper_init() if they support atomic
+ * modeset but do not yet advertise DRIVER_ATOMIC (note that fb-helper
+ * does not require ASYNC commits).
+ */
+ bool atomic;
};
#ifdef CONFIG_DRM_FBDEV_EMULATION
int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
struct fb_info *info);
-bool drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper);
+int drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper);
struct fb_info *drm_fb_helper_alloc_fbi(struct drm_fb_helper *fb_helper);
void drm_fb_helper_unregister_fbi(struct drm_fb_helper *fb_helper);
return 0;
}
-static inline bool
+static inline int
drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper)
{
- return true;
+ return 0;
}
static inline struct fb_info *
static inline void
drm_gem_object_unreference(struct drm_gem_object *obj)
{
- if (obj != NULL)
+ if (obj != NULL) {
+ WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
+
kref_put(&obj->refcount, drm_gem_object_free);
+ }
}
static inline void
struct ww_acquire_ctx ww_ctx;
- /**
+ /*
* Contended lock: if a lock is contended you should only call
* drm_modeset_backoff() which drops locks and slow-locks the
* contended lock.
*/
struct drm_modeset_lock *contended;
- /**
+ /*
* list of held locks (drm_modeset_lock)
*/
struct list_head locked;
- /**
+ /*
* Trylock mode, use only for panic handlers!
*/
bool trylock_only;
* Used for locking CRTCs and other modeset resources.
*/
struct drm_modeset_lock {
- /**
+ /*
* modeset lock
*/
struct ww_mutex mutex;
- /**
+ /*
* Resources that are locked as part of an atomic update are added
* to a list (so we know what to unlock at the end).
*/
#ifndef __DRM_OF_H__
#define __DRM_OF_H__
+struct component_master_ops;
+struct device;
struct drm_device;
struct device_node;
#ifdef CONFIG_OF
extern uint32_t drm_of_find_possible_crtcs(struct drm_device *dev,
struct device_node *port);
+extern int drm_of_component_probe(struct device *dev,
+ int (*compare_of)(struct device *, void *),
+ const struct component_master_ops *m_ops);
#else
static inline uint32_t drm_of_find_possible_crtcs(struct drm_device *dev,
struct device_node *port)
{
return 0;
}
+
+static inline int
+drm_of_component_probe(struct device *dev,
+ int (*compare_of)(struct device *, void *),
+ const struct component_master_ops *m_ops)
+{
+ return -EINVAL;
+}
#endif
#endif /* __DRM_OF_H__ */
*/
struct drm_plane_helper_funcs {
int (*prepare_fb)(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *new_state);
void (*cleanup_fb)(struct drm_plane *plane,
- struct drm_framebuffer *fb,
const struct drm_plane_state *old_state);
int (*atomic_check)(struct drm_plane *plane,
unsigned long page_offset, unsigned long size);
void drm_vma_offset_manager_destroy(struct drm_vma_offset_manager *mgr);
-struct drm_vma_offset_node *drm_vma_offset_lookup(struct drm_vma_offset_manager *mgr,
- unsigned long start,
- unsigned long pages);
struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_manager *mgr,
unsigned long start,
unsigned long pages);
struct file *filp);
/**
- * drm_vma_offset_exact_lookup() - Look up node by exact address
+ * drm_vma_offset_exact_lookup_locked() - Look up node by exact address
* @mgr: Manager object
* @start: Start address (page-based, not byte-based)
* @pages: Size of object (page-based)
*
- * Same as drm_vma_offset_lookup() but does not allow any offset into the node.
+ * Same as drm_vma_offset_lookup_locked() but does not allow any offset into the node.
* It only returns the exact object with the given start address.
*
* RETURNS:
* Node at exact start address @start.
*/
static inline struct drm_vma_offset_node *
-drm_vma_offset_exact_lookup(struct drm_vma_offset_manager *mgr,
- unsigned long start,
- unsigned long pages)
+drm_vma_offset_exact_lookup_locked(struct drm_vma_offset_manager *mgr,
+ unsigned long start,
+ unsigned long pages)
{
struct drm_vma_offset_node *node;
- node = drm_vma_offset_lookup(mgr, start, pages);
+ node = drm_vma_offset_lookup_locked(mgr, start, pages);
return (node && node->vm_node.start == start) ? node : NULL;
}
* drm_vma_offset_lock_lookup() - Lock lookup for extended private use
* @mgr: Manager object
*
- * Lock VMA manager for extended lookups. Only *_locked() VMA function calls
+ * Lock VMA manager for extended lookups. Only locked VMA function calls
* are allowed while holding this lock. All other contexts are blocked from VMA
* until the lock is released via drm_vma_offset_unlock_lookup().
*
* not call any other VMA helpers while holding this lock.
*
* Note: You're in atomic-context while holding this lock!
- *
- * Example:
- * drm_vma_offset_lock_lookup(mgr);
- * node = drm_vma_offset_lookup_locked(mgr);
- * if (node)
- * kref_get_unless_zero(container_of(node, sth, entr));
- * drm_vma_offset_unlock_lookup(mgr);
*/
static inline void drm_vma_offset_lock_lookup(struct drm_vma_offset_manager *mgr)
{
#ifndef _I915_COMPONENT_H_
#define _I915_COMPONENT_H_
+/* MAX_PORT is the number of port
+ * It must be sync with I915_MAX_PORTS defined i915_drv.h
+ * 5 should be enough as only HSW, BDW, SKL need such fix.
+ */
+#define MAX_PORTS 5
+
+/**
+ * struct i915_audio_component_ops - callbacks defined in gfx driver
+ * @owner: the module owner
+ * @get_power: get the POWER_DOMAIN_AUDIO power well
+ * @put_power: put the POWER_DOMAIN_AUDIO power well
+ * @codec_wake_override: Enable/Disable generating the codec wake signal
+ * @get_cdclk_freq: get the Core Display Clock in KHz
+ * @sync_audio_rate: set n/cts based on the sample rate
+ */
+struct i915_audio_component_ops {
+ struct module *owner;
+ void (*get_power)(struct device *);
+ void (*put_power)(struct device *);
+ void (*codec_wake_override)(struct device *, bool enable);
+ int (*get_cdclk_freq)(struct device *);
+ int (*sync_audio_rate)(struct device *, int port, int rate);
+};
+
+struct i915_audio_component_audio_ops {
+ void *audio_ptr;
+ /**
+ * Call from i915 driver, notifying the HDA driver that
+ * pin sense and/or ELD information has changed.
+ * @audio_ptr: HDA driver object
+ * @port: Which port has changed (PORTA / PORTB / PORTC etc)
+ */
+ void (*pin_eld_notify)(void *audio_ptr, int port);
+};
+
+/**
+ * struct i915_audio_component - used for audio video interaction
+ * @dev: the device from gfx driver
+ * @aud_sample_rate: the array of audio sample rate per port
+ * @ops: callback for audio driver calling
+ * @audio_ops: Call from i915 driver
+ */
struct i915_audio_component {
struct device *dev;
+ int aud_sample_rate[MAX_PORTS];
+
+ const struct i915_audio_component_ops *ops;
- const struct i915_audio_component_ops {
- struct module *owner;
- void (*get_power)(struct device *);
- void (*put_power)(struct device *);
- void (*codec_wake_override)(struct device *, bool enable);
- int (*get_cdclk_freq)(struct device *);
- } *ops;
-
- const struct i915_audio_component_audio_ops {
- void *audio_ptr;
- /**
- * Call from i915 driver, notifying the HDA driver that
- * pin sense and/or ELD information has changed.
- * @audio_ptr: HDA driver object
- * @port: Which port has changed (PORTA / PORTB / PORTC etc)
- */
- void (*pin_eld_notify)(void *audio_ptr, int port);
- } *audio_ops;
+ const struct i915_audio_component_audio_ops *audio_ops;
};
#endif /* _I915_COMPONENT_H_ */
#define VGIC_V3_MAX_LRS 16
#define VGIC_MAX_IRQS 1024
#define VGIC_V2_MAX_CPUS 8
-
-/* Sanity checks... */
-#if (KVM_MAX_VCPUS > 255)
-#error Too many KVM VCPUs, the VGIC only supports up to 255 VCPUs for now
-#endif
+#define VGIC_V3_MAX_CPUS 255
#if (VGIC_NR_IRQS_LEGACY & 31)
#error "VGIC_NR_IRQS must be a multiple of 32"
int acpi_pci_irq_enable (struct pci_dev *dev);
void acpi_penalize_isa_irq(int irq, int active);
+bool acpi_isa_irq_available(int irq);
void acpi_penalize_sci_irq(int irq, int trigger, int polarity);
void acpi_pci_irq_disable (struct pci_dev *dev);
#include <linux/sched.h>
#include <linux/blkdev.h>
#include <linux/writeback.h>
+#include <linux/memcontrol.h>
#include <linux/blk-cgroup.h>
#include <linux/backing-dev-defs.h>
#include <linux/slab.h>
* @inode: inode of interest
*
* cgroup writeback requires support from both the bdi and filesystem.
- * Test whether @inode has both.
+ * Also, both memcg and iocg have to be on the default hierarchy. Test
+ * whether all conditions are met.
+ *
+ * Note that the test result may change dynamically on the same inode
+ * depending on how memcg and iocg are configured.
*/
static inline bool inode_cgwb_enabled(struct inode *inode)
{
struct backing_dev_info *bdi = inode_to_bdi(inode);
- return bdi_cap_account_dirty(bdi) &&
+ return cgroup_on_dfl(mem_cgroup_root_css->cgroup) &&
+ cgroup_on_dfl(blkcg_root_css->cgroup) &&
+ bdi_cap_account_dirty(bdi) &&
(bdi->capabilities & BDI_CAP_CGROUP_WRITEBACK) &&
(inode->i_sb->s_iflags & SB_I_CGROUPWB);
}
BLK_MQ_F_SHOULD_MERGE = 1 << 0,
BLK_MQ_F_TAG_SHARED = 1 << 1,
BLK_MQ_F_SG_MERGE = 1 << 2,
- BLK_MQ_F_SYSFS_UP = 1 << 3,
BLK_MQ_F_DEFER_ISSUE = 1 << 4,
BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
BLK_MQ_F_ALLOC_POLICY_BITS = 1,
void blk_mq_cancel_requeue_work(struct request_queue *q);
void blk_mq_kick_requeue_list(struct request_queue *q);
void blk_mq_abort_requeue_list(struct request_queue *q);
-void blk_mq_complete_request(struct request *rq);
+void blk_mq_complete_request(struct request *rq, int error);
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
void blk_mq_run_hw_queues(struct request_queue *q, bool async);
void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
-void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx *hctx, busy_iter_fn *fn,
- void *priv);
void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
void *priv);
void blk_mq_freeze_queue(struct request_queue *q);
struct blk_mq_tag_set *tag_set;
struct list_head tag_set_list;
struct bio_set *bio_split;
+
+ bool mq_sysfs_init_done;
};
#define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */
((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
}
+static inline bool bio_will_gap(struct request_queue *q, struct bio *prev,
+ struct bio *next)
+{
+ if (!bio_has_data(prev))
+ return false;
+
+ return bvec_gap_to_prev(q, &prev->bi_io_vec[prev->bi_vcnt - 1],
+ next->bi_io_vec[0].bv_offset);
+}
+
+static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
+{
+ return bio_will_gap(req->q, req->biotail, bio);
+}
+
+static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
+{
+ return bio_will_gap(req->q, bio, req->bio);
+}
+
struct work_struct;
int kblockd_schedule_work(struct work_struct *work);
int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay);
return q->limits.max_integrity_segments;
}
+static inline bool integrity_req_gap_back_merge(struct request *req,
+ struct bio *next)
+{
+ struct bio_integrity_payload *bip = bio_integrity(req->bio);
+ struct bio_integrity_payload *bip_next = bio_integrity(next);
+
+ return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
+ bip_next->bip_vec[0].bv_offset);
+}
+
+static inline bool integrity_req_gap_front_merge(struct request *req,
+ struct bio *bio)
+{
+ struct bio_integrity_payload *bip = bio_integrity(bio);
+ struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
+
+ return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
+ bip_next->bip_vec[0].bv_offset);
+}
+
#else /* CONFIG_BLK_DEV_INTEGRITY */
struct bio;
{
return 0;
}
+static inline bool integrity_req_gap_back_merge(struct request *req,
+ struct bio *next)
+{
+ return false;
+}
+static inline bool integrity_req_gap_front_merge(struct request *req,
+ struct bio *bio)
+{
+ return false;
+}
#endif /* CONFIG_BLK_DEV_INTEGRITY */
CEPH_FEATURE_OSDMAP_ENC | \
CEPH_FEATURE_CRUSH_TUNABLES3 | \
CEPH_FEATURE_OSD_PRIMARY_AFFINITY | \
+ CEPH_FEATURE_MSGR_KEEPALIVE2 | \
CEPH_FEATURE_CRUSH_V4)
#define CEPH_FEATURES_REQUIRED_DEFAULT \
bool out_kvec_is_msg; /* kvec refers to out_msg */
int out_more; /* there is more data after the kvecs */
__le64 out_temp_ack; /* for writing an ack */
+ struct ceph_timespec out_temp_keepalive2; /* for writing keepalive2
+ stamp */
/* message in temps */
struct ceph_msg_header in_hdr;
int in_base_pos; /* bytes read */
__le64 in_temp_ack; /* for reading an ack */
- struct timespec last_keepalive_ack;
+ struct timespec last_keepalive_ack; /* keepalive2 ack stamp */
struct delayed_work work; /* send|recv work */
unsigned long delay; /* current delay interval */
unsigned int depends_on;
};
-extern struct percpu_rw_semaphore cgroup_threadgroup_rwsem;
-
-/**
- * cgroup_threadgroup_change_begin - threadgroup exclusion for cgroups
- * @tsk: target task
- *
- * Called from threadgroup_change_begin() and allows cgroup operations to
- * synchronize against threadgroup changes using a percpu_rw_semaphore.
- */
-static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk)
-{
- percpu_down_read(&cgroup_threadgroup_rwsem);
-}
-
-/**
- * cgroup_threadgroup_change_end - threadgroup exclusion for cgroups
- * @tsk: target task
- *
- * Called from threadgroup_change_end(). Counterpart of
- * cgroup_threadcgroup_change_begin().
- */
-static inline void cgroup_threadgroup_change_end(struct task_struct *tsk)
-{
- percpu_up_read(&cgroup_threadgroup_rwsem);
-}
+void cgroup_threadgroup_change_begin(struct task_struct *tsk);
+void cgroup_threadgroup_change_end(struct task_struct *tsk);
#else /* CONFIG_CGROUPS */
struct clock_event_device;
struct module;
-/* Clock event mode commands for legacy ->set_mode(): OBSOLETE */
-enum clock_event_mode {
- CLOCK_EVT_MODE_UNUSED,
- CLOCK_EVT_MODE_SHUTDOWN,
- CLOCK_EVT_MODE_PERIODIC,
- CLOCK_EVT_MODE_ONESHOT,
- CLOCK_EVT_MODE_RESUME,
-};
-
/*
* Possible states of a clock event device.
*
* @min_delta_ns: minimum delta value in ns
* @mult: nanosecond to cycles multiplier
* @shift: nanoseconds to cycles divisor (power of two)
- * @mode: operating mode, relevant only to ->set_mode(), OBSOLETE
* @state_use_accessors:current state of the device, assigned by the core code
* @features: features
* @retries: number of forced programming retries
- * @set_mode: legacy set mode function, only for modes <= CLOCK_EVT_MODE_RESUME.
- * @set_state_periodic: switch state to periodic, if !set_mode
- * @set_state_oneshot: switch state to oneshot, if !set_mode
- * @set_state_oneshot_stopped: switch state to oneshot_stopped, if !set_mode
- * @set_state_shutdown: switch state to shutdown, if !set_mode
- * @tick_resume: resume clkevt device, if !set_mode
+ * @set_state_periodic: switch state to periodic
+ * @set_state_oneshot: switch state to oneshot
+ * @set_state_oneshot_stopped: switch state to oneshot_stopped
+ * @set_state_shutdown: switch state to shutdown
+ * @tick_resume: resume clkevt device
* @broadcast: function to broadcast events
* @min_delta_ticks: minimum delta value in ticks stored for reconfiguration
* @max_delta_ticks: maximum delta value in ticks stored for reconfiguration
u64 min_delta_ns;
u32 mult;
u32 shift;
- enum clock_event_mode mode;
enum clock_event_state state_use_accessors;
unsigned int features;
unsigned long retries;
- /*
- * State transition callback(s): Only one of the two groups should be
- * defined:
- * - set_mode(), only for modes <= CLOCK_EVT_MODE_RESUME.
- * - set_state_{shutdown|periodic|oneshot|oneshot_stopped}(), tick_resume().
- */
- void (*set_mode)(enum clock_event_mode mode, struct clock_event_device *);
int (*set_state_periodic)(struct clock_event_device *);
int (*set_state_oneshot)(struct clock_event_device *);
int (*set_state_oneshot_stopped)(struct clock_event_device *);
#define CPUFREQ_SHARED_TYPE_ANY (3) /* Freq can be set from any dependent CPU*/
#ifdef CONFIG_CPU_FREQ
+struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu);
struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu);
void cpufreq_cpu_put(struct cpufreq_policy *policy);
#else
+static inline struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
+{
+ return NULL;
+}
static inline struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
{
return NULL;
* The "flags" parameter's possible values are
* explained above with "DEVFREQ_FLAG_*" macros.
* @get_dev_status: The device should provide the current performance
- * status to devfreq, which is used by governors.
+ * status to devfreq. Governors are recommended not to
+ * use this directly. Instead, governors are recommended
+ * to use devfreq_update_stats() along with
+ * devfreq.last_status.
* @get_cur_freq: The device should provide the current frequency
* at which it is operating.
* @exit: An optional callback that is called when devfreq
struct delayed_work work;
unsigned long previous_freq;
+ struct devfreq_dev_status last_status;
void *data; /* private data for governors */
extern void devm_devfreq_unregister_opp_notifier(struct device *dev,
struct devfreq *devfreq);
+/**
+ * devfreq_update_stats() - update the last_status pointer in struct devfreq
+ * @df: the devfreq instance whose status needs updating
+ *
+ * Governors are recommended to use this function along with last_status,
+ * which allows other entities to reuse the last_status without affecting
+ * the values fetched later by governors.
+ */
+static inline int devfreq_update_stats(struct devfreq *df)
+{
+ return df->profile->get_dev_status(df->dev.parent, &df->last_status);
+}
+
#if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND)
/**
* struct devfreq_simple_ondemand_data - void *data fed to struct devfreq
struct devfreq *devfreq)
{
}
+
+static inline int devfreq_update_stats(struct devfreq *df)
+{
+ return -EINVAL;
+}
#endif /* CONFIG_PM_DEVFREQ */
#endif /* __LINUX_DEVFREQ_H__ */
#define FB_EVENT_GET_REQ 0x0D
/* Unbind from the console if possible */
#define FB_EVENT_FB_UNBIND 0x0E
-/* CONSOLE-SPECIFIC: remap all consoles to new fb - for vga switcheroo */
+/* CONSOLE-SPECIFIC: remap all consoles to new fb - for vga_switcheroo */
#define FB_EVENT_REMAP_ALL_CONSOLE 0x0F
/* A hardware display blank early change occured */
#define FB_EARLY_EVENT_BLANK 0x10
return false;
}
+/**
+ * fence_is_later - return if f1 is chronologically later than f2
+ * @f1: [in] the first fence from the same context
+ * @f2: [in] the second fence from the same context
+ *
+ * Returns true if f1 is chronologically later than f2. Both fences must be
+ * from the same context, since a seqno is not re-used across contexts.
+ */
+static inline bool fence_is_later(struct fence *f1, struct fence *f2)
+{
+ if (WARN_ON(f1->context != f2->context))
+ return false;
+
+ return f1->seqno - f2->seqno < INT_MAX;
+}
+
/**
* fence_later - return the chronologically later fence
* @f1: [in] the first fence from the same context
* set if enable_signaling wasn't called, and enabling that here is
* overkill.
*/
- if (f2->seqno - f1->seqno <= INT_MAX)
- return fence_is_signaled(f2) ? NULL : f2;
- else
+ if (fence_is_later(f1, f2))
return fence_is_signaled(f1) ? NULL : f1;
+ else
+ return fence_is_signaled(f2) ? NULL : f2;
}
signed long fence_wait_timeout(struct fence *, bool intr, signed long timeout);
-
+signed long fence_wait_any_timeout(struct fence **fences, uint32_t count,
+ bool intr, signed long timeout);
/**
* fence_wait - sleep until the fence gets signaled
extern struct files_struct init_files;
extern struct fs_struct init_fs;
+#ifdef CONFIG_CGROUPS
+#define INIT_GROUP_RWSEM(sig) \
+ .group_rwsem = __RWSEM_INITIALIZER(sig.group_rwsem),
+#else
+#define INIT_GROUP_RWSEM(sig)
+#endif
+
#ifdef CONFIG_CPUSETS
#define INIT_CPUSET_SEQ(tsk) \
.mems_allowed_seq = SEQCNT_ZERO(tsk.mems_allowed_seq),
INIT_PREV_CPUTIME(sig) \
.cred_guard_mutex = \
__MUTEX_INITIALIZER(sig.cred_guard_mutex), \
+ INIT_GROUP_RWSEM(sig) \
}
extern struct nsproxy init_nsproxy;
return iova >> iova_shift(iovad);
}
-int iommu_iova_cache_init(void);
-void iommu_iova_cache_destroy(void);
+int iova_cache_get(void);
+void iova_cache_put(void);
struct iova *alloc_iova_mem(void);
void free_iova_mem(struct iova *iova);
/*
* Return value for chip->irq_set_affinity()
*
- * IRQ_SET_MASK_OK - OK, core updates irq_data.affinity
- * IRQ_SET_MASK_NOCPY - OK, chip did update irq_data.affinity
+ * IRQ_SET_MASK_OK - OK, core updates irq_common_data.affinity
+ * IRQ_SET_MASK_NOCPY - OK, chip did update irq_common_data.affinity
* IRQ_SET_MASK_OK_DONE - Same as IRQ_SET_MASK_OK for core. Special code to
* support stacked irqchips, which indicates skipping
* all descendent irqchips.
* struct irq_common_data - per irq data shared by all irqchips
* @state_use_accessors: status information for irq chip functions.
* Use accessor functions to deal with it
+ * @node: node index useful for balancing
+ * @handler_data: per-IRQ data for the irq_chip methods
+ * @affinity: IRQ affinity on SMP
+ * @msi_desc: MSI descriptor
*/
struct irq_common_data {
unsigned int state_use_accessors;
+#ifdef CONFIG_NUMA
+ unsigned int node;
+#endif
+ void *handler_data;
+ struct msi_desc *msi_desc;
+ cpumask_var_t affinity;
};
/**
* @mask: precomputed bitmask for accessing the chip registers
* @irq: interrupt number
* @hwirq: hardware interrupt number, local to the interrupt domain
- * @node: node index useful for balancing
* @common: point to data shared by all irqchips
* @chip: low level interrupt hardware access
* @domain: Interrupt translation domain; responsible for mapping
* between hwirq number and linux irq number.
* @parent_data: pointer to parent struct irq_data to support hierarchy
* irq_domain
- * @handler_data: per-IRQ data for the irq_chip methods
* @chip_data: platform-specific per-chip private data for the chip
* methods, to allow shared chip implementations
- * @msi_desc: MSI descriptor
- * @affinity: IRQ affinity on SMP
- *
- * The fields here need to overlay the ones in irq_desc until we
- * cleaned up the direct references and switched everything over to
- * irq_data.
*/
struct irq_data {
u32 mask;
unsigned int irq;
unsigned long hwirq;
- unsigned int node;
struct irq_common_data *common;
struct irq_chip *chip;
struct irq_domain *domain;
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
struct irq_data *parent_data;
#endif
- void *handler_data;
void *chip_data;
- struct msi_desc *msi_desc;
- cpumask_var_t affinity;
};
/*
* IRQD_IRQ_MASKED - Masked state of the interrupt
* IRQD_IRQ_INPROGRESS - In progress state of the interrupt
* IRQD_WAKEUP_ARMED - Wakeup mode armed
+ * IRQD_FORWARDED_TO_VCPU - The interrupt is forwarded to a VCPU
*/
enum {
IRQD_TRIGGER_MASK = 0xf,
IRQD_IRQ_MASKED = (1 << 17),
IRQD_IRQ_INPROGRESS = (1 << 18),
IRQD_WAKEUP_ARMED = (1 << 19),
+ IRQD_FORWARDED_TO_VCPU = (1 << 20),
};
#define __irqd_to_state(d) ((d)->common->state_use_accessors)
return __irqd_to_state(d) & IRQD_WAKEUP_ARMED;
}
+static inline bool irqd_is_forwarded_to_vcpu(struct irq_data *d)
+{
+ return __irqd_to_state(d) & IRQD_FORWARDED_TO_VCPU;
+}
+
+static inline void irqd_set_forwarded_to_vcpu(struct irq_data *d)
+{
+ __irqd_to_state(d) |= IRQD_FORWARDED_TO_VCPU;
+}
+
+static inline void irqd_clr_forwarded_to_vcpu(struct irq_data *d)
+{
+ __irqd_to_state(d) &= ~IRQD_FORWARDED_TO_VCPU;
+}
/*
* Functions for chained handlers which can be enabled/disabled by the
* Built-in IRQ handlers for various IRQ types,
* callable via desc->handle_irq()
*/
-extern void handle_level_irq(unsigned int irq, struct irq_desc *desc);
-extern void handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc);
-extern void handle_edge_irq(unsigned int irq, struct irq_desc *desc);
-extern void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc);
-extern void handle_simple_irq(unsigned int irq, struct irq_desc *desc);
-extern void handle_percpu_irq(unsigned int irq, struct irq_desc *desc);
-extern void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc);
-extern void handle_bad_irq(unsigned int irq, struct irq_desc *desc);
+extern void handle_level_irq(struct irq_desc *desc);
+extern void handle_fasteoi_irq(struct irq_desc *desc);
+extern void handle_edge_irq(struct irq_desc *desc);
+extern void handle_edge_eoi_irq(struct irq_desc *desc);
+extern void handle_simple_irq(struct irq_desc *desc);
+extern void handle_percpu_irq(struct irq_desc *desc);
+extern void handle_percpu_devid_irq(struct irq_desc *desc);
+extern void handle_bad_irq(struct irq_desc *desc);
extern void handle_nested_irq(unsigned int irq);
extern int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg);
static inline void *irq_get_handler_data(unsigned int irq)
{
struct irq_data *d = irq_get_irq_data(irq);
- return d ? d->handler_data : NULL;
+ return d ? d->common->handler_data : NULL;
}
static inline void *irq_data_get_irq_handler_data(struct irq_data *d)
{
- return d->handler_data;
+ return d->common->handler_data;
}
static inline struct msi_desc *irq_get_msi_desc(unsigned int irq)
{
struct irq_data *d = irq_get_irq_data(irq);
- return d ? d->msi_desc : NULL;
+ return d ? d->common->msi_desc : NULL;
}
static inline struct msi_desc *irq_data_get_msi_desc(struct irq_data *d)
{
- return d->msi_desc;
+ return d->common->msi_desc;
}
static inline u32 irq_get_trigger_type(unsigned int irq)
return d ? irqd_get_trigger_type(d) : 0;
}
-static inline int irq_data_get_node(struct irq_data *d)
+static inline int irq_common_data_get_node(struct irq_common_data *d)
{
+#ifdef CONFIG_NUMA
return d->node;
+#else
+ return 0;
+#endif
+}
+
+static inline int irq_data_get_node(struct irq_data *d)
+{
+ return irq_common_data_get_node(d->common);
}
static inline struct cpumask *irq_get_affinity_mask(int irq)
{
struct irq_data *d = irq_get_irq_data(irq);
- return d ? d->affinity : NULL;
+ return d ? d->common->affinity : NULL;
}
static inline struct cpumask *irq_data_get_affinity_mask(struct irq_data *d)
{
- return d->affinity;
+ return d->common->affinity;
}
unsigned int arch_dynirq_lower_bound(unsigned int from);
static inline struct irq_desc *irq_data_to_desc(struct irq_data *data)
{
-#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
- return irq_to_desc(data->irq);
-#else
- return container_of(data, struct irq_desc, irq_data);
-#endif
+ return container_of(data->common, struct irq_desc, irq_common_data);
}
static inline unsigned int irq_desc_get_irq(struct irq_desc *desc)
static inline void *irq_desc_get_handler_data(struct irq_desc *desc)
{
- return desc->irq_data.handler_data;
+ return desc->irq_common_data.handler_data;
}
static inline struct msi_desc *irq_desc_get_msi_desc(struct irq_desc *desc)
{
- return desc->irq_data.msi_desc;
+ return desc->irq_common_data.msi_desc;
}
/*
* Architectures call this to let the generic IRQ layer
- * handle an interrupt. If the descriptor is attached to an
- * irqchip-style controller then we call the ->handle_irq() handler,
- * and it calls __do_IRQ() if it's attached to an irqtype-style controller.
+ * handle an interrupt.
*/
-static inline void generic_handle_irq_desc(unsigned int irq, struct irq_desc *desc)
+static inline void generic_handle_irq_desc(struct irq_desc *desc)
{
- desc->handle_irq(irq, desc);
+ desc->handle_irq(desc);
}
int generic_handle_irq(unsigned int irq);
return irq_desc_has_action(irq_to_desc(irq));
}
-/* caller has locked the irq_desc and both params are valid */
-static inline void __irq_set_handler_locked(unsigned int irq,
- irq_flow_handler_t handler)
-{
- struct irq_desc *desc;
-
- desc = irq_to_desc(irq);
- desc->handle_irq = handler;
-}
-
-/* caller has locked the irq_desc and both params are valid */
-static inline void
-__irq_set_chip_handler_name_locked(unsigned int irq, struct irq_chip *chip,
- irq_flow_handler_t handler, const char *name)
-{
- struct irq_desc *desc;
-
- desc = irq_to_desc(irq);
- irq_desc_get_irq_data(desc)->chip = chip;
- desc->handle_irq = handler;
- desc->name = name;
-}
-
/**
* irq_set_handler_locked - Set irq handler from a locked region
* @data: Pointer to the irq_data structure which identifies the irq
IRQ_DOMAIN_FLAG_NONCORE = (1 << 16),
};
+static inline struct device_node *irq_domain_get_of_node(struct irq_domain *d)
+{
+ return d->of_node;
+}
+
#ifdef CONFIG_IRQ_DOMAIN
struct irq_domain *__irq_domain_add(struct device_node *of_node, int size,
irq_hw_number_t hwirq_max, int direct_max,
struct irq_desc;
struct irq_data;
-typedef void (*irq_flow_handler_t)(unsigned int irq, struct irq_desc *desc);
+typedef void (*irq_flow_handler_t)(struct irq_desc *desc);
typedef void (*irq_preflow_handler_t)(struct irq_data *data);
#endif
*
* DEFINE_STATIC_KEY_TRUE(key);
* DEFINE_STATIC_KEY_FALSE(key);
- * static_key_likely()
- * statick_key_unlikely()
+ * static_branch_likely()
+ * static_branch_unlikely()
*
* Jump labels provide an interface to generate dynamic branches using
* self-modifying code. Assuming toolchain and architecture support, if we
* statement, setting the key to true requires us to patch in a jump
* to the out-of-line of true branch.
*
- * In addtion to static_branch_{enable,disable}, we can also reference count
+ * In addition to static_branch_{enable,disable}, we can also reference count
* the key or branch direction via static_branch_{inc,dec}. Thus,
* static_branch_inc() can be thought of as a 'make more true' and
- * static_branch_dec() as a 'make more false'. The inc()/dec()
- * interface is meant to be used exclusively from the inc()/dec() for a given
- * key.
+ * static_branch_dec() as a 'make more false'.
*
* Since this relies on modifying code, the branch modifying functions
* must be considered absolute slow paths (machine wide synchronization etc.).
* percpu counter.
*/
struct mem_cgroup_stat_cpu __percpu *stat;
- spinlock_t pcp_counter_lock;
#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
struct cg_proto tcp_mem;
u8 rsvd[8];
};
-struct mlx5_cmd_query_special_contexts_mbox_in {
- struct mlx5_inbox_hdr hdr;
- u8 rsvd[8];
-};
-
-struct mlx5_cmd_query_special_contexts_mbox_out {
- struct mlx5_outbox_hdr hdr;
- __be32 dump_fill_mkey;
- __be32 resd_lkey;
-};
-
struct mlx5_cmd_layout {
u8 type;
u8 rsvd0[3];
int mlx5_register_interface(struct mlx5_interface *intf);
void mlx5_unregister_interface(struct mlx5_interface *intf);
int mlx5_core_query_vendor_id(struct mlx5_core_dev *mdev, u32 *vendor_id);
-int mlx5_core_query_special_context(struct mlx5_core_dev *dev, u32 *rsvd_lkey);
struct mlx5_profile {
u64 mask;
#endif
}
+#ifdef CONFIG_MEMCG
+static inline struct mem_cgroup *page_memcg(struct page *page)
+{
+ return page->mem_cgroup;
+}
+
+static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)
+{
+ page->mem_cgroup = memcg;
+}
+#else
+static inline struct mem_cgroup *page_memcg(struct page *page)
+{
+ return NULL;
+}
+
+static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)
+{
+}
+#endif
+
/*
* Some inline functions in vmstat.h depend on page_zone()
*/
BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
smp_mb__before_atomic();
clear_bit(NAPI_STATE_SCHED, &n->state);
+ clear_bit(NAPI_STATE_NPSVC, &n->state);
}
#ifdef CONFIG_SMP
#include <linux/spinlock.h>
#include <linux/ethtool.h>
#include <linux/mii.h>
+#include <linux/module.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/mod_devicetable.h>
* PHYs should register using this structure
*/
struct mii_bus {
+ struct module *owner;
const char *name;
char id[MII_BUS_ID_SIZE];
void *priv;
return mdiobus_alloc_size(0);
}
-int mdiobus_register(struct mii_bus *bus);
+int __mdiobus_register(struct mii_bus *bus, struct module *owner);
+#define mdiobus_register(bus) __mdiobus_register(bus, THIS_MODULE)
void mdiobus_unregister(struct mii_bus *bus);
void mdiobus_free(struct mii_bus *bus);
struct mii_bus *devm_mdiobus_alloc_size(struct device *dev, int sizeof_priv);
struct phy_c45_device_ids *c45_ids);
struct phy_device *get_phy_device(struct mii_bus *bus, int addr, bool is_c45);
int phy_device_register(struct phy_device *phy);
+void phy_device_remove(struct phy_device *phydev);
int phy_init_hw(struct phy_device *phydev);
int phy_suspend(struct phy_device *phydev);
int phy_resume(struct phy_device *phydev);
struct rcu_synchronize *rs_array);
#define _wait_rcu_gp(checktiny, ...) \
-do { \
- call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \
- const int __n = ARRAY_SIZE(__crcu_array); \
- struct rcu_synchronize __rs_array[__n]; \
- \
- __wait_rcu_gp(checktiny, __n, __crcu_array, __rs_array); \
+do { \
+ call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \
+ struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \
+ __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \
+ __crcu_array, __rs_array); \
} while (0)
#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)
unsigned audit_tty_log_passwd;
struct tty_audit_buf *tty_audit_buf;
#endif
+#ifdef CONFIG_CGROUPS
+ /*
+ * group_rwsem prevents new tasks from entering the threadgroup and
+ * member tasks from exiting,a more specifically, setting of
+ * PF_EXITING. fork and exit paths are protected with this rwsem
+ * using threadgroup_change_begin/end(). Users which require
+ * threadgroup to remain stable should use threadgroup_[un]lock()
+ * which also takes care of exec path. Currently, cgroup is the
+ * only user.
+ */
+ struct rw_semaphore group_rwsem;
+#endif
oom_flags_t oom_flags;
short oom_score_adj; /* OOM kill score adjustment */
unsigned long arg4,
unsigned long arg5)
{
- return cap_task_prctl(option, arg2, arg3, arg3, arg5);
+ return cap_task_prctl(option, arg2, arg3, arg4, arg5);
}
static inline void security_task_to_inode(struct task_struct *p, struct inode *inode)
u8 bridged_dnat:1;
__u16 frag_max_size;
struct net_device *physindev;
+
+ /* always valid & non-NULL from FORWARD on, for physdev match */
+ struct net_device *physoutdev;
union {
/* prerouting: detect dnat in orig/reply direction */
__be32 ipv4_daddr;
* skb is out in neigh layer.
*/
char neigh_header[8];
-
- /* always valid & non-NULL from FORWARD on, for physdev match */
- struct net_device *physoutdev;
};
};
#endif
{
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0));
+ else if (skb->ip_summed == CHECKSUM_PARTIAL &&
+ skb_checksum_start_offset(skb) < 0)
+ skb->ip_summed = CHECKSUM_NONE;
}
unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
/**
* struct spi_statistics - statistics for spi transfers
- * @clock: lock protecting this structure
+ * @lock: lock protecting this structure
*
* @messages: number of spi-messages handled
* @transfers: number of spi_transfers handled
#ifndef __HAVE_ARCH_STRLCPY
size_t strlcpy(char *, const char *, size_t);
#endif
+#ifndef __HAVE_ARCH_STRSCPY
+ssize_t __must_check strscpy(char *, const char *, size_t);
+#endif
#ifndef __HAVE_ARCH_STRCAT
extern char * strcat(char *, const char *);
#endif
/*
* Connection of transports
*/
+ unsigned long sock_state;
struct delayed_work connect_worker;
struct sockaddr_storage srcaddr;
unsigned short srcport;
*/
#define TCP_RPC_REPLY (1UL << 6)
+#define XPRT_SOCK_CONNECTING 1U
+
#endif /* __KERNEL__ */
#endif /* _LINUX_SUNRPC_XPRTSOCK_H */
thermal_zone_of_sensor_register(struct device *dev, int id, void *data,
const struct thermal_zone_of_device_ops *ops)
{
- return NULL;
+ return ERR_PTR(-ENODEV);
}
static inline
int power_actor_get_max_power(struct thermal_cooling_device *,
struct thermal_zone_device *tz, u32 *max_power);
+int power_actor_get_min_power(struct thermal_cooling_device *,
+ struct thermal_zone_device *tz, u32 *min_power);
int power_actor_set_power(struct thermal_cooling_device *,
struct thermal_instance *, u32);
struct thermal_zone_device *thermal_zone_device_register(const char *, int, int,
static inline int power_actor_get_max_power(struct thermal_cooling_device *cdev,
struct thermal_zone_device *tz, u32 *max_power)
{ return 0; }
+static inline int power_actor_get_min_power(struct thermal_cooling_device *cdev,
+ struct thermal_zone_device *tz,
+ u32 *min_power)
+{ return -ENODEV; }
static inline int power_actor_set_power(struct thermal_cooling_device *cdev,
struct thermal_instance *tz, u32 power)
{ return 0; }
cpumask_or(mask, mask, tick_nohz_full_mask);
}
+static inline int housekeeping_any_cpu(void)
+{
+ return cpumask_any_and(housekeeping_mask, cpu_online_mask);
+}
+
extern void tick_nohz_full_kick(void);
extern void tick_nohz_full_kick_cpu(int cpu);
extern void tick_nohz_full_kick_all(void);
extern void __tick_nohz_task_switch(void);
#else
+static inline int housekeeping_any_cpu(void)
+{
+ return smp_processor_id();
+}
static inline bool tick_nohz_full_enabled(void) { return false; }
static inline bool tick_nohz_full_cpu(int cpu) { return false; }
static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { }
*/
int pio_dma_border; /* default is 64byte */
- u32 type;
+ uintptr_t type;
u32 enable_gpio;
/*
/*
+ * vga_switcheroo.h - Support for laptop with dual GPU using one set of outputs
+ *
* Copyright (c) 2010 Red Hat Inc.
* Author : Dave Airlie <airlied@redhat.com>
*
- * Licensed under GPLv2
+ * Copyright (c) 2015 Lukas Wunner <lukas@wunner.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS
+ * IN THE SOFTWARE.
*
- * vga_switcheroo.h - Support for laptop with dual GPU using one set of outputs
*/
#ifndef _LINUX_VGA_SWITCHEROO_H_
struct pci_dev;
+/**
+ * enum vga_switcheroo_state - client power state
+ * @VGA_SWITCHEROO_OFF: off
+ * @VGA_SWITCHEROO_ON: on
+ * @VGA_SWITCHEROO_INIT: client has registered with vga_switcheroo but
+ * vga_switcheroo is not enabled, i.e. no second client or no handler
+ * has registered. Only used in vga_switcheroo_get_client_state() which
+ * in turn is only called from hda_intel.c
+ * @VGA_SWITCHEROO_NOT_FOUND: client has not registered with vga_switcheroo.
+ * Only used in vga_switcheroo_get_client_state() which in turn is only
+ * called from hda_intel.c
+ *
+ * Client power state.
+ */
enum vga_switcheroo_state {
VGA_SWITCHEROO_OFF,
VGA_SWITCHEROO_ON,
VGA_SWITCHEROO_NOT_FOUND,
};
+/**
+ * enum vga_switcheroo_client_id - client identifier
+ * @VGA_SWITCHEROO_UNKNOWN_ID: initial identifier assigned to vga clients.
+ * Determining the id requires the handler, so GPUs are given their
+ * true id in a delayed fashion in vga_switcheroo_enable()
+ * @VGA_SWITCHEROO_IGD: integrated graphics device
+ * @VGA_SWITCHEROO_DIS: discrete graphics device
+ * @VGA_SWITCHEROO_MAX_CLIENTS: currently no more than two GPUs are supported
+ *
+ * Client identifier. Audio clients use the same identifier & 0x100.
+ */
enum vga_switcheroo_client_id {
+ VGA_SWITCHEROO_UNKNOWN_ID = -1,
VGA_SWITCHEROO_IGD,
VGA_SWITCHEROO_DIS,
VGA_SWITCHEROO_MAX_CLIENTS,
};
+/**
+ * struct vga_switcheroo_handler - handler callbacks
+ * @init: initialize handler.
+ * Optional. This gets called when vga_switcheroo is enabled, i.e. when
+ * two vga clients have registered. It allows the handler to perform
+ * some delayed initialization that depends on the existence of the
+ * vga clients. Currently only the radeon and amdgpu drivers use this.
+ * The return value is ignored
+ * @switchto: switch outputs to given client.
+ * Mandatory. For muxless machines this should be a no-op. Returning 0
+ * denotes success, anything else failure (in which case the switch is
+ * aborted)
+ * @power_state: cut or reinstate power of given client.
+ * Optional. The return value is ignored
+ * @get_client_id: determine if given pci device is integrated or discrete GPU.
+ * Mandatory
+ *
+ * Handler callbacks. The multiplexer itself. The @switchto and @get_client_id
+ * methods are mandatory, all others may be set to NULL.
+ */
struct vga_switcheroo_handler {
+ int (*init)(void);
int (*switchto)(enum vga_switcheroo_client_id id);
int (*power_state)(enum vga_switcheroo_client_id id,
enum vga_switcheroo_state state);
- int (*init)(void);
- int (*get_client_id)(struct pci_dev *pdev);
+ enum vga_switcheroo_client_id (*get_client_id)(struct pci_dev *pdev);
};
+/**
+ * struct vga_switcheroo_client_ops - client callbacks
+ * @set_gpu_state: do the equivalent of suspend/resume for the card.
+ * Mandatory. This should not cut power to the discrete GPU,
+ * which is the job of the handler
+ * @reprobe: poll outputs.
+ * Optional. This gets called after waking the GPU and switching
+ * the outputs to it
+ * @can_switch: check if the device is in a position to switch now.
+ * Mandatory. The client should return false if a user space process
+ * has one of its device files open
+ *
+ * Client callbacks. A client can be either a GPU or an audio device on a GPU.
+ * The @set_gpu_state and @can_switch methods are mandatory, @reprobe may be
+ * set to NULL. For audio clients, the @reprobe member is bogus.
+ */
struct vga_switcheroo_client_ops {
void (*set_gpu_state)(struct pci_dev *dev, enum vga_switcheroo_state);
void (*reprobe)(struct pci_dev *dev);
bool driver_power_control);
int vga_switcheroo_register_audio_client(struct pci_dev *pdev,
const struct vga_switcheroo_client_ops *ops,
- int id, bool active);
+ enum vga_switcheroo_client_id id);
void vga_switcheroo_client_fb_set(struct pci_dev *dev,
struct fb_info *info);
-int vga_switcheroo_register_handler(struct vga_switcheroo_handler *handler);
+int vga_switcheroo_register_handler(const struct vga_switcheroo_handler *handler);
void vga_switcheroo_unregister_handler(void);
int vga_switcheroo_process_delayed_switch(void);
-int vga_switcheroo_get_client_state(struct pci_dev *dev);
+enum vga_switcheroo_state vga_switcheroo_get_client_state(struct pci_dev *dev);
void vga_switcheroo_set_dynamic_switch(struct pci_dev *pdev, enum vga_switcheroo_state dynamic);
static inline int vga_switcheroo_register_client(struct pci_dev *dev,
const struct vga_switcheroo_client_ops *ops, bool driver_power_control) { return 0; }
static inline void vga_switcheroo_client_fb_set(struct pci_dev *dev, struct fb_info *info) {}
-static inline int vga_switcheroo_register_handler(struct vga_switcheroo_handler *handler) { return 0; }
+static inline int vga_switcheroo_register_handler(const struct vga_switcheroo_handler *handler) { return 0; }
static inline int vga_switcheroo_register_audio_client(struct pci_dev *pdev,
const struct vga_switcheroo_client_ops *ops,
- int id, bool active) { return 0; }
+ enum vga_switcheroo_client_id id) { return 0; }
static inline void vga_switcheroo_unregister_handler(void) {}
static inline int vga_switcheroo_process_delayed_switch(void) { return 0; }
-static inline int vga_switcheroo_get_client_state(struct pci_dev *dev) { return VGA_SWITCHEROO_ON; }
+static inline enum vga_switcheroo_state vga_switcheroo_get_client_state(struct pci_dev *dev) { return VGA_SWITCHEROO_ON; }
static inline void vga_switcheroo_set_dynamic_switch(struct pci_dev *pdev, enum vga_switcheroo_state dynamic) {}
typedef int wait_bit_action_f(struct wait_bit_key *);
void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
-void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, int nr,
- void *key);
+void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
#define wake_up_poll(x, m) \
__wake_up(x, TASK_NORMAL, 1, (void *) (m))
#define wake_up_locked_poll(x, m) \
- __wake_up_locked_key((x), TASK_NORMAL, 1, (void *) (m))
+ __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
#define wake_up_interruptible_poll(x, m) \
__wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
#define wake_up_interruptible_sync_poll(x, m) \
#define UNIX_GC_MAYBE_CYCLE 1
struct socket_wq peer_wq;
};
-#define unix_sk(__sk) ((struct unix_sock *)__sk)
+
+static inline struct unix_sock *unix_sk(struct sock *sk)
+{
+ return (struct unix_sock *)sk;
+}
#define peer_wait peer_wq.wait
#define FLOWI_FLAG_ANYSRC 0x01
#define FLOWI_FLAG_KNOWN_NH 0x02
#define FLOWI_FLAG_VRFSRC 0x04
+#define FLOWI_FLAG_SKIP_NH_OIF 0x08
__u32 flowic_secid;
struct flowi_tunnel flowic_tun_key;
};
void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
struct inet_hashinfo *hashinfo);
-void inet_twsk_schedule(struct inet_timewait_sock *tw, const int timeo);
+void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo,
+ bool rearm);
+
+static void inline inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo)
+{
+ __inet_twsk_schedule(tw, timeo, false);
+}
+
+static void inline inet_twsk_reschedule(struct inet_timewait_sock *tw, int timeo)
+{
+ __inet_twsk_schedule(tw, timeo, true);
+}
+
void inet_twsk_deschedule_put(struct inet_timewait_sock *tw);
void inet_twsk_purge(struct inet_hashinfo *hashinfo,
struct nl_info *info, struct mx6_config *mxc);
int fib6_del(struct rt6_info *rt, struct nl_info *info);
-void inet6_rt_notify(int event, struct rt6_info *rt, struct nl_info *info);
+void inet6_rt_notify(int event, struct rt6_info *rt, struct nl_info *info,
+ unsigned int flags);
void fib6_run_gc(unsigned long expires, struct net *net, bool force);
__be32 o_key;
};
+struct ip6_tnl_dst {
+ seqlock_t lock;
+ struct dst_entry __rcu *dst;
+ u32 cookie;
+};
+
/* IPv6 tunnel */
struct ip6_tnl {
struct ip6_tnl __rcu *next; /* next tunnel in list */
struct net *net; /* netns for packet i/o */
struct __ip6_tnl_parm parms; /* tunnel configuration parameters */
struct flowi fl; /* flowi template for xmit */
- struct dst_entry *dst_cache; /* cached dst */
- u32 dst_cookie;
+ struct ip6_tnl_dst __percpu *dst_cache; /* cached dst */
int err_count;
unsigned long err_time;
__u8 encap_limit; /* tunnel encapsulation limit */
} __packed;
-struct dst_entry *ip6_tnl_dst_check(struct ip6_tnl *t);
+struct dst_entry *ip6_tnl_dst_get(struct ip6_tnl *t);
+int ip6_tnl_dst_init(struct ip6_tnl *t);
+void ip6_tnl_dst_destroy(struct ip6_tnl *t);
void ip6_tnl_dst_reset(struct ip6_tnl *t);
-void ip6_tnl_dst_store(struct ip6_tnl *t, struct dst_entry *dst);
+void ip6_tnl_dst_set(struct ip6_tnl *t, struct dst_entry *dst);
int ip6_tnl_rcv_ctl(struct ip6_tnl *t, const struct in6_addr *laddr,
const struct in6_addr *raddr);
int ip6_tnl_xmit_ctl(struct ip6_tnl *t, const struct in6_addr *laddr,
struct net_device_stats *stats = &dev->stats;
int pkt_len, err;
- pkt_len = skb->len;
+ pkt_len = skb->len - skb_inner_network_offset(skb);
err = ip6_local_out_sk(sk, skb);
if (net_xmit_eval(err) == 0) {
rcu_read_lock();
tb = fib_get_table(net, RT_TABLE_MAIN);
- if (tb && !fib_table_lookup(tb, flp, res, flags | FIB_LOOKUP_NOREF))
- err = 0;
+ if (tb)
+ err = fib_table_lookup(tb, flp, res, flags | FIB_LOOKUP_NOREF);
+
+ if (err == -EAGAIN)
+ err = -ENETUNREACH;
rcu_read_unlock();
struct fib_result *res, unsigned int flags)
{
struct fib_table *tb;
- int err;
+ int err = -ENETUNREACH;
flags |= FIB_LOOKUP_NOREF;
if (net->ipv4.fib_has_custom_rules)
res->tclassid = 0;
- for (err = 0; !err; err = -ENETUNREACH) {
- tb = rcu_dereference_rtnl(net->ipv4.fib_main);
- if (tb && !fib_table_lookup(tb, flp, res, flags))
- break;
+ tb = rcu_dereference_rtnl(net->ipv4.fib_main);
+ if (tb)
+ err = fib_table_lookup(tb, flp, res, flags);
+
+ if (!err)
+ goto out;
+
+ tb = rcu_dereference_rtnl(net->ipv4.fib_default);
+ if (tb)
+ err = fib_table_lookup(tb, flp, res, flags);
- tb = rcu_dereference_rtnl(net->ipv4.fib_default);
- if (tb && !fib_table_lookup(tb, flp, res, flags))
- break;
- }
+out:
+ if (err == -EAGAIN)
+ err = -ENETUNREACH;
rcu_read_unlock();
int iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
__be32 src, __be32 dst, u8 proto,
u8 tos, u8 ttl, __be16 df, bool xnet);
+struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md,
+ gfp_t flags);
struct sk_buff *iptunnel_handle_offloads(struct sk_buff *skb, bool gre_csum,
int gso_type_mask);
flow_flags |= FLOWI_FLAG_ANYSRC;
if (netif_index_is_vrf(sock_net(sk), oif))
- flow_flags |= FLOWI_FLAG_VRFSRC;
+ flow_flags |= FLOWI_FLAG_VRFSRC | FLOWI_FLAG_SKIP_NH_OIF;
flowi4_init_output(fl4, oif, sk->sk_mark, tos, RT_SCOPE_UNIVERSE,
protocol, flow_flags, dst, src, dport, sport);
struct opa_port_state_info {
struct opa_port_states port_states;
- u16 link_width_downgrade_tx_active;
- u16 link_width_downgrade_rx_active;
+ __be16 link_width_downgrade_tx_active;
+ __be16 link_width_downgrade_rx_active;
};
struct opa_port_info {
#define DF_EMULATED_VPD_UNIT_SERIAL 0x00000004
#define DF_USING_UDEV_PATH 0x00000008
#define DF_USING_ALIAS 0x00000010
+#define DF_READ_ONLY 0x00000020
/* Physical device queue depth */
u32 queue_depth;
/* Used for SPC-2 reservations enforce of ISIDs */
__SYSCALL(__NR_bpf, sys_bpf)
#define __NR_execveat 281
__SC_COMP(__NR_execveat, sys_execveat, compat_sys_execveat)
-#define __NR_membarrier 282
+#define __NR_userfaultfd 282
+__SYSCALL(__NR_userfaultfd, sys_userfaultfd)
+#define __NR_membarrier 283
__SYSCALL(__NR_membarrier, sys_membarrier)
#undef __NR_syscalls
-#define __NR_syscalls 283
+#define __NR_syscalls 284
/*
* All syscalls below here should go away really,
* these are provided for both review and as a porting
* help for the C library version.
-*
+ *
* Last chance: are any of these important enough to
* enable by default?
*/
header-y += via_drm.h
header-y += vmwgfx_drm.h
header-y += msm_drm.h
+header-y += virtgpu_drm.h
#define AMDGPU_FAMILY_CI 120 /* Bonaire, Hawaii */
#define AMDGPU_FAMILY_KV 125 /* Kaveri, Kabini, Mullins */
#define AMDGPU_FAMILY_VI 130 /* Iceland, Tonga */
-#define AMDGPU_FAMILY_CZ 135 /* Carrizo */
+#define AMDGPU_FAMILY_CZ 135 /* Carrizo, Stoney */
#endif
struct drm_mode_modeinfo {
__u32 clock;
- __u16 hdisplay, hsync_start, hsync_end, htotal, hskew;
- __u16 vdisplay, vsync_start, vsync_end, vtotal, vscan;
+ __u16 hdisplay;
+ __u16 hsync_start;
+ __u16 hsync_end;
+ __u16 htotal;
+ __u16 hskew;
+ __u16 vdisplay;
+ __u16 vsync_start;
+ __u16 vsync_end;
+ __u16 vtotal;
+ __u16 vscan;
__u32 vrefresh;
__u32 count_crtcs;
__u32 count_connectors;
__u32 count_encoders;
- __u32 min_width, max_width;
- __u32 min_height, max_height;
+ __u32 min_width;
+ __u32 max_width;
+ __u32 min_height;
+ __u32 max_height;
};
struct drm_mode_crtc {
__u32 crtc_id; /**< Id */
__u32 fb_id; /**< Id of framebuffer */
- __u32 x, y; /**< Position on the frameuffer */
+ __u32 x; /**< x Position on the framebuffer */
+ __u32 y; /**< y Position on the framebuffer */
__u32 gamma_size;
__u32 mode_valid;
__u32 flags; /* see above flags */
/* Signed dest location allows it to be partially off screen */
- __s32 crtc_x, crtc_y;
- __u32 crtc_w, crtc_h;
+ __s32 crtc_x;
+ __s32 crtc_y;
+ __u32 crtc_w;
+ __u32 crtc_h;
/* Source values are 16.16 fixed point */
- __u32 src_x, src_y;
- __u32 src_h, src_w;
+ __u32 src_x;
+ __u32 src_y;
+ __u32 src_h;
+ __u32 src_w;
};
struct drm_mode_get_plane {
__u32 connector_type_id;
__u32 connection;
- __u32 mm_width, mm_height; /**< HxW in millimeters */
+ __u32 mm_width; /**< width in millimeters */
+ __u32 mm_height; /**< height in millimeters */
__u32 subpixel;
__u32 pad;
struct drm_mode_fb_cmd {
__u32 fb_id;
- __u32 width, height;
+ __u32 width;
+ __u32 height;
__u32 pitch;
__u32 bpp;
__u32 depth;
struct drm_mode_fb_cmd2 {
__u32 fb_id;
- __u32 width, height;
+ __u32 width;
+ __u32 height;
__u32 pixel_format; /* fourcc code from drm_fourcc.h */
__u32 flags; /* see above flags */
#ifndef _I810_DRM_H_
#define _I810_DRM_H_
+#include <drm/drm.h>
+
/* WARNING: These defines must be the same as what the Xserver uses.
* if you change them, you must change the defines in the Xserver.
*/
#define EXEC_OBJECT_NEEDS_FENCE (1<<0)
#define EXEC_OBJECT_NEEDS_GTT (1<<1)
#define EXEC_OBJECT_WRITE (1<<2)
-#define __EXEC_OBJECT_UNKNOWN_FLAGS -(EXEC_OBJECT_WRITE<<1)
+#define EXEC_OBJECT_SUPPORTS_48B_ADDRESS (1<<3)
+#define __EXEC_OBJECT_UNKNOWN_FLAGS -(EXEC_OBJECT_SUPPORTS_48B_ADDRESS<<1)
__u64 flags;
__u64 rsvd1;
#define DRM_NOUVEAU_EVENT_NVIF 0x80000000
-/* reserved object handles when using deprecated object APIs - these
- * are here so that libdrm can allow interoperability with the new
- * object APIs
- */
-#define NOUVEAU_ABI16_CLIENT 0xffffffff
-#define NOUVEAU_ABI16_DEVICE 0xdddddddd
-#define NOUVEAU_ABI16_CHAN(n) (0xcccc0000 | (n))
-
#define NOUVEAU_GEM_DOMAIN_CPU (1 << 0)
#define NOUVEAU_GEM_DOMAIN_VRAM (1 << 1)
#define NOUVEAU_GEM_DOMAIN_GART (1 << 2)
#ifndef __R128_DRM_H__
#define __R128_DRM_H__
+#include <drm/drm.h>
+
/* WARNING: If you change any of these defines, make sure to change the
* defines in the X server file (r128_sarea.h)
*/
#ifndef __SAVAGE_DRM_H__
#define __SAVAGE_DRM_H__
+#include <drm/drm.h>
+
#ifndef __SAVAGE_SAREA_DEFINES__
#define __SAVAGE_SAREA_DEFINES__
unsigned long offset, size;
} drm_sis_fb_t;
-struct sis_file_private {
- struct list_head obj_list;
-};
-
#endif /* __SIS_DRM_H__ */
drm_via_blitsync_t sync;
} drm_via_dmablit_t;
-struct via_file_private {
- struct list_head obj_list;
-};
-
#endif /* _VIA_DRM_H_ */
--- /dev/null
+/*
+ * Copyright 2013 Red Hat
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef VIRTGPU_DRM_H
+#define VIRTGPU_DRM_H
+
+#include <stddef.h>
+#include "drm/drm.h"
+
+/* Please note that modifications to all structs defined here are
+ * subject to backwards-compatibility constraints.
+ *
+ * Do not use pointers, use uint64_t instead for 32 bit / 64 bit user/kernel
+ * compatibility Keep fields aligned to their size
+ */
+
+#define DRM_VIRTGPU_MAP 0x01
+#define DRM_VIRTGPU_EXECBUFFER 0x02
+#define DRM_VIRTGPU_GETPARAM 0x03
+#define DRM_VIRTGPU_RESOURCE_CREATE 0x04
+#define DRM_VIRTGPU_RESOURCE_INFO 0x05
+#define DRM_VIRTGPU_TRANSFER_FROM_HOST 0x06
+#define DRM_VIRTGPU_TRANSFER_TO_HOST 0x07
+#define DRM_VIRTGPU_WAIT 0x08
+#define DRM_VIRTGPU_GET_CAPS 0x09
+
+struct drm_virtgpu_map {
+ uint64_t offset; /* use for mmap system call */
+ uint32_t handle;
+ uint32_t pad;
+};
+
+struct drm_virtgpu_execbuffer {
+ uint32_t flags; /* for future use */
+ uint32_t size;
+ uint64_t command; /* void* */
+ uint64_t bo_handles;
+ uint32_t num_bo_handles;
+ uint32_t pad;
+};
+
+#define VIRTGPU_PARAM_3D_FEATURES 1 /* do we have 3D features in the hw */
+
+struct drm_virtgpu_getparam {
+ uint64_t param;
+ uint64_t value;
+};
+
+/* NO_BO flags? NO resource flag? */
+/* resource flag for y_0_top */
+struct drm_virtgpu_resource_create {
+ uint32_t target;
+ uint32_t format;
+ uint32_t bind;
+ uint32_t width;
+ uint32_t height;
+ uint32_t depth;
+ uint32_t array_size;
+ uint32_t last_level;
+ uint32_t nr_samples;
+ uint32_t flags;
+ uint32_t bo_handle; /* if this is set - recreate a new resource attached to this bo ? */
+ uint32_t res_handle; /* returned by kernel */
+ uint32_t size; /* validate transfer in the host */
+ uint32_t stride; /* validate transfer in the host */
+};
+
+struct drm_virtgpu_resource_info {
+ uint32_t bo_handle;
+ uint32_t res_handle;
+ uint32_t size;
+ uint32_t stride;
+};
+
+struct drm_virtgpu_3d_box {
+ uint32_t x;
+ uint32_t y;
+ uint32_t z;
+ uint32_t w;
+ uint32_t h;
+ uint32_t d;
+};
+
+struct drm_virtgpu_3d_transfer_to_host {
+ uint32_t bo_handle;
+ struct drm_virtgpu_3d_box box;
+ uint32_t level;
+ uint32_t offset;
+};
+
+struct drm_virtgpu_3d_transfer_from_host {
+ uint32_t bo_handle;
+ struct drm_virtgpu_3d_box box;
+ uint32_t level;
+ uint32_t offset;
+};
+
+#define VIRTGPU_WAIT_NOWAIT 1 /* like it */
+struct drm_virtgpu_3d_wait {
+ uint32_t handle; /* 0 is an invalid handle */
+ uint32_t flags;
+};
+
+struct drm_virtgpu_get_caps {
+ uint32_t cap_set_id;
+ uint32_t cap_set_ver;
+ uint64_t addr;
+ uint32_t size;
+ uint32_t pad;
+};
+
+#define DRM_IOCTL_VIRTGPU_MAP \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_MAP, struct drm_virtgpu_map)
+
+#define DRM_IOCTL_VIRTGPU_EXECBUFFER \
+ DRM_IOW(DRM_COMMAND_BASE + DRM_VIRTGPU_EXECBUFFER,\
+ struct drm_virtgpu_execbuffer)
+
+#define DRM_IOCTL_VIRTGPU_GETPARAM \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GETPARAM,\
+ struct drm_virtgpu_getparam)
+
+#define DRM_IOCTL_VIRTGPU_RESOURCE_CREATE \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_CREATE, \
+ struct drm_virtgpu_resource_create)
+
+#define DRM_IOCTL_VIRTGPU_RESOURCE_INFO \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_INFO, \
+ struct drm_virtgpu_resource_info)
+
+#define DRM_IOCTL_VIRTGPU_TRANSFER_FROM_HOST \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_FROM_HOST, \
+ struct drm_virtgpu_3d_transfer_from_host)
+
+#define DRM_IOCTL_VIRTGPU_TRANSFER_TO_HOST \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_TO_HOST, \
+ struct drm_virtgpu_3d_transfer_to_host)
+
+#define DRM_IOCTL_VIRTGPU_WAIT \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_WAIT, \
+ struct drm_virtgpu_3d_wait)
+
+#define DRM_IOCTL_VIRTGPU_GET_CAPS \
+ DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GET_CAPS, \
+ struct drm_virtgpu_get_caps)
+
+#endif
LWTUNNEL_IP_SRC,
LWTUNNEL_IP_TTL,
LWTUNNEL_IP_TOS,
- LWTUNNEL_IP_SPORT,
- LWTUNNEL_IP_DPORT,
LWTUNNEL_IP_FLAGS,
__LWTUNNEL_IP_MAX,
};
LWTUNNEL_IP6_SRC,
LWTUNNEL_IP6_HOPLIMIT,
LWTUNNEL_IP6_TC,
- LWTUNNEL_IP6_SPORT,
- LWTUNNEL_IP6_DPORT,
LWTUNNEL_IP6_FLAGS,
__LWTUNNEL_IP6_MAX,
};
#include <linux/types.h>
-#include <linux/compiler.h>
-
#define UFFD_API ((__u64)0xAA)
/*
* After implementing the respective features it will become:
#include <linux/types.h>
+#define VIRTIO_GPU_F_VIRGL 0
+
enum virtio_gpu_ctrl_type {
VIRTIO_GPU_UNDEFINED = 0,
VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D,
VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING,
VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING,
+ VIRTIO_GPU_CMD_GET_CAPSET_INFO,
+ VIRTIO_GPU_CMD_GET_CAPSET,
+
+ /* 3d commands */
+ VIRTIO_GPU_CMD_CTX_CREATE = 0x0200,
+ VIRTIO_GPU_CMD_CTX_DESTROY,
+ VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE,
+ VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE,
+ VIRTIO_GPU_CMD_RESOURCE_CREATE_3D,
+ VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D,
+ VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D,
+ VIRTIO_GPU_CMD_SUBMIT_3D,
/* cursor commands */
VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
/* success responses */
VIRTIO_GPU_RESP_OK_NODATA = 0x1100,
VIRTIO_GPU_RESP_OK_DISPLAY_INFO,
+ VIRTIO_GPU_RESP_OK_CAPSET_INFO,
+ VIRTIO_GPU_RESP_OK_CAPSET,
/* error responses */
VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
} pmodes[VIRTIO_GPU_MAX_SCANOUTS];
};
+/* data passed in the control vq, 3d related */
+
+struct virtio_gpu_box {
+ __le32 x, y, z;
+ __le32 w, h, d;
+};
+
+/* VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D, VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D */
+struct virtio_gpu_transfer_host_3d {
+ struct virtio_gpu_ctrl_hdr hdr;
+ struct virtio_gpu_box box;
+ __le64 offset;
+ __le32 resource_id;
+ __le32 level;
+ __le32 stride;
+ __le32 layer_stride;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_CREATE_3D */
+#define VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP (1 << 0)
+struct virtio_gpu_resource_create_3d {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 resource_id;
+ __le32 target;
+ __le32 format;
+ __le32 bind;
+ __le32 width;
+ __le32 height;
+ __le32 depth;
+ __le32 array_size;
+ __le32 last_level;
+ __le32 nr_samples;
+ __le32 flags;
+ __le32 padding;
+};
+
+/* VIRTIO_GPU_CMD_CTX_CREATE */
+struct virtio_gpu_ctx_create {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 nlen;
+ __le32 padding;
+ char debug_name[64];
+};
+
+/* VIRTIO_GPU_CMD_CTX_DESTROY */
+struct virtio_gpu_ctx_destroy {
+ struct virtio_gpu_ctrl_hdr hdr;
+};
+
+/* VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE, VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE */
+struct virtio_gpu_ctx_resource {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 resource_id;
+ __le32 padding;
+};
+
+/* VIRTIO_GPU_CMD_SUBMIT_3D */
+struct virtio_gpu_cmd_submit {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 size;
+ __le32 padding;
+};
+
+#define VIRTIO_GPU_CAPSET_VIRGL 1
+
+/* VIRTIO_GPU_CMD_GET_CAPSET_INFO */
+struct virtio_gpu_get_capset_info {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 capset_index;
+ __le32 padding;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET_INFO */
+struct virtio_gpu_resp_capset_info {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 capset_id;
+ __le32 capset_max_version;
+ __le32 capset_max_size;
+ __le32 padding;
+};
+
+/* VIRTIO_GPU_CMD_GET_CAPSET */
+struct virtio_gpu_get_capset {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 capset_id;
+ __le32 capset_version;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET */
+struct virtio_gpu_resp_capset {
+ struct virtio_gpu_ctrl_hdr hdr;
+ uint8_t capset_data[];
+};
+
#define VIRTIO_GPU_EVENT_DISPLAY (1 << 0)
struct virtio_gpu_config {
__u32 events_read;
__u32 events_clear;
__u32 num_scanouts;
- __u32 reserved;
+ __u32 num_capsets;
};
/* simple formats for fbcon/X use */
/* VIDCON0 */
#define VIDCON0_SWRESET (1 << 28)
+#define VIDCON0_CLKVALUP (1 << 14)
+#define VIDCON0_VLCKFREE (1 << 5)
#define VIDCON0_STOP_STATUS (1 << 2)
#define VIDCON0_ENVID (1 << 1)
#define VIDCON0_ENVID_F (1 << 0)
/* DECON_UPDATE */
#define STANDALONE_UPDATE_F (1 << 0)
+/* DECON_VIDCON1 */
+#define VIDCON1_VCLK_MASK (0x3 << 9)
+#define VIDCON1_VCLK_RUN_VDEN_DISABLE (0x3 << 9)
+#define VIDCON1_VCLK_HOLD (0x0 << 9)
+#define VIDCON1_VCLK_RUN (0x1 << 9)
+
+
/* DECON_VIDTCON00 */
#define VIDTCON00_VBPD_F(x) (((x) & 0xfff) << 16)
#define VIDTCON00_VFPD_F(x) ((x) & 0xfff)
#define TRIGCON_TRIGEN_PER_F (1 << 31)
#define TRIGCON_TRIGEN_F (1 << 30)
#define TRIGCON_TE_AUTO_MASK (1 << 29)
+#define TRIGCON_WB_SWTRIGCMD (1 << 28)
+#define TRIGCON_SWTRIGCMD_W4BUF (1 << 26)
+#define TRIGCON_TRIGMODE_W4BUF (1 << 25)
+#define TRIGCON_SWTRIGCMD_W3BUF (1 << 21)
+#define TRIGCON_TRIGMODE_W3BUF (1 << 20)
+#define TRIGCON_SWTRIGCMD_W2BUF (1 << 16)
+#define TRIGCON_TRIGMODE_W2BUF (1 << 15)
+#define TRIGCON_SWTRIGCMD_W1BUF (1 << 11)
+#define TRIGCON_TRIGMODE_W1BUF (1 << 10)
+#define TRIGCON_SWTRIGCMD_W0BUF (1 << 6)
+#define TRIGCON_TRIGMODE_W0BUF (1 << 5)
+#define TRIGCON_HWTRIGMASK_I80_RGB (1 << 4)
+#define TRIGCON_HWTRIGEN_I80_RGB (1 << 3)
+#define TRIGCON_HWTRIG_INV_I80_RGB (1 << 2)
#define TRIGCON_SWTRIGCMD (1 << 1)
#define TRIGCON_SWTRIGEN (1 << 0)
+/* DECON_CRCCTRL */
+#define CRCCTRL_CRCCLKEN (0x1 << 2)
+#define CRCCTRL_CRCSTART_F (0x1 << 1)
+#define CRCCTRL_CRCEN (0x1 << 0)
+#define CRCCTRL_MASK (0x7)
+
#endif /* EXYNOS_REGS_DECON_H */
#define SHUTDOWN_suspend 2 /* Clean up, save suspend info, kill. */
#define SHUTDOWN_crash 3 /* Tell controller we've crashed. */
#define SHUTDOWN_watchdog 4 /* Restart because watchdog time expired. */
+/*
+ * Domain asked to perform 'soft reset' for it. The expected behavior is to
+ * reset internal Xen state for the domain returning it to the point where it
+ * was created but leaving the domain's memory contents and vCPU contexts
+ * intact. This will allow the domain to start over and set up all Xen specific
+ * interfaces again.
+ */
+#define SHUTDOWN_soft_reset 5
#endif /* __XEN_PUBLIC_SCHED_H__ */
return retval;
}
- /* ipc_addid() locks msq upon success. */
- id = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni);
- if (id < 0) {
- ipc_rcu_putref(msq, msg_rcu_free);
- return id;
- }
-
msq->q_stime = msq->q_rtime = 0;
msq->q_ctime = get_seconds();
msq->q_cbytes = msq->q_qnum = 0;
INIT_LIST_HEAD(&msq->q_receivers);
INIT_LIST_HEAD(&msq->q_senders);
+ /* ipc_addid() locks msq upon success. */
+ id = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni);
+ if (id < 0) {
+ ipc_rcu_putref(msq, msg_rcu_free);
+ return id;
+ }
+
ipc_unlock_object(&msq->q_perm);
rcu_read_unlock();
if (IS_ERR(file))
goto no_file;
- id = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
- if (id < 0) {
- error = id;
- goto no_id;
- }
-
shp->shm_cprid = task_tgid_vnr(current);
shp->shm_lprid = 0;
shp->shm_atim = shp->shm_dtim = 0;
shp->shm_nattch = 0;
shp->shm_file = file;
shp->shm_creator = current;
+
+ id = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
+ if (id < 0) {
+ error = id;
+ goto no_id;
+ }
+
list_add(&shp->shm_clist, ¤t->sysvshm.shm_clist);
/*
rcu_read_lock();
spin_lock(&new->lock);
+ current_euid_egid(&euid, &egid);
+ new->cuid = new->uid = euid;
+ new->gid = new->cgid = egid;
+
id = idr_alloc(&ids->ipcs_idr, new,
(next_id < 0) ? 0 : ipcid_to_idx(next_id), 0,
GFP_NOWAIT);
ids->in_use++;
- current_euid_egid(&euid, &egid);
- new->cuid = new->uid = euid;
- new->gid = new->cgid = egid;
-
if (next_id < 0) {
new->seq = ids->seq++;
if (ids->seq > IPCID_SEQ_MAX)
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/rwsem.h>
-#include <linux/percpu-rwsem.h>
#include <linux/string.h>
#include <linux/sort.h>
#include <linux/kmod.h>
*/
static DEFINE_SPINLOCK(release_agent_path_lock);
-struct percpu_rw_semaphore cgroup_threadgroup_rwsem;
-
#define cgroup_assert_mutex_or_rcu_locked() \
RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
!lockdep_is_held(&cgroup_mutex), \
return cset;
}
+void cgroup_threadgroup_change_begin(struct task_struct *tsk)
+{
+ down_read(&tsk->signal->group_rwsem);
+}
+
+void cgroup_threadgroup_change_end(struct task_struct *tsk)
+{
+ up_read(&tsk->signal->group_rwsem);
+}
+
+/**
+ * threadgroup_lock - lock threadgroup
+ * @tsk: member task of the threadgroup to lock
+ *
+ * Lock the threadgroup @tsk belongs to. No new task is allowed to enter
+ * and member tasks aren't allowed to exit (as indicated by PF_EXITING) or
+ * change ->group_leader/pid. This is useful for cases where the threadgroup
+ * needs to stay stable across blockable operations.
+ *
+ * fork and exit explicitly call threadgroup_change_{begin|end}() for
+ * synchronization. While held, no new task will be added to threadgroup
+ * and no existing live task will have its PF_EXITING set.
+ *
+ * de_thread() does threadgroup_change_{begin|end}() when a non-leader
+ * sub-thread becomes a new leader.
+ */
+static void threadgroup_lock(struct task_struct *tsk)
+{
+ down_write(&tsk->signal->group_rwsem);
+}
+
+/**
+ * threadgroup_unlock - unlock threadgroup
+ * @tsk: member task of the threadgroup to unlock
+ *
+ * Reverse threadgroup_lock().
+ */
+static inline void threadgroup_unlock(struct task_struct *tsk)
+{
+ up_write(&tsk->signal->group_rwsem);
+}
+
static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
{
struct cgroup *root_cgrp = kf_root->kn->priv;
lockdep_assert_held(&css_set_rwsem);
/*
- * We are synchronized through cgroup_threadgroup_rwsem against
- * PF_EXITING setting such that we can't race against cgroup_exit()
- * changing the css_set to init_css_set and dropping the old one.
+ * We are synchronized through threadgroup_lock() against PF_EXITING
+ * setting such that we can't race against cgroup_exit() changing the
+ * css_set to init_css_set and dropping the old one.
*/
WARN_ON_ONCE(tsk->flags & PF_EXITING);
old_cset = task_css_set(tsk);
* @src_cset and add it to @preloaded_csets, which should later be cleaned
* up by cgroup_migrate_finish().
*
- * This function may be called without holding cgroup_threadgroup_rwsem
- * even if the target is a process. Threads may be created and destroyed
- * but as long as cgroup_mutex is not dropped, no new css_set can be put
- * into play and the preloaded css_sets are guaranteed to cover all
- * migrations.
+ * This function may be called without holding threadgroup_lock even if the
+ * target is a process. Threads may be created and destroyed but as long
+ * as cgroup_mutex is not dropped, no new css_set can be put into play and
+ * the preloaded css_sets are guaranteed to cover all migrations.
*/
static void cgroup_migrate_add_src(struct css_set *src_cset,
struct cgroup *dst_cgrp,
* @threadgroup: whether @leader points to the whole process or a single task
*
* Migrate a process or task denoted by @leader to @cgrp. If migrating a
- * process, the caller must be holding cgroup_threadgroup_rwsem. The
+ * process, the caller must be holding threadgroup_lock of @leader. The
* caller is also responsible for invoking cgroup_migrate_add_src() and
* cgroup_migrate_prepare_dst() on the targets before invoking this
* function and following up with cgroup_migrate_finish().
* @leader: the task or the leader of the threadgroup to be attached
* @threadgroup: attach the whole threadgroup?
*
- * Call holding cgroup_mutex and cgroup_threadgroup_rwsem.
+ * Call holding cgroup_mutex and threadgroup_lock of @leader.
*/
static int cgroup_attach_task(struct cgroup *dst_cgrp,
struct task_struct *leader, bool threadgroup)
if (!cgrp)
return -ENODEV;
- percpu_down_write(&cgroup_threadgroup_rwsem);
+retry_find_task:
rcu_read_lock();
if (pid) {
tsk = find_task_by_vpid(pid);
if (!tsk) {
+ rcu_read_unlock();
ret = -ESRCH;
- goto out_unlock_rcu;
+ goto out_unlock_cgroup;
}
} else {
tsk = current;
*/
if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
ret = -EINVAL;
- goto out_unlock_rcu;
+ rcu_read_unlock();
+ goto out_unlock_cgroup;
}
get_task_struct(tsk);
rcu_read_unlock();
+ threadgroup_lock(tsk);
+ if (threadgroup) {
+ if (!thread_group_leader(tsk)) {
+ /*
+ * a race with de_thread from another thread's exec()
+ * may strip us of our leadership, if this happens,
+ * there is no choice but to throw this task away and
+ * try again; this is
+ * "double-double-toil-and-trouble-check locking".
+ */
+ threadgroup_unlock(tsk);
+ put_task_struct(tsk);
+ goto retry_find_task;
+ }
+ }
+
ret = cgroup_procs_write_permission(tsk, cgrp, of);
if (!ret)
ret = cgroup_attach_task(cgrp, tsk, threadgroup);
- put_task_struct(tsk);
- goto out_unlock_threadgroup;
+ threadgroup_unlock(tsk);
-out_unlock_rcu:
- rcu_read_unlock();
-out_unlock_threadgroup:
- percpu_up_write(&cgroup_threadgroup_rwsem);
+ put_task_struct(tsk);
+out_unlock_cgroup:
cgroup_kn_unlock(of->kn);
return ret ?: nbytes;
}
lockdep_assert_held(&cgroup_mutex);
- percpu_down_write(&cgroup_threadgroup_rwsem);
-
/* look up all csses currently attached to @cgrp's subtree */
down_read(&css_set_rwsem);
css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
goto out_finish;
last_task = task;
+ threadgroup_lock(task);
+ /* raced against de_thread() from another thread? */
+ if (!thread_group_leader(task)) {
+ threadgroup_unlock(task);
+ put_task_struct(task);
+ continue;
+ }
+
ret = cgroup_migrate(src_cset->dfl_cgrp, task, true);
+ threadgroup_unlock(task);
put_task_struct(task);
if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret))
out_finish:
cgroup_migrate_finish(&preloaded_csets);
- percpu_up_write(&cgroup_threadgroup_rwsem);
return ret;
}
unsigned long key;
int ssid, err;
- BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem));
BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files));
BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files));
PERF_EVENT_STATE_INACTIVE;
}
-/*
- * Called at perf_event creation and when events are attached/detached from a
- * group.
- */
-static void perf_event__read_size(struct perf_event *event)
+static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
{
int entry = sizeof(u64); /* value */
int size = 0;
entry += sizeof(u64);
if (event->attr.read_format & PERF_FORMAT_GROUP) {
- nr += event->group_leader->nr_siblings;
+ nr += nr_siblings;
size += sizeof(u64);
}
event->read_size = size;
}
-static void perf_event__header_size(struct perf_event *event)
+static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
{
struct perf_sample_data *data;
- u64 sample_type = event->attr.sample_type;
u16 size = 0;
- perf_event__read_size(event);
-
if (sample_type & PERF_SAMPLE_IP)
size += sizeof(data->ip);
event->header_size = size;
}
+/*
+ * Called at perf_event creation and when events are attached/detached from a
+ * group.
+ */
+static void perf_event__header_size(struct perf_event *event)
+{
+ __perf_event_read_size(event,
+ event->group_leader->nr_siblings);
+ __perf_event_header_size(event, event->attr.sample_type);
+}
+
static void perf_event__id_header_size(struct perf_event *event)
{
struct perf_sample_data *data;
event->id_header_size = size;
}
+static bool perf_event_validate_size(struct perf_event *event)
+{
+ /*
+ * The values computed here will be over-written when we actually
+ * attach the event.
+ */
+ __perf_event_read_size(event, event->group_leader->nr_siblings + 1);
+ __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);
+ perf_event__id_header_size(event);
+
+ /*
+ * Sum the lot; should not exceed the 64k limit we have on records.
+ * Conservative limit to allow for callchains and other variable fields.
+ */
+ if (event->read_size + event->header_size +
+ event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
+ return false;
+
+ return true;
+}
+
static void perf_group_attach(struct perf_event *event)
{
struct perf_event *group_leader = event->group_leader, *pos;
if (move_group) {
gctx = group_leader->ctx;
+ mutex_lock_double(&gctx->mutex, &ctx->mutex);
+ } else {
+ mutex_lock(&ctx->mutex);
+ }
+ if (!perf_event_validate_size(event)) {
+ err = -E2BIG;
+ goto err_locked;
+ }
+
+ /*
+ * Must be under the same ctx::mutex as perf_install_in_context(),
+ * because we need to serialize with concurrent event creation.
+ */
+ if (!exclusive_event_installable(event, ctx)) {
+ /* exclusive and group stuff are assumed mutually exclusive */
+ WARN_ON_ONCE(move_group);
+
+ err = -EBUSY;
+ goto err_locked;
+ }
+
+ WARN_ON_ONCE(ctx->parent_ctx);
+
+ if (move_group) {
/*
* See perf_event_ctx_lock() for comments on the details
* of swizzling perf_event::ctx.
*/
- mutex_lock_double(&gctx->mutex, &ctx->mutex);
-
perf_remove_from_context(group_leader, false);
list_for_each_entry(sibling, &group_leader->sibling_list,
perf_remove_from_context(sibling, false);
put_ctx(gctx);
}
- } else {
- mutex_lock(&ctx->mutex);
- }
- WARN_ON_ONCE(ctx->parent_ctx);
-
- if (move_group) {
/*
* Wait for everybody to stop referencing the events through
* the old lists, before installing it on new lists.
perf_event__state_init(group_leader);
perf_install_in_context(ctx, group_leader, group_leader->cpu);
get_ctx(ctx);
- }
- if (!exclusive_event_installable(event, ctx)) {
- err = -EBUSY;
- mutex_unlock(&ctx->mutex);
- fput(event_file);
- goto err_context;
+ /*
+ * Now that all events are installed in @ctx, nothing
+ * references @gctx anymore, so drop the last reference we have
+ * on it.
+ */
+ put_ctx(gctx);
}
+ /*
+ * Precalculate sample_data sizes; do while holding ctx::mutex such
+ * that we're serialized against further additions and before
+ * perf_install_in_context() which is the point the event is active and
+ * can use these values.
+ */
+ perf_event__header_size(event);
+ perf_event__id_header_size(event);
+
perf_install_in_context(ctx, event, event->cpu);
perf_unpin_context(ctx);
- if (move_group) {
+ if (move_group)
mutex_unlock(&gctx->mutex);
- put_ctx(gctx);
- }
mutex_unlock(&ctx->mutex);
put_online_cpus();
list_add_tail(&event->owner_entry, ¤t->perf_event_list);
mutex_unlock(¤t->perf_event_mutex);
- /*
- * Precalculate sample_data sizes
- */
- perf_event__header_size(event);
- perf_event__id_header_size(event);
-
/*
* Drop the reference on the group_event after placing the
* new event on the sibling_list. This ensures destruction
fd_install(event_fd, event_file);
return event_fd;
+err_locked:
+ if (move_group)
+ mutex_unlock(&gctx->mutex);
+ mutex_unlock(&ctx->mutex);
+/* err_file: */
+ fput(event_file);
err_context:
perf_unpin_context(ctx);
put_ctx(ctx);
tty_audit_fork(sig);
sched_autogroup_fork(sig);
+#ifdef CONFIG_CGROUPS
+ init_rwsem(&sig->group_rwsem);
+#endif
+
sig->oom_score_adj = current->signal->oom_score_adj;
sig->oom_score_adj_min = current->signal->oom_score_adj_min;
if (!desc)
return -EINVAL;
- desc->irq_data.handler_data = data;
+ desc->irq_common_data.handler_data = data;
irq_put_desc_unlock(desc, flags);
return 0;
}
if (!desc)
return -EINVAL;
- desc->irq_data.msi_desc = entry;
+ desc->irq_common_data.msi_desc = entry;
if (entry && !irq_offset)
entry->irq = irq_base;
irq_put_desc_unlock(desc, flags);
/**
* handle_simple_irq - Simple and software-decoded IRQs.
- * @irq: the interrupt number
* @desc: the interrupt description structure for this irq
*
* Simple interrupts are either sent from a demultiplexing interrupt
* Note: The caller is expected to handle the ack, clear, mask and
* unmask issues if necessary.
*/
-void
-handle_simple_irq(unsigned int irq, struct irq_desc *desc)
+void handle_simple_irq(struct irq_desc *desc)
{
raw_spin_lock(&desc->lock);
/**
* handle_level_irq - Level type irq handler
- * @irq: the interrupt number
* @desc: the interrupt description structure for this irq
*
* Level type interrupts are active as long as the hardware line has
* it after the associated handler has acknowledged the device, so the
* interrupt line is back to inactive.
*/
-void
-handle_level_irq(unsigned int irq, struct irq_desc *desc)
+void handle_level_irq(struct irq_desc *desc)
{
raw_spin_lock(&desc->lock);
mask_ack_irq(desc);
/**
* handle_fasteoi_irq - irq handler for transparent controllers
- * @irq: the interrupt number
* @desc: the interrupt description structure for this irq
*
* Only a single callback will be issued to the chip: an ->eoi()
* for modern forms of interrupt handlers, which handle the flow
* details in hardware, transparently.
*/
-void
-handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
+void handle_fasteoi_irq(struct irq_desc *desc)
{
struct irq_chip *chip = desc->irq_data.chip;
/**
* handle_edge_irq - edge type IRQ handler
- * @irq: the interrupt number
* @desc: the interrupt description structure for this irq
*
* Interrupt occures on the falling and/or rising edge of a hardware
* the handler was running. If all pending interrupts are handled, the
* loop is left.
*/
-void
-handle_edge_irq(unsigned int irq, struct irq_desc *desc)
+void handle_edge_irq(struct irq_desc *desc)
{
raw_spin_lock(&desc->lock);
#ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
/**
* handle_edge_eoi_irq - edge eoi type IRQ handler
- * @irq: the interrupt number
* @desc: the interrupt description structure for this irq
*
* Similar as the above handle_edge_irq, but using eoi and w/o the
* mask/unmask logic.
*/
-void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc)
+void handle_edge_eoi_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
/**
* handle_percpu_irq - Per CPU local irq handler
- * @irq: the interrupt number
* @desc: the interrupt description structure for this irq
*
* Per CPU interrupts on SMP machines without locking requirements
*/
-void
-handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
+void handle_percpu_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
/**
* handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
- * @irq: the interrupt number
* @desc: the interrupt description structure for this irq
*
* Per CPU interrupts on SMP machines without locking requirements. Same as
* contain the real device id for the cpu on which this handler is
* called
*/
-void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc)
+void handle_percpu_devid_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct irqaction *action = desc->action;
void *dev_id = raw_cpu_ptr(action->percpu_dev_id);
+ unsigned int irq = irq_desc_get_irq(desc);
irqreturn_t res;
kstat_incr_irqs_this_cpu(desc);
return;
__irq_do_set_handler(desc, handle, 1, NULL);
- desc->irq_data.handler_data = data;
+ desc->irq_common_data.handler_data = data;
irq_put_desc_busunlock(desc, flags);
}
/**
* handle_bad_irq - handle spurious and unhandled irqs
- * @irq: the interrupt number
* @desc: description of the interrupt
*
* Handles spurious and unhandled IRQ's. It also prints a debugmessage.
*/
-void handle_bad_irq(unsigned int irq, struct irq_desc *desc)
+void handle_bad_irq(struct irq_desc *desc)
{
+ unsigned int irq = irq_desc_get_irq(desc);
+
print_irq_desc(irq, desc);
kstat_incr_irqs_this_cpu(desc);
ack_bad_irq(irq);
}
+EXPORT_SYMBOL_GPL(handle_bad_irq);
/*
* Special, empty irq handler:
static inline int irq_desc_get_node(struct irq_desc *desc)
{
- return irq_data_get_node(&desc->irq_data);
+ return irq_common_data_get_node(&desc->irq_common_data);
}
#ifdef CONFIG_PM_SLEEP
#ifdef CONFIG_SMP
static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
{
- if (!zalloc_cpumask_var_node(&desc->irq_data.affinity, gfp, node))
+ if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
+ gfp, node))
return -ENOMEM;
#ifdef CONFIG_GENERIC_PENDING_IRQ
if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
- free_cpumask_var(desc->irq_data.affinity);
+ free_cpumask_var(desc->irq_common_data.affinity);
return -ENOMEM;
}
#endif
static void desc_smp_init(struct irq_desc *desc, int node)
{
- desc->irq_data.node = node;
- cpumask_copy(desc->irq_data.affinity, irq_default_affinity);
+ cpumask_copy(desc->irq_common_data.affinity, irq_default_affinity);
#ifdef CONFIG_GENERIC_PENDING_IRQ
cpumask_clear(desc->pending_mask);
#endif
+#ifdef CONFIG_NUMA
+ desc->irq_common_data.node = node;
+#endif
}
#else
{
int cpu;
+ desc->irq_common_data.handler_data = NULL;
+ desc->irq_common_data.msi_desc = NULL;
+
desc->irq_data.common = &desc->irq_common_data;
desc->irq_data.irq = irq;
desc->irq_data.chip = &no_irq_chip;
desc->irq_data.chip_data = NULL;
- desc->irq_data.handler_data = NULL;
- desc->irq_data.msi_desc = NULL;
irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
desc->handle_irq = handle_bad_irq;
#ifdef CONFIG_GENERIC_PENDING_IRQ
free_cpumask_var(desc->pending_mask);
#endif
- free_cpumask_var(desc->irq_data.affinity);
+ free_cpumask_var(desc->irq_common_data.affinity);
}
#else
static inline void free_masks(struct irq_desc *desc) { }
if (!desc)
return -EINVAL;
- generic_handle_irq_desc(irq, desc);
+ generic_handle_irq_desc(desc);
return 0;
}
EXPORT_SYMBOL_GPL(generic_handle_irq);
child->parent_data = irq_data;
irq_data->irq = child->irq;
irq_data->common = child->common;
- irq_data->node = child->node;
irq_data->domain = domain;
}
switch (ret) {
case IRQ_SET_MASK_OK:
case IRQ_SET_MASK_OK_DONE:
- cpumask_copy(data->affinity, mask);
+ cpumask_copy(desc->irq_common_data.affinity, mask);
case IRQ_SET_MASK_OK_NOCOPY:
irq_set_thread_affinity(desc);
ret = 0;
if (irq_move_pending(&desc->irq_data))
irq_get_pending(cpumask, desc);
else
- cpumask_copy(cpumask, desc->irq_data.affinity);
+ cpumask_copy(cpumask, desc->irq_common_data.affinity);
raw_spin_unlock_irqrestore(&desc->lock, flags);
notify->notify(notify, cpumask);
* one of the targets is online.
*/
if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
- if (cpumask_intersects(desc->irq_data.affinity,
+ if (cpumask_intersects(desc->irq_common_data.affinity,
cpu_online_mask))
- set = desc->irq_data.affinity;
+ set = desc->irq_common_data.affinity;
else
irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
}
* This code is triggered unconditionally. Check the affinity
* mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
*/
- if (desc->irq_data.affinity)
- cpumask_copy(mask, desc->irq_data.affinity);
+ if (desc->irq_common_data.affinity)
+ cpumask_copy(mask, desc->irq_common_data.affinity);
else
valid = false;
raw_spin_unlock_irq(&desc->lock);
#include <linux/seq_file.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
+#include <linux/mutex.h>
#include "internals.h"
static int show_irq_affinity(int type, struct seq_file *m, void *v)
{
struct irq_desc *desc = irq_to_desc((long)m->private);
- const struct cpumask *mask = desc->irq_data.affinity;
+ const struct cpumask *mask = desc->irq_common_data.affinity;
#ifdef CONFIG_GENERIC_PENDING_IRQ
if (irqd_is_setaffinity_pending(&desc->irq_data))
void register_irq_proc(unsigned int irq, struct irq_desc *desc)
{
+ static DEFINE_MUTEX(register_lock);
char name [MAX_NAMELEN];
- if (!root_irq_dir || (desc->irq_data.chip == &no_irq_chip) || desc->dir)
+ if (!root_irq_dir || (desc->irq_data.chip == &no_irq_chip))
return;
+ /*
+ * irq directories are registered only when a handler is
+ * added, not when the descriptor is created, so multiple
+ * tasks might try to register at the same time.
+ */
+ mutex_lock(®ister_lock);
+
+ if (desc->dir)
+ goto out_unlock;
+
memset(name, 0, MAX_NAMELEN);
sprintf(name, "%d", irq);
/* create /proc/irq/1234 */
desc->dir = proc_mkdir(name, root_irq_dir);
if (!desc->dir)
- return;
+ goto out_unlock;
#ifdef CONFIG_SMP
/* create /proc/irq/<irq>/smp_affinity */
proc_create_data("spurious", 0444, desc->dir,
&irq_spurious_proc_fops, (void *)(long)irq);
+
+out_unlock:
+ mutex_unlock(®ister_lock);
}
void unregister_irq_proc(unsigned int irq, struct irq_desc *desc)
clear_bit(irq, irqs_resend);
desc = irq_to_desc(irq);
local_irq_disable();
- desc->handle_irq(irq, desc);
+ desc->handle_irq(desc);
local_irq_enable();
}
}
static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
int trylock, int read, int check, int hardirqs_off,
struct lockdep_map *nest_lock, unsigned long ip,
- int references)
+ int references, int pin_count)
{
struct task_struct *curr = current;
struct lock_class *class = NULL;
hlock->waittime_stamp = 0;
hlock->holdtime_stamp = lockstat_clock();
#endif
- hlock->pin_count = 0;
+ hlock->pin_count = pin_count;
if (check && !mark_irqflags(curr, hlock))
return 0;
hlock_class(hlock)->subclass, hlock->trylock,
hlock->read, hlock->check, hlock->hardirqs_off,
hlock->nest_lock, hlock->acquire_ip,
- hlock->references))
+ hlock->references, hlock->pin_count))
return 0;
}
hlock_class(hlock)->subclass, hlock->trylock,
hlock->read, hlock->check, hlock->hardirqs_off,
hlock->nest_lock, hlock->acquire_ip,
- hlock->references))
+ hlock->references, hlock->pin_count))
return 0;
}
current->lockdep_recursion = 1;
trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
__lock_acquire(lock, subclass, trylock, read, check,
- irqs_disabled_flags(flags), nest_lock, ip, 0);
+ irqs_disabled_flags(flags), nest_lock, ip, 0, 0);
current->lockdep_recursion = 0;
raw_local_irq_restore(flags);
}
if (pv_enabled())
goto queue;
- if (virt_queued_spin_lock(lock))
+ if (virt_spin_lock(lock))
return;
/*
static void __init
rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
{
+ static struct lock_class_key rcu_exp_sched_rdp_class;
unsigned long flags;
struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
struct rcu_node *rnp = rcu_get_root(rsp);
mutex_init(&rdp->exp_funnel_mutex);
rcu_boot_init_nocb_percpu_data(rdp);
raw_spin_unlock_irqrestore(&rnp->lock, flags);
+ if (rsp == &rcu_sched_state)
+ lockdep_set_class_and_name(&rdp->exp_funnel_mutex,
+ &rcu_exp_sched_rdp_class,
+ "rcu_data_exp_sched");
}
/*
int i, cpu = smp_processor_id();
struct sched_domain *sd;
- if (!idle_cpu(cpu))
+ if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
return cpu;
rcu_read_lock();
for_each_domain(cpu, sd) {
for_each_cpu(i, sched_domain_span(sd)) {
- if (!idle_cpu(i)) {
+ if (!idle_cpu(i) && is_housekeeping_cpu(cpu)) {
cpu = i;
goto unlock;
}
}
}
+
+ if (!is_housekeeping_cpu(cpu))
+ cpu = housekeeping_any_cpu();
unlock:
rcu_read_unlock();
return cpu;
* If a task dies, then it sets TASK_DEAD in tsk->state and calls
* schedule one last time. The schedule call will never return, and
* the scheduled task must drop that reference.
- * The test for TASK_DEAD must occur while the runqueue locks are
- * still held, otherwise prev could be scheduled on another cpu, die
- * there before we look at prev->state, and then the reference would
- * be dropped twice.
- * Manfred Spraul <manfred@colorfullife.com>
+ *
+ * We must observe prev->state before clearing prev->on_cpu (in
+ * finish_lock_switch), otherwise a concurrent wakeup can get prev
+ * running on another CPU and we could rave with its RUNNING -> DEAD
+ * transition, resulting in a double drop.
*/
prev_state = prev->state;
vtime_task_switch(prev);
/*
* Check if only the current task is running on the cpu.
+ *
+ * Caution: this function does not check that the caller has disabled
+ * preemption, thus the result might have a time-of-check-to-time-of-use
+ * race. The caller is responsible to use it correctly, for example:
+ *
+ * - from a non-preemptable section (of course)
+ *
+ * - from a thread that is bound to a single CPU
+ *
+ * - in a loop with very short iterations (e.g. a polling loop)
*/
bool single_task_running(void)
{
- if (cpu_rq(smp_processor_id())->nr_running == 1)
- return true;
- else
- return false;
+ return raw_rq()->nr_running == 1;
}
EXPORT_SYMBOL(single_task_running);
idle->state = TASK_RUNNING;
idle->se.exec_start = sched_clock();
- do_set_cpus_allowed(idle, cpumask_of(cpu));
+#ifdef CONFIG_SMP
+ /*
+ * Its possible that init_idle() gets called multiple times on a task,
+ * in that case do_set_cpus_allowed() will not do the right thing.
+ *
+ * And since this is boot we can forgo the serialization.
+ */
+ set_cpus_allowed_common(idle, cpumask_of(cpu));
+#endif
/*
* We're having a chicken and egg problem, even though we are
* holding rq->lock, the cpu isn't yet set to this cpu so the
rq->curr = rq->idle = idle;
idle->on_rq = TASK_ON_RQ_QUEUED;
-#if defined(CONFIG_SMP)
+#ifdef CONFIG_SMP
idle->on_cpu = 1;
#endif
raw_spin_unlock(&rq->lock);
idle->sched_class = &idle_sched_class;
ftrace_graph_init_idle_task(idle, cpu);
vtime_init_idle(idle, cpu);
-#if defined(CONFIG_SMP)
+#ifdef CONFIG_SMP
sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
#endif
}
break;
/*
- * Ensure rq->lock covers the entire task selection
- * until the migration.
+ * pick_next_task assumes pinned rq->lock.
*/
lockdep_pin_lock(&rq->lock);
next = pick_next_task(rq, &fake_task);
BUG_ON(!next);
next->sched_class->put_prev_task(rq, next);
+ /*
+ * Rules for changing task_struct::cpus_allowed are holding
+ * both pi_lock and rq->lock, such that holding either
+ * stabilizes the mask.
+ *
+ * Drop rq->lock is not quite as disastrous as it usually is
+ * because !cpu_active at this point, which means load-balance
+ * will not interfere. Also, stop-machine.
+ */
+ lockdep_unpin_lock(&rq->lock);
+ raw_spin_unlock(&rq->lock);
+ raw_spin_lock(&next->pi_lock);
+ raw_spin_lock(&rq->lock);
+
+ /*
+ * Since we're inside stop-machine, _nothing_ should have
+ * changed the task, WARN if weird stuff happened, because in
+ * that case the above rq->lock drop is a fail too.
+ */
+ if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
+ raw_spin_unlock(&next->pi_lock);
+ continue;
+ }
+
/* Find suitable destination for @next, with force if needed. */
dest_cpu = select_fallback_rq(dead_rq->cpu, next);
- lockdep_unpin_lock(&rq->lock);
rq = __migrate_task(rq, next, dest_cpu);
if (rq != dead_rq) {
raw_spin_unlock(&rq->lock);
rq = dead_rq;
raw_spin_lock(&rq->lock);
}
+ raw_spin_unlock(&next->pi_lock);
}
rq->stop = stop;
* After ->on_cpu is cleared, the task can be moved to a different CPU.
* We must ensure this doesn't happen until the switch is completely
* finished.
+ *
+ * Pairs with the control dependency and rmb in try_to_wake_up().
*/
- smp_wmb();
- prev->on_cpu = 0;
+ smp_store_release(&prev->on_cpu, 0);
#endif
#ifdef CONFIG_DEBUG_SPINLOCK
/* this is a valid case when another task releases the spinlock */
}
EXPORT_SYMBOL_GPL(__wake_up_locked);
-void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, int nr,
- void *key)
+void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
{
- __wake_up_common(q, mode, nr, 0, key);
+ __wake_up_common(q, mode, 1, 0, key);
}
EXPORT_SYMBOL_GPL(__wake_up_locked_key);
if (!list_empty(&wait->task_list))
list_del_init(&wait->task_list);
else if (waitqueue_active(q))
- __wake_up_locked_key(q, mode, 1, key);
+ __wake_up_locked_key(q, mode, key);
spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(abort_exclusive_wait);
static int __clockevents_switch_state(struct clock_event_device *dev,
enum clock_event_state state)
{
- /* Transition with legacy set_mode() callback */
- if (dev->set_mode) {
- /* Legacy callback doesn't support new modes */
- if (state > CLOCK_EVT_STATE_ONESHOT)
- return -ENOSYS;
- /*
- * 'clock_event_state' and 'clock_event_mode' have 1-to-1
- * mapping until *_ONESHOT, and so a simple cast will work.
- */
- dev->set_mode((enum clock_event_mode)state, dev);
- dev->mode = (enum clock_event_mode)state;
- return 0;
- }
-
if (dev->features & CLOCK_EVT_FEAT_DUMMY)
return 0;
{
int ret = 0;
- if (dev->set_mode) {
- dev->set_mode(CLOCK_EVT_MODE_RESUME, dev);
- dev->mode = CLOCK_EVT_MODE_RESUME;
- } else if (dev->tick_resume) {
+ if (dev->tick_resume)
ret = dev->tick_resume(dev);
- }
return ret;
}
}
EXPORT_SYMBOL_GPL(clockevents_unbind_device);
-/* Sanity check of state transition callbacks */
-static int clockevents_sanity_check(struct clock_event_device *dev)
-{
- /* Legacy set_mode() callback */
- if (dev->set_mode) {
- /* We shouldn't be supporting new modes now */
- WARN_ON(dev->set_state_periodic || dev->set_state_oneshot ||
- dev->set_state_shutdown || dev->tick_resume ||
- dev->set_state_oneshot_stopped);
-
- BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
- return 0;
- }
-
- if (dev->features & CLOCK_EVT_FEAT_DUMMY)
- return 0;
-
- return 0;
-}
-
/**
* clockevents_register_device - register a clock event device
* @dev: device to register
{
unsigned long flags;
- BUG_ON(clockevents_sanity_check(dev));
-
/* Initialize state to DETACHED */
clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
continue;
/* Check the deviation from the watchdog clocksource. */
- if ((abs(cs_nsec - wd_nsec) > WATCHDOG_THRESHOLD)) {
+ if (abs64(cs_nsec - wd_nsec) > WATCHDOG_THRESHOLD) {
pr_warn("timekeeping watchdog: Marking clocksource '%s' as unstable because the skew is too large:\n",
cs->name);
pr_warn(" '%s' wd_now: %llx wd_last: %llx mask: %llx\n",
* the set mode function!
*/
clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
- dev->mode = CLOCK_EVT_MODE_UNUSED;
clockevents_exchange_device(dev, NULL);
dev->event_handler = clockevents_handle_noop;
td->evtdev = NULL;
__setup("nohz_full=", tick_nohz_full_setup);
static int tick_nohz_cpu_down_callback(struct notifier_block *nfb,
- unsigned long action,
- void *hcpu)
+ unsigned long action,
+ void *hcpu)
{
unsigned int cpu = (unsigned long)hcpu;
switch (action & ~CPU_TASKS_FROZEN) {
case CPU_DOWN_PREPARE:
/*
- * If we handle the timekeeping duty for full dynticks CPUs,
- * we can't safely shutdown that CPU.
+ * The boot CPU handles housekeeping duty (unbound timers,
+ * workqueues, timekeeping, ...) on behalf of full dynticks
+ * CPUs. It must remain online when nohz full is enabled.
*/
if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
return NOTIFY_BAD;
cpu_notifier(tick_nohz_cpu_down_callback, 0);
pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
cpumask_pr_args(tick_nohz_full_mask));
+
+ /*
+ * We need at least one CPU to handle housekeeping work such
+ * as timekeeping, unbound timers, workqueues, ...
+ */
+ WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
}
#endif
negative = (tick_error < 0);
/* Sort out the magnitude of the correction */
- tick_error = abs(tick_error);
+ tick_error = abs64(tick_error);
for (adj = 0; tick_error > interval; adj++)
tick_error >>= 1;
(unsigned long long) dev->min_delta_ns);
SEQ_printf(m, " mult: %u\n", dev->mult);
SEQ_printf(m, " shift: %u\n", dev->shift);
- SEQ_printf(m, " mode: %d\n", dev->mode);
+ SEQ_printf(m, " mode: %d\n", clockevent_get_state(dev));
SEQ_printf(m, " next_event: %Ld nsecs\n",
(unsigned long long) ktime_to_ns(dev->next_event));
print_name_offset(m, dev->set_next_event);
SEQ_printf(m, "\n");
- if (dev->set_mode) {
- SEQ_printf(m, " set_mode: ");
- print_name_offset(m, dev->set_mode);
+ if (dev->set_state_shutdown) {
+ SEQ_printf(m, " shutdown: ");
+ print_name_offset(m, dev->set_state_shutdown);
SEQ_printf(m, "\n");
- } else {
- if (dev->set_state_shutdown) {
- SEQ_printf(m, " shutdown: ");
- print_name_offset(m, dev->set_state_shutdown);
- SEQ_printf(m, "\n");
- }
+ }
- if (dev->set_state_periodic) {
- SEQ_printf(m, " periodic: ");
- print_name_offset(m, dev->set_state_periodic);
- SEQ_printf(m, "\n");
- }
+ if (dev->set_state_periodic) {
+ SEQ_printf(m, " periodic: ");
+ print_name_offset(m, dev->set_state_periodic);
+ SEQ_printf(m, "\n");
+ }
- if (dev->set_state_oneshot) {
- SEQ_printf(m, " oneshot: ");
- print_name_offset(m, dev->set_state_oneshot);
- SEQ_printf(m, "\n");
- }
+ if (dev->set_state_oneshot) {
+ SEQ_printf(m, " oneshot: ");
+ print_name_offset(m, dev->set_state_oneshot);
+ SEQ_printf(m, "\n");
+ }
- if (dev->set_state_oneshot_stopped) {
- SEQ_printf(m, " oneshot stopped: ");
- print_name_offset(m, dev->set_state_oneshot_stopped);
- SEQ_printf(m, "\n");
- }
+ if (dev->set_state_oneshot_stopped) {
+ SEQ_printf(m, " oneshot stopped: ");
+ print_name_offset(m, dev->set_state_oneshot_stopped);
+ SEQ_printf(m, "\n");
+ }
- if (dev->tick_resume) {
- SEQ_printf(m, " resume: ");
- print_name_offset(m, dev->tick_resume);
- SEQ_printf(m, "\n");
- }
+ if (dev->tick_resume) {
+ SEQ_printf(m, " resume: ");
+ print_name_offset(m, dev->tick_resume);
+ SEQ_printf(m, "\n");
}
SEQ_printf(m, " event_handler: ");
static inline bool need_flush(struct iommu_map_table *iommu)
{
- return (iommu->lazy_flush != NULL &&
- (iommu->flags & IOMMU_NEED_FLUSH) != 0);
+ return ((iommu->flags & IOMMU_NEED_FLUSH) != 0);
}
static inline void set_flush(struct iommu_map_table *iommu)
goto bail;
}
}
- if (n < pool->hint || need_flush(iommu)) {
+ if (iommu->lazy_flush &&
+ (n < pool->hint || need_flush(iommu))) {
clear_flush(iommu);
iommu->lazy_flush(iommu);
}
head = rht_dereference_bucket(new_tbl->buckets[new_hash],
new_tbl, new_hash);
- if (rht_is_a_nulls(head))
- INIT_RHT_NULLS_HEAD(entry->next, ht, new_hash);
- else
- RCU_INIT_POINTER(entry->next, head);
+ RCU_INIT_POINTER(entry->next, head);
rcu_assign_pointer(new_tbl->buckets[new_hash], entry);
spin_unlock(new_bucket_lock);
#include <linux/bug.h>
#include <linux/errno.h>
+#include <asm/byteorder.h>
+#include <asm/word-at-a-time.h>
+#include <asm/page.h>
+
#ifndef __HAVE_ARCH_STRNCASECMP
/**
* strncasecmp - Case insensitive, length-limited string comparison
EXPORT_SYMBOL(strlcpy);
#endif
+#ifndef __HAVE_ARCH_STRSCPY
+/**
+ * strscpy - Copy a C-string into a sized buffer
+ * @dest: Where to copy the string to
+ * @src: Where to copy the string from
+ * @count: Size of destination buffer
+ *
+ * Copy the string, or as much of it as fits, into the dest buffer.
+ * The routine returns the number of characters copied (not including
+ * the trailing NUL) or -E2BIG if the destination buffer wasn't big enough.
+ * The behavior is undefined if the string buffers overlap.
+ * The destination buffer is always NUL terminated, unless it's zero-sized.
+ *
+ * Preferred to strlcpy() since the API doesn't require reading memory
+ * from the src string beyond the specified "count" bytes, and since
+ * the return value is easier to error-check than strlcpy()'s.
+ * In addition, the implementation is robust to the string changing out
+ * from underneath it, unlike the current strlcpy() implementation.
+ *
+ * Preferred to strncpy() since it always returns a valid string, and
+ * doesn't unnecessarily force the tail of the destination buffer to be
+ * zeroed. If the zeroing is desired, it's likely cleaner to use strscpy()
+ * with an overflow test, then just memset() the tail of the dest buffer.
+ */
+ssize_t strscpy(char *dest, const char *src, size_t count)
+{
+ const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
+ size_t max = count;
+ long res = 0;
+
+ if (count == 0)
+ return -E2BIG;
+
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ /*
+ * If src is unaligned, don't cross a page boundary,
+ * since we don't know if the next page is mapped.
+ */
+ if ((long)src & (sizeof(long) - 1)) {
+ size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1));
+ if (limit < max)
+ max = limit;
+ }
+#else
+ /* If src or dest is unaligned, don't do word-at-a-time. */
+ if (((long) dest | (long) src) & (sizeof(long) - 1))
+ max = 0;
+#endif
+
+ while (max >= sizeof(unsigned long)) {
+ unsigned long c, data;
+
+ c = *(unsigned long *)(src+res);
+ if (has_zero(c, &data, &constants)) {
+ data = prep_zero_mask(c, data, &constants);
+ data = create_zero_mask(data);
+ *(unsigned long *)(dest+res) = c & zero_bytemask(data);
+ return res + find_zero(data);
+ }
+ *(unsigned long *)(dest+res) = c;
+ res += sizeof(unsigned long);
+ count -= sizeof(unsigned long);
+ max -= sizeof(unsigned long);
+ }
+
+ while (count) {
+ char c;
+
+ c = src[res];
+ dest[res] = c;
+ if (!c)
+ return res;
+ res++;
+ count--;
+ }
+
+ /* Hit buffer length without finding a NUL; force NUL-termination. */
+ if (res)
+ dest[res-1] = '\0';
+
+ return -E2BIG;
+}
+EXPORT_SYMBOL(strscpy);
+#endif
+
#ifndef __HAVE_ARCH_STRCAT
/**
* strcat - Append one %NUL-terminated string to another
}
exp = divisor[units] / (u32)blk_size;
- if (size >= exp) {
+ /*
+ * size must be strictly greater than exp here to ensure that remainder
+ * is greater than divisor[units] coming out of the if below.
+ */
+ if (size > exp) {
remainder = do_div(size, divisor[units]);
remainder *= blk_size;
i++;
list_for_each_entry(page, &pool->page_list, page_list) {
if (dma < page->dma)
continue;
- if (dma < (page->dma + pool->allocation))
+ if ((dma - page->dma) < pool->allocation)
return page;
}
return NULL;
iov_iter_count(i));
again:
+ /*
+ * Bring in the user page that we will copy from _first_.
+ * Otherwise there's a nasty deadlock on copying from the
+ * same page as we're writing to, without it being marked
+ * up-to-date.
+ *
+ * Not only is this an optimisation, but it is also required
+ * to check that the address is actually valid, when atomic
+ * usercopies are used, below.
+ */
+ if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
+ status = -EFAULT;
+ break;
+ }
+
status = a_ops->write_begin(file, mapping, pos, bytes, flags,
&page, &fsdata);
if (unlikely(status < 0))
if (mapping_writably_mapped(mapping))
flush_dcache_page(page);
- /*
- * 'page' is now locked. If we are trying to copy from a
- * mapping of 'page' in userspace, the copy might fault and
- * would need PageUptodate() to complete. But, page can not be
- * made Uptodate without acquiring the page lock, which we hold.
- * Deadlock. Avoid with pagefault_disable(). Fix up below with
- * iov_iter_fault_in_readable().
- */
- pagefault_disable();
+
copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
- pagefault_enable();
flush_dcache_page(page);
status = a_ops->write_end(file, mapping, pos, bytes, copied,
*/
bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
iov_iter_single_seg_count(i));
- /*
- * This is the fallback to recover if the copy from
- * userspace above faults.
- */
- if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
- status = -EFAULT;
- break;
- }
goto again;
}
pos += copied;
if (iter_vma == vma)
continue;
+ /*
+ * Shared VMAs have their own reserves and do not affect
+ * MAP_PRIVATE accounting but it is possible that a shared
+ * VMA is using the same page so check and skip such VMAs.
+ */
+ if (iter_vma->vm_flags & VM_MAYSHARE)
+ continue;
+
/*
* Unmap the page from other VMAs without their own reserves.
* They get marked to be SIGKILLed if they fault in these
if (unlikely(*shadow_addr)) {
u16 shadow_first_bytes = *(u16 *)shadow_addr;
- s8 last_byte = (addr + 15) & KASAN_SHADOW_MASK;
if (unlikely(shadow_first_bytes))
return true;
- if (likely(!last_byte))
+ if (likely(IS_ALIGNED(addr, 8)))
return false;
return memory_is_poisoned_1(addr + 15);
}
/*
+ * Return page count for single (non recursive) @memcg.
+ *
* Implementation Note: reading percpu statistics for memcg.
*
* Both of vmstat[] and percpu_counter has threshold and do periodic
* synchronization to implement "quick" read. There are trade-off between
* reading cost and precision of value. Then, we may have a chance to implement
- * a periodic synchronizion of counter in memcg's counter.
+ * a periodic synchronization of counter in memcg's counter.
*
* But this _read() function is used for user interface now. The user accounts
* memory usage by memory cgroup and he _always_ requires exact value because
*
* If there are kernel internal actions which can make use of some not-exact
* value, and reading all cpu value can be performance bottleneck in some
- * common workload, threashold and synchonization as vmstat[] should be
+ * common workload, threshold and synchronization as vmstat[] should be
* implemented.
*/
-static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
- enum mem_cgroup_stat_index idx)
+static unsigned long
+mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx)
{
long val = 0;
int cpu;
+ /* Per-cpu values can be negative, use a signed accumulator */
for_each_possible_cpu(cpu)
val += per_cpu(memcg->stat->count[idx], cpu);
+ /*
+ * Summing races with updates, so val may be negative. Avoid exposing
+ * transient negative values.
+ */
+ if (val < 0)
+ val = 0;
return val;
}
for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
continue;
- pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
+ pr_cont(" %s:%luKB", mem_cgroup_stat_names[i],
K(mem_cgroup_read_stat(iter, i)));
}
enum mem_cgroup_stat_index idx)
{
struct mem_cgroup *iter;
- long val = 0;
+ unsigned long val = 0;
- /* Per-cpu values can be negative, use a signed accumulator */
for_each_mem_cgroup_tree(iter, memcg)
val += mem_cgroup_read_stat(iter, idx);
- if (val < 0) /* race ? */
- val = 0;
return val;
}
for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
continue;
- seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
+ seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i],
mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
}
(u64)memsw * PAGE_SIZE);
for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
- long long val = 0;
+ unsigned long long val = 0;
if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
continue;
for_each_mem_cgroup_tree(mi, memcg)
val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
- seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
+ seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val);
}
for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
if (memcg_wb_domain_init(memcg, GFP_KERNEL))
goto out_free_stat;
- spin_lock_init(&memcg->pcp_counter_lock);
return memcg;
out_free_stat:
if (PageSwapBacked(page))
SetPageSwapBacked(newpage);
+ /*
+ * Indirectly called below, migrate_page_copy() copies PG_dirty and thus
+ * needs newpage's memcg set to transfer memcg dirty page accounting.
+ * So perform memcg migration in two steps:
+ * 1. set newpage->mem_cgroup (here)
+ * 2. clear page->mem_cgroup (below)
+ */
+ set_page_memcg(newpage, page_memcg(page));
+
mapping = page_mapping(page);
if (!mapping)
rc = migrate_page(mapping, newpage, page, mode);
rc = fallback_migrate_page(mapping, newpage, page, mode);
if (rc != MIGRATEPAGE_SUCCESS) {
+ set_page_memcg(newpage, NULL);
newpage->mapping = NULL;
} else {
- mem_cgroup_migrate(page, newpage, false);
+ set_page_memcg(page, NULL);
if (page_was_mapped)
remove_migration_ptes(page, newpage);
page->mapping = NULL;
if (rc != MIGRATEPAGE_SUCCESS && put_new_page)
put_new_page(new_hpage, private);
else
- put_page(new_hpage);
+ putback_active_hugepage(new_hpage);
if (result) {
if (rc)
void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
struct rb_node **rb_link, struct rb_node *rb_parent)
{
- WARN_ONCE(vma->vm_file && !vma->vm_ops, "missing vma->vm_ops");
-
/* Update tracking information for the gap following the new vma. */
if (vma->vm_next)
vma_gap_update(vma->vm_next);
int vma_wants_writenotify(struct vm_area_struct *vma)
{
vm_flags_t vm_flags = vma->vm_flags;
+ const struct vm_operations_struct *vm_ops = vma->vm_ops;
/* If it was private or non-writable, the write bit is already clear */
if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
return 0;
/* The backer wishes to know when pages are first written to? */
- if (vma->vm_ops && vma->vm_ops->page_mkwrite)
+ if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite))
return 1;
/* The open routine did something to the protections that pgprot_modify
*/
WARN_ON_ONCE(addr != vma->vm_start);
- /* All file mapping must have ->vm_ops set */
- if (!vma->vm_ops) {
- static const struct vm_operations_struct dummy_ops = {};
- vma->vm_ops = &dummy_ops;
- }
-
addr = vma->vm_start;
vm_flags = vma->vm_flags;
} else if (vm_flags & VM_SHARED) {
size += BYTES_PER_WORD;
}
#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
- if (size >= kmalloc_size(INDEX_NODE + 1)
- && cachep->object_size > cache_line_size()
- && ALIGN(size, cachep->align) < PAGE_SIZE) {
+ /*
+ * To activate debug pagealloc, off-slab management is necessary
+ * requirement. In early phase of initialization, small sized slab
+ * doesn't get initialized so it would not be possible. So, we need
+ * to check size >= 256. It guarantees that all necessary small
+ * sized slab is initialized in current slab initialization sequence.
+ */
+ if (!slab_early_init && size >= kmalloc_size(INDEX_NODE) &&
+ size >= 256 && cachep->object_size > cache_line_size() &&
+ ALIGN(size, cachep->align) < PAGE_SIZE) {
cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align);
size = PAGE_SIZE;
}
if (!memcg)
return true;
#ifdef CONFIG_CGROUP_WRITEBACK
- if (memcg->css.cgroup)
+ if (cgroup_on_dfl(memcg->css.cgroup))
return true;
#endif
return false;
static int clip_encap(struct atm_vcc *vcc, int mode)
{
+ if (!CLIP_VCC(vcc))
+ return -EBADFD;
+
CLIP_VCC(vcc)->encap = mode;
return 0;
}
if (!conn)
return 1;
- chan = conn->smp;
- if (!chan) {
- BT_ERR("SMP security requested but not available");
- return 1;
- }
-
if (!hci_dev_test_flag(hcon->hdev, HCI_LE_ENABLED))
return 1;
if (smp_ltk_encrypt(conn, hcon->pending_sec_level))
return 0;
+ chan = conn->smp;
+ if (!chan) {
+ BT_ERR("SMP security requested but not available");
+ return 1;
+ }
+
l2cap_chan_lock(chan);
/* If SMP is already in progress ignore this request */
ih = igmpv3_report_hdr(skb);
num = ntohs(ih->ngrec);
- len = sizeof(*ih);
+ len = skb_transport_offset(skb) + sizeof(*ih);
for (i = 0; i < num; i++) {
len += sizeof(*grec);
icmp6h = icmp6_hdr(skb);
num = ntohs(icmp6h->icmp6_dataun.un_data16[1]);
- len = sizeof(*icmp6h);
+ len = skb_transport_offset(skb) + sizeof(*icmp6h);
for (i = 0; i < num; i++) {
__be16 *nsrcs, _nsrcs;
dout("prepare_write_keepalive %p\n", con);
con_out_kvec_reset(con);
if (con->peer_features & CEPH_FEATURE_MSGR_KEEPALIVE2) {
- struct timespec ts = CURRENT_TIME;
- struct ceph_timespec ceph_ts;
- ceph_encode_timespec(&ceph_ts, &ts);
+ struct timespec now = CURRENT_TIME;
+
con_out_kvec_add(con, sizeof(tag_keepalive2), &tag_keepalive2);
- con_out_kvec_add(con, sizeof(ceph_ts), &ceph_ts);
+ ceph_encode_timespec(&con->out_temp_keepalive2, &now);
+ con_out_kvec_add(con, sizeof(con->out_temp_keepalive2),
+ &con->out_temp_keepalive2);
} else {
con_out_kvec_add(con, sizeof(tag_keepalive), &tag_keepalive);
}
while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
msleep(1);
+ while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
+ msleep(1);
hrtimer_cancel(&n->timer);
{
int idx = 0;
struct fib_rule *rule;
+ int err = 0;
rcu_read_lock();
list_for_each_entry_rcu(rule, &ops->rules_list, list) {
if (idx < cb->args[1])
goto skip;
- if (fib_nl_fill_rule(skb, rule, NETLINK_CB(cb->skb).portid,
- cb->nlh->nlmsg_seq, RTM_NEWRULE,
- NLM_F_MULTI, ops) < 0)
+ err = fib_nl_fill_rule(skb, rule, NETLINK_CB(cb->skb).portid,
+ cb->nlh->nlmsg_seq, RTM_NEWRULE,
+ NLM_F_MULTI, ops);
+ if (err)
break;
skip:
idx++;
cb->args[1] = idx;
rules_ops_put(ops);
- return skb->len;
+ return err;
}
static int fib_nl_dumprule(struct sk_buff *skb, struct netlink_callback *cb)
if (ops == NULL)
return -EAFNOSUPPORT;
- return dump_rules(skb, cb, ops);
+ dump_rules(skb, cb, ops);
+
+ return skb->len;
}
rcu_read_lock();
bpf_src = BPF_X;
} else {
insn->dst_reg = BPF_REG_A;
- insn->src_reg = BPF_REG_X;
insn->imm = fp->k;
bpf_src = BPF_SRC(fp->code);
+ insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0;
}
/* Common case where 'jump_false' is next insn. */
static const char fmt_hex[] = "%#x\n";
static const char fmt_long_hex[] = "%#lx\n";
static const char fmt_dec[] = "%d\n";
-static const char fmt_udec[] = "%u\n";
static const char fmt_ulong[] = "%lu\n";
static const char fmt_u64[] = "%llu\n";
if (netif_running(netdev)) {
struct ethtool_cmd cmd;
if (!__ethtool_get_settings(netdev, &cmd))
- ret = sprintf(buf, fmt_udec, ethtool_cmd_speed(&cmd));
+ ret = sprintf(buf, fmt_dec, ethtool_cmd_speed(&cmd));
}
rtnl_unlock();
return ret;
return ret == 0 ? dev->of_node == data : ret;
}
+/*
+ * of_find_net_device_by_node - lookup the net device for the device node
+ * @np: OF device node
+ *
+ * Looks up the net_device structure corresponding with the device node.
+ * If successful, returns a pointer to the net_device with the embedded
+ * struct device refcount incremented by one, or NULL on failure. The
+ * refcount must be dropped when done with the net_device.
+ */
struct net_device *of_find_net_device_by_node(struct device_node *np)
{
struct device *dev;
*/
static int poll_one_napi(struct napi_struct *napi, int budget)
{
- int work;
+ int work = 0;
/* net_rx_action's ->poll() invocations and our's are
* synchronized by this test which is only made while
if (!test_bit(NAPI_STATE_SCHED, &napi->state))
return budget;
- set_bit(NAPI_STATE_NPSVC, &napi->state);
+ /* If we set this bit but see that it has already been set,
+ * that indicates that napi has been disabled and we need
+ * to abort this operation
+ */
+ if (test_and_set_bit(NAPI_STATE_NPSVC, &napi->state))
+ goto out;
work = napi->poll(napi, budget);
WARN_ONCE(work > budget, "%pF exceeded budget in poll\n", napi->poll);
clear_bit(NAPI_STATE_NPSVC, &napi->state);
+out:
return budget - work;
}
u32 portid = NETLINK_CB(cb->skb).portid;
u32 seq = cb->nlh->nlmsg_seq;
u32 filter_mask = 0;
+ int err;
if (nlmsg_len(cb->nlh) > sizeof(struct ifinfomsg)) {
struct nlattr *extfilt;
struct net_device *br_dev = netdev_master_upper_dev_get(dev);
if (br_dev && br_dev->netdev_ops->ndo_bridge_getlink) {
- if (idx >= cb->args[0] &&
- br_dev->netdev_ops->ndo_bridge_getlink(
- skb, portid, seq, dev, filter_mask,
- NLM_F_MULTI) < 0)
- break;
+ if (idx >= cb->args[0]) {
+ err = br_dev->netdev_ops->ndo_bridge_getlink(
+ skb, portid, seq, dev,
+ filter_mask, NLM_F_MULTI);
+ if (err < 0 && err != -EOPNOTSUPP)
+ break;
+ }
idx++;
}
if (ops->ndo_bridge_getlink) {
- if (idx >= cb->args[0] &&
- ops->ndo_bridge_getlink(skb, portid, seq, dev,
- filter_mask,
- NLM_F_MULTI) < 0)
- break;
+ if (idx >= cb->args[0]) {
+ err = ops->ndo_bridge_getlink(skb, portid,
+ seq, dev,
+ filter_mask,
+ NLM_F_MULTI);
+ if (err < 0 && err != -EOPNOTSUPP)
+ break;
+ }
idx++;
}
}
*/
unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
{
+ unsigned char *data = skb->data;
+
BUG_ON(len > skb->len);
- skb->len -= len;
- BUG_ON(skb->len < skb->data_len);
- skb_postpull_rcsum(skb, skb->data, len);
- return skb->data += len;
+ __skb_pull(skb, len);
+ skb_postpull_rcsum(skb, data, len);
+ return skb->data;
}
EXPORT_SYMBOL_GPL(skb_pull_rcsum);
return;
kfree(rsk_prot->slab_name);
rsk_prot->slab_name = NULL;
- if (rsk_prot->slab) {
- kmem_cache_destroy(rsk_prot->slab);
- rsk_prot->slab = NULL;
- }
+ kmem_cache_destroy(rsk_prot->slab);
+ rsk_prot->slab = NULL;
}
static int req_prot_init(const struct proto *prot)
list_del(&prot->node);
mutex_unlock(&proto_list_mutex);
- if (prot->slab != NULL) {
- kmem_cache_destroy(prot->slab);
- prot->slab = NULL;
- }
+ kmem_cache_destroy(prot->slab);
+ prot->slab = NULL;
req_prot_cleanup(prot->rsk_prot);
void dccp_ackvec_exit(void)
{
- if (dccp_ackvec_slab != NULL) {
- kmem_cache_destroy(dccp_ackvec_slab);
- dccp_ackvec_slab = NULL;
- }
- if (dccp_ackvec_record_slab != NULL) {
- kmem_cache_destroy(dccp_ackvec_record_slab);
- dccp_ackvec_record_slab = NULL;
- }
+ kmem_cache_destroy(dccp_ackvec_slab);
+ dccp_ackvec_slab = NULL;
+ kmem_cache_destroy(dccp_ackvec_record_slab);
+ dccp_ackvec_record_slab = NULL;
}
static void ccid_kmem_cache_destroy(struct kmem_cache *slab)
{
- if (slab != NULL)
- kmem_cache_destroy(slab);
+ kmem_cache_destroy(slab);
}
static int __init ccid_activate(struct ccid_operations *ccid_ops)
tw->tw_ipv6only = sk->sk_ipv6only;
}
#endif
- /* Linkage updates. */
- __inet_twsk_hashdance(tw, sk, &dccp_hashinfo);
/* Get the TIME_WAIT timeout firing. */
if (timeo < rto)
timeo = DCCP_TIMEWAIT_LEN;
inet_twsk_schedule(tw, timeo);
+ /* Linkage updates. */
+ __inet_twsk_hashdance(tw, sk, &dccp_hashinfo);
inet_twsk_put(tw);
} else {
/* Sorry, if we're out of memory, just CLOSE this
port_index++;
}
kfree(pd->chip[i].rtable);
+
+ /* Drop our reference to the MDIO bus device */
+ if (pd->chip[i].host_dev)
+ put_device(pd->chip[i].host_dev);
}
kfree(pd->chip);
}
return -EPROBE_DEFER;
ethernet = of_parse_phandle(np, "dsa,ethernet", 0);
- if (!ethernet)
- return -EINVAL;
+ if (!ethernet) {
+ ret = -EINVAL;
+ goto out_put_mdio;
+ }
ethernet_dev = of_find_net_device_by_node(ethernet);
- if (!ethernet_dev)
- return -EPROBE_DEFER;
+ if (!ethernet_dev) {
+ ret = -EPROBE_DEFER;
+ goto out_put_mdio;
+ }
pd = kzalloc(sizeof(*pd), GFP_KERNEL);
- if (!pd)
- return -ENOMEM;
+ if (!pd) {
+ ret = -ENOMEM;
+ goto out_put_ethernet;
+ }
dev->platform_data = pd;
pd->of_netdev = ethernet_dev;
cd = &pd->chip[chip_index];
cd->of_node = child;
- cd->host_dev = &mdio_bus->dev;
+
+ /* When assigning the host device, increment its refcount */
+ cd->host_dev = get_device(&mdio_bus->dev);
sw_addr = of_get_property(child, "reg", NULL);
if (!sw_addr)
ret = -EPROBE_DEFER;
goto out_free_chip;
}
+
+ /* Drop the mdio_bus device ref, replacing the host
+ * device with the mdio_bus_switch device, keeping
+ * the refcount from of_mdio_find_bus() above.
+ */
+ put_device(cd->host_dev);
cd->host_dev = &mdio_bus_switch->dev;
}
}
}
+ /* The individual chips hold their own refcount on the mdio bus,
+ * so drop ours */
+ put_device(&mdio_bus->dev);
+
return 0;
out_free_chip:
out_free:
kfree(pd);
dev->platform_data = NULL;
+out_put_ethernet:
+ put_device(ðernet_dev->dev);
+out_put_mdio:
+ put_device(&mdio_bus->dev);
return ret;
}
return;
dsa_of_free_platform_data(pd);
+ put_device(&pd->of_netdev->dev);
kfree(pd);
}
#else
static int dsa_slave_port_attr_set(struct net_device *dev,
struct switchdev_attr *attr)
{
- int ret = 0;
+ struct dsa_slave_priv *p = netdev_priv(dev);
+ struct dsa_switch *ds = p->parent;
+ int ret;
switch (attr->id) {
case SWITCHDEV_ATTR_PORT_STP_STATE:
- if (attr->trans == SWITCHDEV_TRANS_COMMIT)
- ret = dsa_slave_stp_update(dev, attr->u.stp_state);
+ if (attr->trans == SWITCHDEV_TRANS_PREPARE)
+ ret = ds->drv->port_stp_update ? 0 : -EOPNOTSUPP;
+ else
+ ret = ds->drv->port_stp_update(ds, p->port,
+ attr->u.stp_state);
break;
default:
ret = -EOPNOTSUPP;
trailer = skb_tail_pointer(skb) - 4;
if (trailer[0] != 0x80 || (trailer[1] & 0xf8) != 0x00 ||
- (trailer[3] & 0xef) != 0x00 || trailer[3] != 0x00)
+ (trailer[2] & 0xef) != 0x00 || trailer[3] != 0x00)
goto out_drop;
source_port = trailer[1] & 7;
#include <net/arp.h>
#include <net/ax25.h>
#include <net/netrom.h>
+#include <net/dst_metadata.h>
+#include <net/ip_tunnels.h>
#include <linux/uaccess.h>
struct net_device *dev, __be32 src_ip,
const unsigned char *dest_hw,
const unsigned char *src_hw,
- const unsigned char *target_hw, struct sk_buff *oskb)
+ const unsigned char *target_hw,
+ struct dst_entry *dst)
{
struct sk_buff *skb;
if (!skb)
return;
- if (oskb)
- skb_dst_copy(skb, oskb);
-
+ skb_dst_set(skb, dst);
arp_xmit(skb);
}
__be32 target = *(__be32 *)neigh->primary_key;
int probes = atomic_read(&neigh->probes);
struct in_device *in_dev;
+ struct dst_entry *dst = NULL;
rcu_read_lock();
in_dev = __in_dev_get_rcu(dev);
}
}
+ if (skb && !(dev->priv_flags & IFF_XMIT_DST_RELEASE))
+ dst = dst_clone(skb_dst(skb));
arp_send_dst(ARPOP_REQUEST, ETH_P_ARP, target, dev, saddr,
- dst_hw, dev->dev_addr, NULL,
- dev->priv_flags & IFF_XMIT_DST_RELEASE ? NULL : skb);
+ dst_hw, dev->dev_addr, NULL, dst);
}
static int arp_ignore(struct in_device *in_dev, __be32 sip, __be32 tip)
int addr_type;
struct neighbour *n;
struct net *net = dev_net(dev);
+ struct dst_entry *reply_dst = NULL;
bool is_garp = false;
/* arp_rcv below verifies the ARP header and verifies the device
* cache.
*/
+ if (arp->ar_op == htons(ARPOP_REQUEST) && skb_metadata_dst(skb))
+ reply_dst = (struct dst_entry *)
+ iptunnel_metadata_reply(skb_metadata_dst(skb),
+ GFP_ATOMIC);
+
/* Special case: IPv4 duplicate address detection packet (RFC2131) */
if (sip == 0) {
if (arp->ar_op == htons(ARPOP_REQUEST) &&
inet_addr_type_dev_table(net, dev, tip) == RTN_LOCAL &&
!arp_ignore(in_dev, sip, tip))
- arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
- dev->dev_addr, sha);
+ arp_send_dst(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip,
+ sha, dev->dev_addr, sha, reply_dst);
goto out;
}
if (!dont_send) {
n = neigh_event_ns(&arp_tbl, sha, &sip, dev);
if (n) {
- arp_send(ARPOP_REPLY, ETH_P_ARP, sip,
- dev, tip, sha, dev->dev_addr,
- sha);
+ arp_send_dst(ARPOP_REPLY, ETH_P_ARP,
+ sip, dev, tip, sha,
+ dev->dev_addr, sha,
+ reply_dst);
neigh_release(n);
}
}
if (NEIGH_CB(skb)->flags & LOCALLY_ENQUEUED ||
skb->pkt_type == PACKET_HOST ||
NEIGH_VAR(in_dev->arp_parms, PROXY_DELAY) == 0) {
- arp_send(ARPOP_REPLY, ETH_P_ARP, sip,
- dev, tip, sha, dev->dev_addr,
- sha);
+ arp_send_dst(ARPOP_REPLY, ETH_P_ARP,
+ sip, dev, tip, sha,
+ dev->dev_addr, sha,
+ reply_dst);
} else {
pneigh_enqueue(&arp_tbl,
in_dev->arp_parms, skb);
fl4.flowi4_tos = tos;
fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
fl4.flowi4_tun_key.tun_id = 0;
+ fl4.flowi4_flags = 0;
no_addr = idev->ifa_list == NULL;
nh->nh_flags & RTNH_F_LINKDOWN &&
!(fib_flags & FIB_LOOKUP_IGNORE_LINKSTATE))
continue;
- if (!(flp->flowi4_flags & FLOWI_FLAG_VRFSRC)) {
+ if (!(flp->flowi4_flags & FLOWI_FLAG_SKIP_NH_OIF)) {
if (flp->flowi4_oif &&
flp->flowi4_oif != nh->nh_oif)
continue;
fl4.flowi4_mark = mark;
fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos);
fl4.flowi4_proto = IPPROTO_ICMP;
- fl4.flowi4_oif = vrf_master_ifindex(skb->dev) ? : skb->dev->ifindex;
+ fl4.flowi4_oif = vrf_master_ifindex(skb->dev);
security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
rt = ip_route_output_key(net, &fl4);
if (IS_ERR(rt))
fl4->flowi4_proto = IPPROTO_ICMP;
fl4->fl4_icmp_type = type;
fl4->fl4_icmp_code = code;
- fl4->flowi4_oif = vrf_master_ifindex(skb_in->dev) ? : skb_in->dev->ifindex;
+ fl4->flowi4_oif = vrf_master_ifindex(skb_in->dev);
security_skb_classify_flow(skb_in, flowi4_to_flowi(fl4));
rt = __ip_route_output_key(net, fl4);
req->num_timeout = 0;
req->sk = NULL;
+ setup_timer(&req->rsk_timer, reqsk_timer_handler, (unsigned long)req);
+ mod_timer_pinned(&req->rsk_timer, jiffies + timeout);
+ req->rsk_hash = hash;
+
/* before letting lookups find us, make sure all req fields
* are committed to memory and refcnt initialized.
*/
smp_wmb();
atomic_set(&req->rsk_refcnt, 2);
- setup_timer(&req->rsk_timer, reqsk_timer_handler, (unsigned long)req);
- req->rsk_hash = hash;
spin_lock(&queue->syn_wait_lock);
req->dl_next = lopt->syn_table[hash];
lopt->syn_table[hash] = req;
spin_unlock(&queue->syn_wait_lock);
-
- mod_timer_pinned(&req->rsk_timer, jiffies + timeout);
}
EXPORT_SYMBOL(reqsk_queue_hash_req);
/*
* Step 2: Hash TW into tcp ehash chain.
* Notes :
- * - tw_refcnt is set to 3 because :
+ * - tw_refcnt is set to 4 because :
* - We have one reference from bhash chain.
* - We have one reference from ehash chain.
+ * - We have one reference from timer.
+ * - One reference for ourself (our caller will release it).
* We can use atomic_set() because prior spin_lock()/spin_unlock()
* committed into memory all tw fields.
*/
- atomic_set(&tw->tw_refcnt, 1 + 1 + 1);
+ atomic_set(&tw->tw_refcnt, 4);
inet_twsk_add_node_rcu(tw, &ehead->chain);
/* Step 3: Remove SK from hash chain */
}
EXPORT_SYMBOL(inet_twsk_deschedule_put);
-void inet_twsk_schedule(struct inet_timewait_sock *tw, const int timeo)
+void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
{
/* timeout := RTO * 3.5
*
*/
tw->tw_kill = timeo <= 4*HZ;
- if (!mod_timer_pinned(&tw->tw_timer, jiffies + timeo)) {
- atomic_inc(&tw->tw_refcnt);
+ if (!rearm) {
+ BUG_ON(mod_timer_pinned(&tw->tw_timer, jiffies + timeo));
atomic_inc(&tw->tw_dr->tw_count);
+ } else {
+ mod_timer_pending(&tw->tw_timer, jiffies + timeo);
}
}
-EXPORT_SYMBOL_GPL(inet_twsk_schedule);
+EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
void inet_twsk_purge(struct inet_hashinfo *hashinfo,
struct inet_timewait_death_row *twdr, int family)
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/rtnetlink.h>
+#include <net/dst_metadata.h>
int iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
__be32 src, __be32 dst, __u8 proto,
__u8 tos, __u8 ttl, __be16 df, bool xnet)
{
- int pkt_len = skb->len;
+ int pkt_len = skb->len - skb_inner_network_offset(skb);
struct iphdr *iph;
int err;
}
EXPORT_SYMBOL_GPL(iptunnel_pull_header);
+struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md,
+ gfp_t flags)
+{
+ struct metadata_dst *res;
+ struct ip_tunnel_info *dst, *src;
+
+ if (!md || md->u.tun_info.mode & IP_TUNNEL_INFO_TX)
+ return NULL;
+
+ res = metadata_dst_alloc(0, flags);
+ if (!res)
+ return NULL;
+
+ dst = &res->u.tun_info;
+ src = &md->u.tun_info;
+ dst->key.tun_id = src->key.tun_id;
+ if (src->mode & IP_TUNNEL_INFO_IPV6)
+ memcpy(&dst->key.u.ipv6.dst, &src->key.u.ipv6.src,
+ sizeof(struct in6_addr));
+ else
+ dst->key.u.ipv4.dst = src->key.u.ipv4.src;
+ dst->mode = src->mode | IP_TUNNEL_INFO_TX;
+
+ return res;
+}
+EXPORT_SYMBOL_GPL(iptunnel_metadata_reply);
+
struct sk_buff *iptunnel_handle_offloads(struct sk_buff *skb,
bool csum_help,
int gso_type_mask)
[LWTUNNEL_IP_SRC] = { .type = NLA_U32 },
[LWTUNNEL_IP_TTL] = { .type = NLA_U8 },
[LWTUNNEL_IP_TOS] = { .type = NLA_U8 },
- [LWTUNNEL_IP_SPORT] = { .type = NLA_U16 },
- [LWTUNNEL_IP_DPORT] = { .type = NLA_U16 },
[LWTUNNEL_IP_FLAGS] = { .type = NLA_U16 },
};
if (tb[LWTUNNEL_IP_TOS])
tun_info->key.tos = nla_get_u8(tb[LWTUNNEL_IP_TOS]);
- if (tb[LWTUNNEL_IP_SPORT])
- tun_info->key.tp_src = nla_get_be16(tb[LWTUNNEL_IP_SPORT]);
-
- if (tb[LWTUNNEL_IP_DPORT])
- tun_info->key.tp_dst = nla_get_be16(tb[LWTUNNEL_IP_DPORT]);
-
if (tb[LWTUNNEL_IP_FLAGS])
tun_info->key.tun_flags = nla_get_u16(tb[LWTUNNEL_IP_FLAGS]);
nla_put_be32(skb, LWTUNNEL_IP_SRC, tun_info->key.u.ipv4.src) ||
nla_put_u8(skb, LWTUNNEL_IP_TOS, tun_info->key.tos) ||
nla_put_u8(skb, LWTUNNEL_IP_TTL, tun_info->key.ttl) ||
- nla_put_u16(skb, LWTUNNEL_IP_SPORT, tun_info->key.tp_src) ||
- nla_put_u16(skb, LWTUNNEL_IP_DPORT, tun_info->key.tp_dst) ||
nla_put_u16(skb, LWTUNNEL_IP_FLAGS, tun_info->key.tun_flags))
return -ENOMEM;
+ nla_total_size(4) /* LWTUNNEL_IP_SRC */
+ nla_total_size(1) /* LWTUNNEL_IP_TOS */
+ nla_total_size(1) /* LWTUNNEL_IP_TTL */
- + nla_total_size(2) /* LWTUNNEL_IP_SPORT */
- + nla_total_size(2) /* LWTUNNEL_IP_DPORT */
+ nla_total_size(2); /* LWTUNNEL_IP_FLAGS */
}
[LWTUNNEL_IP6_SRC] = { .len = sizeof(struct in6_addr) },
[LWTUNNEL_IP6_HOPLIMIT] = { .type = NLA_U8 },
[LWTUNNEL_IP6_TC] = { .type = NLA_U8 },
- [LWTUNNEL_IP6_SPORT] = { .type = NLA_U16 },
- [LWTUNNEL_IP6_DPORT] = { .type = NLA_U16 },
[LWTUNNEL_IP6_FLAGS] = { .type = NLA_U16 },
};
if (tb[LWTUNNEL_IP6_TC])
tun_info->key.tos = nla_get_u8(tb[LWTUNNEL_IP6_TC]);
- if (tb[LWTUNNEL_IP6_SPORT])
- tun_info->key.tp_src = nla_get_be16(tb[LWTUNNEL_IP6_SPORT]);
-
- if (tb[LWTUNNEL_IP6_DPORT])
- tun_info->key.tp_dst = nla_get_be16(tb[LWTUNNEL_IP6_DPORT]);
-
if (tb[LWTUNNEL_IP6_FLAGS])
tun_info->key.tun_flags = nla_get_u16(tb[LWTUNNEL_IP6_FLAGS]);
nla_put_in6_addr(skb, LWTUNNEL_IP6_SRC, &tun_info->key.u.ipv6.src) ||
nla_put_u8(skb, LWTUNNEL_IP6_HOPLIMIT, tun_info->key.tos) ||
nla_put_u8(skb, LWTUNNEL_IP6_TC, tun_info->key.ttl) ||
- nla_put_u16(skb, LWTUNNEL_IP6_SPORT, tun_info->key.tp_src) ||
- nla_put_u16(skb, LWTUNNEL_IP6_DPORT, tun_info->key.tp_dst) ||
nla_put_u16(skb, LWTUNNEL_IP6_FLAGS, tun_info->key.tun_flags))
return -ENOMEM;
+ nla_total_size(16) /* LWTUNNEL_IP6_SRC */
+ nla_total_size(1) /* LWTUNNEL_IP6_HOPLIMIT */
+ nla_total_size(1) /* LWTUNNEL_IP6_TC */
- + nla_total_size(2) /* LWTUNNEL_IP6_SPORT */
- + nla_total_size(2) /* LWTUNNEL_IP6_DPORT */
+ nla_total_size(2); /* LWTUNNEL_IP6_FLAGS */
}
fl4.flowi4_mark = skb->mark;
fl4.flowi4_tos = tos;
fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
+ fl4.flowi4_flags = 0;
fl4.daddr = daddr;
fl4.saddr = saddr;
err = fib_lookup(net, &fl4, &res, 0);
struct fib_result res;
struct rtable *rth;
int orig_oif;
+ int err = -ENETUNREACH;
res.tclassid = 0;
res.fi = NULL;
goto make_route;
}
- if (fib_lookup(net, fl4, &res, 0)) {
+ err = fib_lookup(net, fl4, &res, 0);
+ if (err) {
res.fi = NULL;
res.table = NULL;
if (fl4->flowi4_oif) {
res.type = RTN_UNICAST;
goto make_route;
}
- rth = ERR_PTR(-ENETUNREACH);
+ rth = ERR_PTR(err);
goto out;
}
static void bictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event)
{
if (event == CA_EVENT_TX_START) {
- s32 delta = tcp_time_stamp - tcp_sk(sk)->lsndtime;
struct bictcp *ca = inet_csk_ca(sk);
+ u32 now = tcp_time_stamp;
+ s32 delta;
+
+ delta = now - tcp_sk(sk)->lsndtime;
/* We were application limited (idle) for a while.
* Shift epoch_start to keep cwnd growth to cubic curve.
*/
- if (ca->epoch_start && delta > 0)
+ if (ca->epoch_start && delta > 0) {
ca->epoch_start += delta;
+ if (after(ca->epoch_start, now))
+ ca->epoch_start = now;
+ }
return;
}
}
if (tcp_death_row.sysctl_tw_recycle &&
tcptw->tw_ts_recent_stamp &&
tcp_tw_remember_stamp(tw))
- inet_twsk_schedule(tw, tw->tw_timeout);
+ inet_twsk_reschedule(tw, tw->tw_timeout);
else
- inet_twsk_schedule(tw, TCP_TIMEWAIT_LEN);
+ inet_twsk_reschedule(tw, TCP_TIMEWAIT_LEN);
return TCP_TW_ACK;
}
return TCP_TW_SUCCESS;
}
}
- inet_twsk_schedule(tw, TCP_TIMEWAIT_LEN);
+ inet_twsk_reschedule(tw, TCP_TIMEWAIT_LEN);
if (tmp_opt.saw_tstamp) {
tcptw->tw_ts_recent = tmp_opt.rcv_tsval;
* Do not reschedule in the last case.
*/
if (paws_reject || th->ack)
- inet_twsk_schedule(tw, TCP_TIMEWAIT_LEN);
+ inet_twsk_reschedule(tw, TCP_TIMEWAIT_LEN);
return tcp_timewait_check_oow_rate_limit(
tw, skb, LINUX_MIB_TCPACKSKIPPEDTIMEWAIT);
} while (0);
#endif
- /* Linkage updates. */
- __inet_twsk_hashdance(tw, sk, &tcp_hashinfo);
-
/* Get the TIME_WAIT timeout firing. */
if (timeo < rto)
timeo = rto;
}
inet_twsk_schedule(tw, timeo);
+ /* Linkage updates. */
+ __inet_twsk_hashdance(tw, sk, &tcp_hashinfo);
inet_twsk_put(tw);
} else {
/* Sorry, if we're out of memory, just CLOSE this
skb_reserve(skb, MAX_TCP_HEADER);
tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
TCPHDR_ACK | TCPHDR_RST);
+ skb_mstamp_get(&skb->skb_mstamp);
/* Send it off. */
if (tcp_transmit_skb(sk, skb, 0, priority))
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
if (netif_index_is_vrf(net, ipc.oif)) {
flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
RT_SCOPE_UNIVERSE, sk->sk_protocol,
- (flow_flags | FLOWI_FLAG_VRFSRC),
+ (flow_flags | FLOWI_FLAG_VRFSRC |
+ FLOWI_FLAG_SKIP_NH_OIF),
faddr, saddr, dport,
inet->inet_sport);
if (saddr)
fl4->saddr = saddr->a4;
+ fl4->flowi4_flags = FLOWI_FLAG_SKIP_NH_OIF;
+
rt = __ip_route_output_key(net, fl4);
if (!IS_ERR(rt))
return &rt->dst;
rt = addrconf_get_prefix_route(&ifp->peer_addr, 128,
ifp->idev->dev, 0, 0);
- if (rt && ip6_del_rt(rt))
- dst_free(&rt->dst);
+ if (rt)
+ ip6_del_rt(rt);
}
dst_hold(&ifp->rt->dst);
- if (ip6_del_rt(ifp->rt))
- dst_free(&ifp->rt->dst);
+ ip6_del_rt(ifp->rt);
rt_genid_bump_ipv6(net);
break;
kmem_cache_free(fib6_node_kmem, fn);
}
+static void rt6_rcu_free(struct rt6_info *rt)
+{
+ call_rcu(&rt->dst.rcu_head, dst_rcu_free);
+}
+
static void rt6_free_pcpu(struct rt6_info *non_pcpu_rt)
{
int cpu;
ppcpu_rt = per_cpu_ptr(non_pcpu_rt->rt6i_pcpu, cpu);
pcpu_rt = *ppcpu_rt;
if (pcpu_rt) {
- dst_free(&pcpu_rt->dst);
+ rt6_rcu_free(pcpu_rt);
*ppcpu_rt = NULL;
}
}
{
if (atomic_dec_and_test(&rt->rt6i_ref)) {
rt6_free_pcpu(rt);
- dst_free(&rt->dst);
+ rt6_rcu_free(rt);
}
}
*ins = rt;
rt->rt6i_node = fn;
atomic_inc(&rt->rt6i_ref);
- inet6_rt_notify(RTM_NEWROUTE, rt, info);
+ inet6_rt_notify(RTM_NEWROUTE, rt, info, 0);
info->nl_net->ipv6.rt6_stats->fib_rt_entries++;
if (!(fn->fn_flags & RTN_RTINFO)) {
rt->rt6i_node = fn;
rt->dst.rt6_next = iter->dst.rt6_next;
atomic_inc(&rt->rt6i_ref);
- inet6_rt_notify(RTM_NEWROUTE, rt, info);
+ inet6_rt_notify(RTM_NEWROUTE, rt, info, NLM_F_REPLACE);
if (!(fn->fn_flags & RTN_RTINFO)) {
info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
fn->fn_flags |= RTN_RTINFO;
int replace_required = 0;
int sernum = fib6_new_sernum(info->nl_net);
+ if (WARN_ON_ONCE((rt->dst.flags & DST_NOCACHE) &&
+ !atomic_read(&rt->dst.__refcnt)))
+ return -EINVAL;
+
if (info->nlh) {
if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
allow_create = 0;
fib6_start_gc(info->nl_net, rt);
if (!(rt->rt6i_flags & RTF_CACHE))
fib6_prune_clones(info->nl_net, pn);
+ rt->dst.flags &= ~DST_NOCACHE;
}
out:
atomic_inc(&pn->leaf->rt6i_ref);
}
#endif
- dst_free(&rt->dst);
+ if (!(rt->dst.flags & DST_NOCACHE))
+ dst_free(&rt->dst);
}
return err;
st_failure:
if (fn && !(fn->fn_flags & (RTN_RTINFO|RTN_ROOT)))
fib6_repair_tree(info->nl_net, fn);
- dst_free(&rt->dst);
+ if (!(rt->dst.flags & DST_NOCACHE))
+ dst_free(&rt->dst);
return err;
#endif
}
fib6_purge_rt(rt, fn, net);
- inet6_rt_notify(RTM_DELROUTE, rt, info);
+ inet6_rt_notify(RTM_DELROUTE, rt, info, 0);
rt6_release(rt);
}
struct ipv6_tlv_tnl_enc_lim *tel;
__u32 mtu;
case ICMPV6_DEST_UNREACH:
- net_warn_ratelimited("%s: Path to destination invalid or inactive!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Path to destination invalid or inactive!\n",
+ t->parms.name);
break;
case ICMPV6_TIME_EXCEED:
if (code == ICMPV6_EXC_HOPLIMIT) {
- net_warn_ratelimited("%s: Too small hop limit or routing loop in tunnel!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Too small hop limit or routing loop in tunnel!\n",
+ t->parms.name);
}
break;
case ICMPV6_PARAMPROB:
if (teli && teli == be32_to_cpu(info) - 2) {
tel = (struct ipv6_tlv_tnl_enc_lim *) &skb->data[teli];
if (tel->encap_limit == 0) {
- net_warn_ratelimited("%s: Too small encapsulation limit or routing loop in tunnel!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Too small encapsulation limit or routing loop in tunnel!\n",
+ t->parms.name);
}
} else {
- net_warn_ratelimited("%s: Recipient unable to parse tunneled packet!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Recipient unable to parse tunneled packet!\n",
+ t->parms.name);
}
break;
case ICMPV6_PKT_TOOBIG:
}
if (!fl6->flowi6_mark)
- dst = ip6_tnl_dst_check(tunnel);
+ dst = ip6_tnl_dst_get(tunnel);
if (!dst) {
- ndst = ip6_route_output(net, NULL, fl6);
+ dst = ip6_route_output(net, NULL, fl6);
- if (ndst->error)
+ if (dst->error)
goto tx_err_link_failure;
- ndst = xfrm_lookup(net, ndst, flowi6_to_flowi(fl6), NULL, 0);
- if (IS_ERR(ndst)) {
- err = PTR_ERR(ndst);
- ndst = NULL;
+ dst = xfrm_lookup(net, dst, flowi6_to_flowi(fl6), NULL, 0);
+ if (IS_ERR(dst)) {
+ err = PTR_ERR(dst);
+ dst = NULL;
goto tx_err_link_failure;
}
- dst = ndst;
+ ndst = dst;
}
tdev = dst->dev;
skb = new_skb;
}
- if (fl6->flowi6_mark) {
- skb_dst_set(skb, dst);
- ndst = NULL;
- } else {
- skb_dst_set_noref(skb, dst);
- }
+ if (!fl6->flowi6_mark && ndst)
+ ip6_tnl_dst_set(tunnel, ndst);
+ skb_dst_set(skb, dst);
proto = NEXTHDR_GRE;
if (encap_limit >= 0) {
skb_set_inner_protocol(skb, protocol);
ip6tunnel_xmit(NULL, skb, dev);
- if (ndst)
- ip6_tnl_dst_store(tunnel, ndst);
return 0;
tx_err_link_failure:
stats->tx_carrier_errors++;
dst_link_failure(skb);
tx_err_dst_release:
- dst_release(ndst);
+ dst_release(dst);
return err;
}
static void ip6gre_dev_free(struct net_device *dev)
{
+ struct ip6_tnl *t = netdev_priv(dev);
+
+ ip6_tnl_dst_destroy(t);
free_percpu(dev->tstats);
free_netdev(dev);
}
netif_keep_dst(dev);
}
-static int ip6gre_tunnel_init(struct net_device *dev)
+static int ip6gre_tunnel_init_common(struct net_device *dev)
{
struct ip6_tnl *tunnel;
+ int ret;
tunnel = netdev_priv(dev);
tunnel->net = dev_net(dev);
strcpy(tunnel->parms.name, dev->name);
+ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+ if (!dev->tstats)
+ return -ENOMEM;
+
+ ret = ip6_tnl_dst_init(tunnel);
+ if (ret) {
+ free_percpu(dev->tstats);
+ dev->tstats = NULL;
+ return ret;
+ }
+
+ return 0;
+}
+
+static int ip6gre_tunnel_init(struct net_device *dev)
+{
+ struct ip6_tnl *tunnel;
+ int ret;
+
+ ret = ip6gre_tunnel_init_common(dev);
+ if (ret)
+ return ret;
+
+ tunnel = netdev_priv(dev);
+
memcpy(dev->dev_addr, &tunnel->parms.laddr, sizeof(struct in6_addr));
memcpy(dev->broadcast, &tunnel->parms.raddr, sizeof(struct in6_addr));
if (ipv6_addr_any(&tunnel->parms.raddr))
dev->header_ops = &ip6gre_header_ops;
- dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
- if (!dev->tstats)
- return -ENOMEM;
-
return 0;
}
static int ip6gre_tap_init(struct net_device *dev)
{
struct ip6_tnl *tunnel;
+ int ret;
- tunnel = netdev_priv(dev);
+ ret = ip6gre_tunnel_init_common(dev);
+ if (ret)
+ return ret;
- tunnel->dev = dev;
- tunnel->net = dev_net(dev);
- strcpy(tunnel->parms.name, dev->name);
+ tunnel = netdev_priv(dev);
ip6gre_tnl_link_config(tunnel, 1);
- dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
- if (!dev->tstats)
- return -ENOMEM;
-
return 0;
}
frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
&ipv6_hdr(skb)->saddr);
+ hroom = LL_RESERVED_SPACE(rt->dst.dev);
if (skb_has_frag_list(skb)) {
int first_len = skb_pagelen(skb);
struct sk_buff *frag2;
if (first_len - hlen > mtu ||
((first_len - hlen) & 7) ||
- skb_cloned(skb))
+ skb_cloned(skb) ||
+ skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
goto slow_path;
skb_walk_frags(skb, frag) {
/* Correct geometry. */
if (frag->len > mtu ||
((frag->len & 7) && frag->next) ||
- skb_headroom(frag) < hlen)
+ skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr)))
goto slow_path_clean;
/* Partially cloned skb? */
err = 0;
offset = 0;
- frag = skb_shinfo(skb)->frag_list;
- skb_frag_list_init(skb);
/* BUILD HEADER */
*prevhdr = NEXTHDR_FRAGMENT;
if (!tmp_hdr) {
IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
IPSTATS_MIB_FRAGFAILS);
- return -ENOMEM;
+ err = -ENOMEM;
+ goto fail;
}
+ frag = skb_shinfo(skb)->frag_list;
+ skb_frag_list_init(skb);
__skb_pull(skb, hlen);
fh = (struct frag_hdr *)__skb_push(skb, sizeof(struct frag_hdr));
*/
*prevhdr = NEXTHDR_FRAGMENT;
- hroom = LL_RESERVED_SPACE(rt->dst.dev);
troom = rt->dst.dev->needed_tailroom;
/*
* Locking : hash tables are protected by RCU and RTNL
*/
-struct dst_entry *ip6_tnl_dst_check(struct ip6_tnl *t)
+static void ip6_tnl_per_cpu_dst_set(struct ip6_tnl_dst *idst,
+ struct dst_entry *dst)
{
- struct dst_entry *dst = t->dst_cache;
+ write_seqlock_bh(&idst->lock);
+ dst_release(rcu_dereference_protected(
+ idst->dst,
+ lockdep_is_held(&idst->lock.lock)));
+ if (dst) {
+ dst_hold(dst);
+ idst->cookie = rt6_get_cookie((struct rt6_info *)dst);
+ } else {
+ idst->cookie = 0;
+ }
+ rcu_assign_pointer(idst->dst, dst);
+ write_sequnlock_bh(&idst->lock);
+}
+
+struct dst_entry *ip6_tnl_dst_get(struct ip6_tnl *t)
+{
+ struct ip6_tnl_dst *idst;
+ struct dst_entry *dst;
+ unsigned int seq;
+ u32 cookie;
- if (dst && dst->obsolete &&
- !dst->ops->check(dst, t->dst_cookie)) {
- t->dst_cache = NULL;
+ idst = raw_cpu_ptr(t->dst_cache);
+
+ rcu_read_lock();
+ do {
+ seq = read_seqbegin(&idst->lock);
+ dst = rcu_dereference(idst->dst);
+ cookie = idst->cookie;
+ } while (read_seqretry(&idst->lock, seq));
+
+ if (dst && !atomic_inc_not_zero(&dst->__refcnt))
+ dst = NULL;
+ rcu_read_unlock();
+
+ if (dst && dst->obsolete && !dst->ops->check(dst, cookie)) {
+ ip6_tnl_per_cpu_dst_set(idst, NULL);
dst_release(dst);
- return NULL;
+ dst = NULL;
}
-
return dst;
}
-EXPORT_SYMBOL_GPL(ip6_tnl_dst_check);
+EXPORT_SYMBOL_GPL(ip6_tnl_dst_get);
void ip6_tnl_dst_reset(struct ip6_tnl *t)
{
- dst_release(t->dst_cache);
- t->dst_cache = NULL;
+ int i;
+
+ for_each_possible_cpu(i)
+ ip6_tnl_per_cpu_dst_set(raw_cpu_ptr(t->dst_cache), NULL);
}
EXPORT_SYMBOL_GPL(ip6_tnl_dst_reset);
-void ip6_tnl_dst_store(struct ip6_tnl *t, struct dst_entry *dst)
+void ip6_tnl_dst_set(struct ip6_tnl *t, struct dst_entry *dst)
+{
+ ip6_tnl_per_cpu_dst_set(raw_cpu_ptr(t->dst_cache), dst);
+
+}
+EXPORT_SYMBOL_GPL(ip6_tnl_dst_set);
+
+void ip6_tnl_dst_destroy(struct ip6_tnl *t)
{
- struct rt6_info *rt = (struct rt6_info *) dst;
- t->dst_cookie = rt6_get_cookie(rt);
- dst_release(t->dst_cache);
- t->dst_cache = dst;
+ if (!t->dst_cache)
+ return;
+
+ ip6_tnl_dst_reset(t);
+ free_percpu(t->dst_cache);
}
-EXPORT_SYMBOL_GPL(ip6_tnl_dst_store);
+EXPORT_SYMBOL_GPL(ip6_tnl_dst_destroy);
+
+int ip6_tnl_dst_init(struct ip6_tnl *t)
+{
+ int i;
+
+ t->dst_cache = alloc_percpu(struct ip6_tnl_dst);
+ if (!t->dst_cache)
+ return -ENOMEM;
+
+ for_each_possible_cpu(i)
+ seqlock_init(&per_cpu_ptr(t->dst_cache, i)->lock);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(ip6_tnl_dst_init);
/**
* ip6_tnl_lookup - fetch tunnel matching the end-point addresses
static void ip6_dev_free(struct net_device *dev)
{
+ struct ip6_tnl *t = netdev_priv(dev);
+
+ ip6_tnl_dst_destroy(t);
free_percpu(dev->tstats);
free_netdev(dev);
}
struct ipv6_tlv_tnl_enc_lim *tel;
__u32 mtu;
case ICMPV6_DEST_UNREACH:
- net_warn_ratelimited("%s: Path to destination invalid or inactive!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Path to destination invalid or inactive!\n",
+ t->parms.name);
rel_msg = 1;
break;
case ICMPV6_TIME_EXCEED:
if ((*code) == ICMPV6_EXC_HOPLIMIT) {
- net_warn_ratelimited("%s: Too small hop limit or routing loop in tunnel!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Too small hop limit or routing loop in tunnel!\n",
+ t->parms.name);
rel_msg = 1;
}
break;
if (teli && teli == *info - 2) {
tel = (struct ipv6_tlv_tnl_enc_lim *) &skb->data[teli];
if (tel->encap_limit == 0) {
- net_warn_ratelimited("%s: Too small encapsulation limit or routing loop in tunnel!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Too small encapsulation limit or routing loop in tunnel!\n",
+ t->parms.name);
rel_msg = 1;
}
} else {
- net_warn_ratelimited("%s: Recipient unable to parse tunneled packet!\n",
- t->parms.name);
+ net_dbg_ratelimited("%s: Recipient unable to parse tunneled packet!\n",
+ t->parms.name);
}
break;
case ICMPV6_PKT_TOOBIG:
memcpy(&fl6->daddr, addr6, sizeof(fl6->daddr));
neigh_release(neigh);
} else if (!fl6->flowi6_mark)
- dst = ip6_tnl_dst_check(t);
+ dst = ip6_tnl_dst_get(t);
if (!ip6_tnl_xmit_ctl(t, &fl6->saddr, &fl6->daddr))
goto tx_err_link_failure;
if (!dst) {
- ndst = ip6_route_output(net, NULL, fl6);
+ dst = ip6_route_output(net, NULL, fl6);
- if (ndst->error)
+ if (dst->error)
goto tx_err_link_failure;
- ndst = xfrm_lookup(net, ndst, flowi6_to_flowi(fl6), NULL, 0);
- if (IS_ERR(ndst)) {
- err = PTR_ERR(ndst);
- ndst = NULL;
+ dst = xfrm_lookup(net, dst, flowi6_to_flowi(fl6), NULL, 0);
+ if (IS_ERR(dst)) {
+ err = PTR_ERR(dst);
+ dst = NULL;
goto tx_err_link_failure;
}
- dst = ndst;
+ ndst = dst;
}
tdev = dst->dev;
consume_skb(skb);
skb = new_skb;
}
- if (fl6->flowi6_mark) {
- skb_dst_set(skb, dst);
- ndst = NULL;
- } else {
- skb_dst_set_noref(skb, dst);
- }
+
+ if (!fl6->flowi6_mark && ndst)
+ ip6_tnl_dst_set(t, ndst);
+ skb_dst_set(skb, dst);
+
skb->transport_header = skb->network_header;
proto = fl6->flowi6_proto;
ipv6h->saddr = fl6->saddr;
ipv6h->daddr = fl6->daddr;
ip6tunnel_xmit(NULL, skb, dev);
- if (ndst)
- ip6_tnl_dst_store(t, ndst);
return 0;
tx_err_link_failure:
stats->tx_carrier_errors++;
dst_link_failure(skb);
tx_err_dst_release:
- dst_release(ndst);
+ dst_release(dst);
return err;
}
ip6_tnl_dev_init_gen(struct net_device *dev)
{
struct ip6_tnl *t = netdev_priv(dev);
+ int ret;
t->dev = dev;
t->net = dev_net(dev);
dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
if (!dev->tstats)
return -ENOMEM;
+
+ ret = ip6_tnl_dst_init(t);
+ if (ret) {
+ free_percpu(dev->tstats);
+ dev->tstats = NULL;
+ return ret;
+ }
+
return 0;
}
fl6->flowi6_iif = LOOPBACK_IFINDEX;
- if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr))
+ if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) ||
+ fl6->flowi6_oif)
flags |= RT6_LOOKUP_F_IFACE;
if (!ipv6_addr_any(&fl6->saddr))
if (rt) {
if (rt->rt6i_flags & RTF_CACHE) {
dst_hold(&rt->dst);
- if (ip6_del_rt(rt))
- dst_free(&rt->dst);
+ ip6_del_rt(rt);
} else if (rt->rt6i_node && (rt->rt6i_flags & RTF_DEFAULT)) {
rt->rt6i_node->fn_sernum = -1;
}
rt->dst.input = ip6_pkt_prohibit;
break;
case RTN_THROW:
+ case RTN_UNREACHABLE:
default:
rt->dst.error = (cfg->fc_type == RTN_THROW) ? -EAGAIN
- : -ENETUNREACH;
+ : (cfg->fc_type == RTN_UNREACHABLE)
+ ? -EHOSTUNREACH : -ENETUNREACH;
rt->dst.output = ip6_pkt_discard_out;
rt->dst.input = ip6_pkt_discard;
break;
struct fib6_table *table;
struct net *net = dev_net(rt->dst.dev);
- if (rt == net->ipv6.ip6_null_entry) {
+ if (rt == net->ipv6.ip6_null_entry ||
+ rt->dst.flags & DST_NOCACHE) {
err = -ENOENT;
goto out;
}
rt->rt6i_dst.addr = *addr;
rt->rt6i_dst.plen = 128;
rt->rt6i_table = fib6_get_table(net, RT6_TABLE_LOCAL);
+ rt->dst.flags |= DST_NOCACHE;
atomic_set(&rt->dst.__refcnt, 1);
return err;
}
-void inet6_rt_notify(int event, struct rt6_info *rt, struct nl_info *info)
+void inet6_rt_notify(int event, struct rt6_info *rt, struct nl_info *info,
+ unsigned int nlm_flags)
{
struct sk_buff *skb;
struct net *net = info->nl_net;
goto errout;
err = rt6_fill_node(net, skb, rt, NULL, NULL, 0,
- event, info->portid, seq, 0, 0, 0);
+ event, info->portid, seq, 0, 0, nlm_flags);
if (err < 0) {
/* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
WARN_ON(err == -EMSGSIZE);
tunnel = container_of(work, struct l2tp_tunnel, del_work);
sk = l2tp_tunnel_sock_lookup(tunnel);
if (!sk)
- return;
+ goto out;
sock = sk->sk_socket;
}
l2tp_tunnel_sock_put(sk);
+out:
+ l2tp_tunnel_dec_refcount(tunnel);
}
/* Create a socket for the tunnel, if one isn't set up by
*/
int l2tp_tunnel_delete(struct l2tp_tunnel *tunnel)
{
+ l2tp_tunnel_inc_refcount(tunnel);
l2tp_tunnel_closeall(tunnel);
- return (false == queue_work(l2tp_wq, &tunnel->del_work));
+ if (false == queue_work(l2tp_wq, &tunnel->del_work)) {
+ l2tp_tunnel_dec_refcount(tunnel);
+ return 1;
+ }
+ return 0;
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_delete);
bss_conf->cqm_rssi_thold = rssi_thold;
bss_conf->cqm_rssi_hyst = rssi_hyst;
+ sdata->u.mgd.last_cqm_event_signal = 0;
/* tell the driver upon association, unless already associated */
if (sdata->u.mgd.associated &&
continue;
for (j = 0; j < IEEE80211_HT_MCS_MASK_LEN; j++) {
- if (~sdata->rc_rateidx_mcs_mask[i][j])
+ if (~sdata->rc_rateidx_mcs_mask[i][j]) {
sdata->rc_has_mcs_mask[i] = true;
+ break;
+ }
+ }
- if (~sdata->rc_rateidx_vht_mcs_mask[i][j])
+ for (j = 0; j < NL80211_VHT_NSS_MAX; j++) {
+ if (~sdata->rc_rateidx_vht_mcs_mask[i][j]) {
sdata->rc_has_vht_mcs_mask[i] = true;
-
- if (sdata->rc_has_mcs_mask[i] &&
- sdata->rc_has_vht_mcs_mask[i])
break;
+ }
}
}
void nf_log_unregister(struct nf_logger *logger)
{
+ const struct nf_logger *log;
int i;
mutex_lock(&nf_log_mutex);
- for (i = 0; i < NFPROTO_NUMPROTO; i++)
- RCU_INIT_POINTER(loggers[i][logger->type], NULL);
+ for (i = 0; i < NFPROTO_NUMPROTO; i++) {
+ log = nft_log_dereference(loggers[i][logger->type]);
+ if (log == logger)
+ RCU_INIT_POINTER(loggers[i][logger->type], NULL);
+ }
mutex_unlock(&nf_log_mutex);
+ synchronize_rcu();
}
EXPORT_SYMBOL(nf_log_unregister);
static struct nft_expr_type nft_match_type;
+static bool nft_match_cmp(const struct xt_match *match,
+ const char *name, u32 rev, u32 family)
+{
+ return strcmp(match->name, name) == 0 && match->revision == rev &&
+ (match->family == NFPROTO_UNSPEC || match->family == family);
+}
+
static const struct nft_expr_ops *
nft_match_select_ops(const struct nft_ctx *ctx,
const struct nlattr * const tb[])
struct nft_xt *nft_match;
struct xt_match *match;
char *mt_name;
- __u32 rev, family;
+ u32 rev, family;
if (tb[NFTA_MATCH_NAME] == NULL ||
tb[NFTA_MATCH_REV] == NULL ||
list_for_each_entry(nft_match, &nft_match_list, head) {
struct xt_match *match = nft_match->ops.data;
- if (strcmp(match->name, mt_name) == 0 &&
- match->revision == rev && match->family == family) {
+ if (nft_match_cmp(match, mt_name, rev, family)) {
if (!try_module_get(match->me))
return ERR_PTR(-ENOENT);
static struct nft_expr_type nft_target_type;
+static bool nft_target_cmp(const struct xt_target *tg,
+ const char *name, u32 rev, u32 family)
+{
+ return strcmp(tg->name, name) == 0 && tg->revision == rev &&
+ (tg->family == NFPROTO_UNSPEC || tg->family == family);
+}
+
static const struct nft_expr_ops *
nft_target_select_ops(const struct nft_ctx *ctx,
const struct nlattr * const tb[])
struct nft_xt *nft_target;
struct xt_target *target;
char *tg_name;
- __u32 rev, family;
+ u32 rev, family;
if (tb[NFTA_TARGET_NAME] == NULL ||
tb[NFTA_TARGET_REV] == NULL ||
list_for_each_entry(nft_target, &nft_target_list, head) {
struct xt_target *target = nft_target->ops.data;
- if (strcmp(target->name, tg_name) == 0 &&
- target->revision == rev && target->family == family) {
+ if (nft_target_cmp(target, tg_name, rev, family)) {
if (!try_module_get(target->me))
return ERR_PTR(-ENOENT);
return group ? 1 << (group - 1) : 0;
}
+static struct sk_buff *netlink_to_full_skb(const struct sk_buff *skb,
+ gfp_t gfp_mask)
+{
+ unsigned int len = skb_end_offset(skb);
+ struct sk_buff *new;
+
+ new = alloc_skb(len, gfp_mask);
+ if (new == NULL)
+ return NULL;
+
+ NETLINK_CB(new).portid = NETLINK_CB(skb).portid;
+ NETLINK_CB(new).dst_group = NETLINK_CB(skb).dst_group;
+ NETLINK_CB(new).creds = NETLINK_CB(skb).creds;
+
+ memcpy(skb_put(new, len), skb->data, len);
+ return new;
+}
+
int netlink_add_tap(struct netlink_tap *nt)
{
if (unlikely(nt->dev->type != ARPHRD_NETLINK))
int ret = -ENOMEM;
dev_hold(dev);
- nskb = skb_clone(skb, GFP_ATOMIC);
+
+ if (netlink_skb_is_mmaped(skb) || is_vmalloc_addr(skb->head))
+ nskb = netlink_to_full_skb(skb, GFP_ATOMIC);
+ else
+ nskb = skb_clone(skb, GFP_ATOMIC);
if (nskb) {
nskb->dev = dev;
nskb->protocol = htons((u16) sk->sk_protocol);
}
#ifdef CONFIG_NETLINK_MMAP
-static bool netlink_skb_is_mmaped(const struct sk_buff *skb)
-{
- return NETLINK_CB(skb).flags & NETLINK_SKB_MMAPED;
-}
-
static bool netlink_rx_is_mmaped(struct sock *sk)
{
return nlk_sk(sk)->rx_ring.pg_vec != NULL;
}
#else /* CONFIG_NETLINK_MMAP */
-#define netlink_skb_is_mmaped(skb) false
#define netlink_rx_is_mmaped(sk) false
#define netlink_tx_is_mmaped(sk) false
#define netlink_mmap sock_no_mmap
lock_sock(sk);
- err = -EBUSY;
- if (nlk_sk(sk)->portid)
+ err = nlk_sk(sk)->portid == portid ? 0 : -EBUSY;
+ if (nlk_sk(sk)->bound)
goto err;
err = -ENOMEM;
err = -EOVERFLOW;
if (err == -EEXIST)
err = -EADDRINUSE;
- nlk_sk(sk)->portid = 0;
sock_put(sk);
+ goto err;
}
+ /* We need to ensure that the socket is hashed and visible. */
+ smp_wmb();
+ nlk_sk(sk)->bound = portid;
+
err:
release_sock(sk);
return err;
struct sockaddr_nl *nladdr = (struct sockaddr_nl *)addr;
int err;
long unsigned int groups = nladdr->nl_groups;
+ bool bound;
if (addr_len < sizeof(struct sockaddr_nl))
return -EINVAL;
return err;
}
- if (nlk->portid)
+ bound = nlk->bound;
+ if (bound) {
+ /* Ensure nlk->portid is up-to-date. */
+ smp_rmb();
+
if (nladdr->nl_pid != nlk->portid)
return -EINVAL;
+ }
if (nlk->netlink_bind && groups) {
int group;
}
}
- if (!nlk->portid) {
+ /* No need for barriers here as we return to user-space without
+ * using any of the bound attributes.
+ */
+ if (!bound) {
err = nladdr->nl_pid ?
netlink_insert(sk, nladdr->nl_pid) :
netlink_autobind(sock);
!netlink_allowed(sock, NL_CFG_F_NONROOT_SEND))
return -EPERM;
- if (!nlk->portid)
+ /* No need for barriers here as we return to user-space without
+ * using any of the bound attributes.
+ */
+ if (!nlk->bound)
err = netlink_autobind(sock);
if (err == 0) {
dst_group = nlk->dst_group;
}
- if (!nlk->portid) {
+ if (!nlk->bound) {
err = netlink_autobind(sock);
if (err)
goto out;
+ } else {
+ /* Ensure nlk is hashed and visible. */
+ smp_rmb();
}
/* It's a really convoluted way for userland to ask for mmaped
unsigned long state;
size_t max_recvmsg_len;
wait_queue_head_t wait;
+ bool bound;
bool cb_running;
struct netlink_callback cb;
struct mutex *cb_mutex;
return container_of(sk, struct netlink_sock, sk);
}
+static inline bool netlink_skb_is_mmaped(const struct sk_buff *skb)
+{
+#ifdef CONFIG_NETLINK_MMAP
+ return NETLINK_CB(skb).flags & NETLINK_SKB_MMAPED;
+#else
+ return false;
+#endif /* CONFIG_NETLINK_MMAP */
+}
+
struct netlink_table {
struct rhashtable hash;
struct hlist_head mc_list;
config OPENVSWITCH
tristate "Open vSwitch"
depends on INET
- depends on (!NF_CONNTRACK || NF_CONNTRACK)
+ depends on !NF_CONNTRACK || \
+ (NF_CONNTRACK && (!NF_DEFRAG_IPV6 || NF_DEFRAG_IPV6))
select LIBCRC32C
select MPLS
select NET_MPLS_GSO
case NFPROTO_IPV6: {
u8 nexthdr = ipv6_hdr(skb)->nexthdr;
__be16 frag_off;
+ int ofs;
- protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr),
- &nexthdr, &frag_off);
- if (protoff < 0 || (frag_off & htons(~0x7)) != 0) {
+ ofs = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &nexthdr,
+ &frag_off);
+ if (ofs < 0 || (frag_off & htons(~0x7)) != 0) {
pr_debug("proto header not found\n");
return NF_ACCEPT;
}
+ protoff = ofs;
break;
}
default:
if (error)
goto err_kfree_flow;
- ovs_flow_mask_key(&new_flow->key, &key, &mask);
+ ovs_flow_mask_key(&new_flow->key, &key, true, &mask);
/* Extract flow identifier. */
error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
struct sw_flow_key masked_key;
int error;
- ovs_flow_mask_key(&masked_key, key, mask);
+ ovs_flow_mask_key(&masked_key, key, true, mask);
error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
if (error) {
OVS_NLERR(log,
};
#define OVS_ATTR_NESTED -1
+#define OVS_ATTR_VARIABLE -2
static void update_range(struct sw_flow_match *match,
size_t offset, size_t size, bool is_mask)
+ nla_total_size(28); /* OVS_KEY_ATTR_ND */
}
+static const struct ovs_len_tbl ovs_vxlan_ext_key_lens[OVS_VXLAN_EXT_MAX + 1] = {
+ [OVS_VXLAN_EXT_GBP] = { .len = sizeof(u32) },
+};
+
static const struct ovs_len_tbl ovs_tunnel_key_lens[OVS_TUNNEL_KEY_ATTR_MAX + 1] = {
[OVS_TUNNEL_KEY_ATTR_ID] = { .len = sizeof(u64) },
[OVS_TUNNEL_KEY_ATTR_IPV4_SRC] = { .len = sizeof(u32) },
[OVS_TUNNEL_KEY_ATTR_TP_SRC] = { .len = sizeof(u16) },
[OVS_TUNNEL_KEY_ATTR_TP_DST] = { .len = sizeof(u16) },
[OVS_TUNNEL_KEY_ATTR_OAM] = { .len = 0 },
- [OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS] = { .len = OVS_ATTR_NESTED },
- [OVS_TUNNEL_KEY_ATTR_VXLAN_OPTS] = { .len = OVS_ATTR_NESTED },
+ [OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS] = { .len = OVS_ATTR_VARIABLE },
+ [OVS_TUNNEL_KEY_ATTR_VXLAN_OPTS] = { .len = OVS_ATTR_NESTED,
+ .next = ovs_vxlan_ext_key_lens },
};
/* The size of the argument for each %OVS_KEY_ATTR_* Netlink attribute. */
[OVS_KEY_ATTR_CT_LABEL] = { .len = sizeof(struct ovs_key_ct_label) },
};
+static bool check_attr_len(unsigned int attr_len, unsigned int expected_len)
+{
+ return expected_len == attr_len ||
+ expected_len == OVS_ATTR_NESTED ||
+ expected_len == OVS_ATTR_VARIABLE;
+}
+
static bool is_all_zero(const u8 *fp, size_t size)
{
int i;
}
expected_len = ovs_key_lens[type].len;
- if (nla_len(nla) != expected_len && expected_len != OVS_ATTR_NESTED) {
+ if (!check_attr_len(nla_len(nla), expected_len)) {
OVS_NLERR(log, "Key %d has unexpected len %d expected %d",
type, nla_len(nla), expected_len);
return -EINVAL;
return 0;
}
-static const struct nla_policy vxlan_opt_policy[OVS_VXLAN_EXT_MAX + 1] = {
- [OVS_VXLAN_EXT_GBP] = { .type = NLA_U32 },
-};
-
-static int vxlan_tun_opt_from_nlattr(const struct nlattr *a,
+static int vxlan_tun_opt_from_nlattr(const struct nlattr *attr,
struct sw_flow_match *match, bool is_mask,
bool log)
{
- struct nlattr *tb[OVS_VXLAN_EXT_MAX+1];
+ struct nlattr *a;
+ int rem;
unsigned long opt_key_offset;
struct vxlan_metadata opts;
- int err;
BUILD_BUG_ON(sizeof(opts) > sizeof(match->key->tun_opts));
- err = nla_parse_nested(tb, OVS_VXLAN_EXT_MAX, a, vxlan_opt_policy);
- if (err < 0)
- return err;
-
memset(&opts, 0, sizeof(opts));
+ nla_for_each_nested(a, attr, rem) {
+ int type = nla_type(a);
- if (tb[OVS_VXLAN_EXT_GBP])
- opts.gbp = nla_get_u32(tb[OVS_VXLAN_EXT_GBP]);
+ if (type > OVS_VXLAN_EXT_MAX) {
+ OVS_NLERR(log, "VXLAN extension %d out of range max %d",
+ type, OVS_VXLAN_EXT_MAX);
+ return -EINVAL;
+ }
+
+ if (!check_attr_len(nla_len(a),
+ ovs_vxlan_ext_key_lens[type].len)) {
+ OVS_NLERR(log, "VXLAN extension %d has unexpected len %d expected %d",
+ type, nla_len(a),
+ ovs_vxlan_ext_key_lens[type].len);
+ return -EINVAL;
+ }
+
+ switch (type) {
+ case OVS_VXLAN_EXT_GBP:
+ opts.gbp = nla_get_u32(a);
+ break;
+ default:
+ OVS_NLERR(log, "Unknown VXLAN extension attribute %d",
+ type);
+ return -EINVAL;
+ }
+ }
+ if (rem) {
+ OVS_NLERR(log, "VXLAN extension message has %d unknown bytes.",
+ rem);
+ return -EINVAL;
+ }
if (!is_mask)
SW_FLOW_KEY_PUT(match, tun_opts_len, sizeof(opts), false);
return -EINVAL;
}
- if (ovs_tunnel_key_lens[type].len != nla_len(a) &&
- ovs_tunnel_key_lens[type].len != OVS_ATTR_NESTED) {
+ if (!check_attr_len(nla_len(a),
+ ovs_tunnel_key_lens[type].len)) {
OVS_NLERR(log, "Tunnel attr %d has unexpected len %d expected %d",
type, nla_len(a), ovs_tunnel_key_lens[type].len);
return -EINVAL;
/* The nlattr stream should already have been validated */
nla_for_each_nested(nla, attr, rem) {
- if (tbl && tbl[nla_type(nla)].len == OVS_ATTR_NESTED)
- nlattr_set(nla, val, tbl[nla_type(nla)].next);
- else
+ if (tbl[nla_type(nla)].len == OVS_ATTR_NESTED) {
+ if (tbl[nla_type(nla)].next)
+ tbl = tbl[nla_type(nla)].next;
+ nlattr_set(nla, val, tbl);
+ } else {
memset(nla_data(nla), val, nla_len(nla));
+ }
}
}
key_len /= 2;
if (key_type > OVS_KEY_ATTR_MAX ||
- (ovs_key_lens[key_type].len != key_len &&
- ovs_key_lens[key_type].len != OVS_ATTR_NESTED))
+ !check_attr_len(key_len, ovs_key_lens[key_type].len))
return -EINVAL;
if (masked && !validate_masked(nla_data(ovs_key), key_len))
}
void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src,
- const struct sw_flow_mask *mask)
+ bool full, const struct sw_flow_mask *mask)
{
- const long *m = (const long *)((const u8 *)&mask->key +
- mask->range.start);
- const long *s = (const long *)((const u8 *)src +
- mask->range.start);
- long *d = (long *)((u8 *)dst + mask->range.start);
+ int start = full ? 0 : mask->range.start;
+ int len = full ? sizeof *dst : range_n_bytes(&mask->range);
+ const long *m = (const long *)((const u8 *)&mask->key + start);
+ const long *s = (const long *)((const u8 *)src + start);
+ long *d = (long *)((u8 *)dst + start);
int i;
- /* The memory outside of the 'mask->range' are not set since
- * further operations on 'dst' only uses contents within
- * 'mask->range'.
+ /* If 'full' is true then all of 'dst' is fully initialized. Otherwise,
+ * if 'full' is false the memory outside of the 'mask->range' is left
+ * uninitialized. This can be used as an optimization when further
+ * operations on 'dst' only use contents within 'mask->range'.
*/
- for (i = 0; i < range_n_bytes(&mask->range); i += sizeof(long))
+ for (i = 0; i < len; i += sizeof(long))
*d++ = *s++ & *m++;
}
u32 hash;
struct sw_flow_key masked_key;
- ovs_flow_mask_key(&masked_key, unmasked, mask);
+ ovs_flow_mask_key(&masked_key, unmasked, false, mask);
hash = flow_hash(&masked_key, &mask->range);
head = find_bucket(ti, hash);
hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver]) {
bool ovs_flow_cmp(const struct sw_flow *, const struct sw_flow_match *);
void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src,
- const struct sw_flow_mask *mask);
+ bool full, const struct sw_flow_mask *mask);
#endif /* flow_table.h */
} sa;
};
+#define vio_le() virtio_legacy_is_little_endian()
+
#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
goto out_unlock;
if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
- (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
- __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
- __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
- vnet_hdr.hdr_len = __cpu_to_virtio16(false,
- __virtio16_to_cpu(false, vnet_hdr.csum_start) +
- __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
+ (__virtio16_to_cpu(vio_le(), vnet_hdr.csum_start) +
+ __virtio16_to_cpu(vio_le(), vnet_hdr.csum_offset) + 2 >
+ __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len)))
+ vnet_hdr.hdr_len = __cpu_to_virtio16(vio_le(),
+ __virtio16_to_cpu(vio_le(), vnet_hdr.csum_start) +
+ __virtio16_to_cpu(vio_le(), vnet_hdr.csum_offset) + 2);
err = -EINVAL;
- if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
+ if (__virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len) > len)
goto out_unlock;
if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
hlen = LL_RESERVED_SPACE(dev);
tlen = dev->needed_tailroom;
skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
- __virtio16_to_cpu(false, vnet_hdr.hdr_len),
+ __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len),
msg->msg_flags & MSG_DONTWAIT, &err);
if (skb == NULL)
goto out_unlock;
if (po->has_vnet_hdr) {
if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
- u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
- u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
+ u16 s = __virtio16_to_cpu(vio_le(), vnet_hdr.csum_start);
+ u16 o = __virtio16_to_cpu(vio_le(), vnet_hdr.csum_offset);
if (!skb_partial_csum_set(skb, s, o)) {
err = -EINVAL;
goto out_free;
}
skb_shinfo(skb)->gso_size =
- __virtio16_to_cpu(false, vnet_hdr.gso_size);
+ __virtio16_to_cpu(vio_le(), vnet_hdr.gso_size);
skb_shinfo(skb)->gso_type = gso_type;
/* Header must be checked, and gso_segs computed. */
/* This is a hint as to how much should be linear. */
vnet_hdr.hdr_len =
- __cpu_to_virtio16(false, skb_headlen(skb));
+ __cpu_to_virtio16(vio_le(), skb_headlen(skb));
vnet_hdr.gso_size =
- __cpu_to_virtio16(false, sinfo->gso_size);
+ __cpu_to_virtio16(vio_le(), sinfo->gso_size);
if (sinfo->gso_type & SKB_GSO_TCPV4)
vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
else if (sinfo->gso_type & SKB_GSO_TCPV6)
if (skb->ip_summed == CHECKSUM_PARTIAL) {
vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
- vnet_hdr.csum_start = __cpu_to_virtio16(false,
+ vnet_hdr.csum_start = __cpu_to_virtio16(vio_le(),
skb_checksum_start_offset(skb));
- vnet_hdr.csum_offset = __cpu_to_virtio16(false,
+ vnet_hdr.csum_offset = __cpu_to_virtio16(vio_le(),
skb->csum_offset);
} else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
struct fw_head {
u32 mask;
- bool mask_set;
struct fw_filter __rcu *ht[HTSIZE];
struct rcu_head rcu;
};
}
}
} else {
- /* old method */
+ /* Old method: classify the packet using its skb mark. */
if (id && (TC_H_MAJ(id) == 0 ||
!(TC_H_MAJ(id ^ tp->q->handle)))) {
res->classid = id;
static int fw_init(struct tcf_proto *tp)
{
- struct fw_head *head;
-
- head = kzalloc(sizeof(struct fw_head), GFP_KERNEL);
- if (head == NULL)
- return -ENOBUFS;
-
- head->mask_set = false;
- rcu_assign_pointer(tp->root, head);
+ /* We don't allocate fw_head here, because in the old method
+ * we don't need it at all.
+ */
return 0;
}
int err;
if (!opt)
- return handle ? -EINVAL : 0;
+ return handle ? -EINVAL : 0; /* Succeed if it is old method. */
err = nla_parse_nested(tb, TCA_FW_MAX, opt, fw_policy);
if (err < 0)
if (!handle)
return -EINVAL;
- if (!head->mask_set) {
- head->mask = 0xFFFFFFFF;
+ if (!head) {
+ u32 mask = 0xFFFFFFFF;
if (tb[TCA_FW_MASK])
- head->mask = nla_get_u32(tb[TCA_FW_MASK]);
- head->mask_set = true;
+ mask = nla_get_u32(tb[TCA_FW_MASK]);
+
+ head = kzalloc(sizeof(*head), GFP_KERNEL);
+ if (!head)
+ return -ENOBUFS;
+ head->mask = mask;
+
+ rcu_assign_pointer(tp->root, head);
}
f = kzalloc(sizeof(struct fw_filter), GFP_KERNEL);
* within this document.
*
* Our basic strategy is to round-robin transports in priorities
- * according to sctp_state_prio_map[] e.g., if no such
+ * according to sctp_trans_score() e.g., if no such
* transport with state SCTP_ACTIVE exists, round-robin through
* SCTP_UNKNOWN, etc. You get the picture.
*/
-static const u8 sctp_trans_state_to_prio_map[] = {
- [SCTP_ACTIVE] = 3, /* best case */
- [SCTP_UNKNOWN] = 2,
- [SCTP_PF] = 1,
- [SCTP_INACTIVE] = 0, /* worst case */
-};
-
static u8 sctp_trans_score(const struct sctp_transport *trans)
{
- return sctp_trans_state_to_prio_map[trans->state];
+ switch (trans->state) {
+ case SCTP_ACTIVE:
+ return 3; /* best case */
+ case SCTP_UNKNOWN:
+ return 2;
+ case SCTP_PF:
+ return 1;
+ default: /* case SCTP_INACTIVE */
+ return 0; /* worst case */
+ }
}
static struct sctp_transport *sctp_trans_elect_tie(struct sctp_transport *trans1,
unregister_inetaddr_notifier(&sctp_inetaddr_notifier);
}
-static int __net_init sctp_net_init(struct net *net)
+static int __net_init sctp_defaults_init(struct net *net)
{
int status;
sctp_dbg_objcnt_init(net);
- /* Initialize the control inode/socket for handling OOTB packets. */
- if ((status = sctp_ctl_sock_init(net))) {
- pr_err("Failed to initialize the SCTP control sock\n");
- goto err_ctl_sock_init;
- }
-
/* Initialize the local address list. */
INIT_LIST_HEAD(&net->sctp.local_addr_list);
spin_lock_init(&net->sctp.local_addr_lock);
return 0;
-err_ctl_sock_init:
- sctp_dbg_objcnt_exit(net);
- sctp_proc_exit(net);
err_init_proc:
cleanup_sctp_mibs(net);
err_init_mibs:
return status;
}
-static void __net_exit sctp_net_exit(struct net *net)
+static void __net_exit sctp_defaults_exit(struct net *net)
{
/* Free the local address list */
sctp_free_addr_wq(net);
sctp_free_local_addr_list(net);
- /* Free the control endpoint. */
- inet_ctl_sock_destroy(net->sctp.ctl_sock);
-
sctp_dbg_objcnt_exit(net);
sctp_proc_exit(net);
sctp_sysctl_net_unregister(net);
}
-static struct pernet_operations sctp_net_ops = {
- .init = sctp_net_init,
- .exit = sctp_net_exit,
+static struct pernet_operations sctp_defaults_ops = {
+ .init = sctp_defaults_init,
+ .exit = sctp_defaults_exit,
+};
+
+static int __net_init sctp_ctrlsock_init(struct net *net)
+{
+ int status;
+
+ /* Initialize the control inode/socket for handling OOTB packets. */
+ status = sctp_ctl_sock_init(net);
+ if (status)
+ pr_err("Failed to initialize the SCTP control sock\n");
+
+ return status;
+}
+
+static void __net_init sctp_ctrlsock_exit(struct net *net)
+{
+ /* Free the control endpoint. */
+ inet_ctl_sock_destroy(net->sctp.ctl_sock);
+}
+
+static struct pernet_operations sctp_ctrlsock_ops = {
+ .init = sctp_ctrlsock_init,
+ .exit = sctp_ctrlsock_exit,
};
/* Initialize the universe into something sensible. */
sctp_v4_pf_init();
sctp_v6_pf_init();
- status = sctp_v4_protosw_init();
+ status = register_pernet_subsys(&sctp_defaults_ops);
+ if (status)
+ goto err_register_defaults;
+ status = sctp_v4_protosw_init();
if (status)
goto err_protosw_init;
if (status)
goto err_v6_protosw_init;
- status = register_pernet_subsys(&sctp_net_ops);
+ status = register_pernet_subsys(&sctp_ctrlsock_ops);
if (status)
- goto err_register_pernet_subsys;
+ goto err_register_ctrlsock;
status = sctp_v4_add_protocol();
if (status)
err_v6_add_protocol:
sctp_v4_del_protocol();
err_add_protocol:
- unregister_pernet_subsys(&sctp_net_ops);
-err_register_pernet_subsys:
+ unregister_pernet_subsys(&sctp_ctrlsock_ops);
+err_register_ctrlsock:
sctp_v6_protosw_exit();
err_v6_protosw_init:
sctp_v4_protosw_exit();
err_protosw_init:
+ unregister_pernet_subsys(&sctp_defaults_ops);
+err_register_defaults:
sctp_v4_pf_exit();
sctp_v6_pf_exit();
sctp_sysctl_unregister();
sctp_v6_del_protocol();
sctp_v4_del_protocol();
- unregister_pernet_subsys(&sctp_net_ops);
+ unregister_pernet_subsys(&sctp_ctrlsock_ops);
/* Free protosw registrations */
sctp_v6_protosw_exit();
sctp_v4_protosw_exit();
+ unregister_pernet_subsys(&sctp_defaults_ops);
+
/* Unregister with socket layer. */
sctp_v6_pf_exit();
sctp_v4_pf_exit();
int error;
struct sctp_transport *transport = (struct sctp_transport *) peer;
struct sctp_association *asoc = transport->asoc;
- struct net *net = sock_net(asoc->base.sk);
+ struct sock *sk = asoc->base.sk;
+ struct net *net = sock_net(sk);
/* Check whether a task is in the sock. */
- bh_lock_sock(asoc->base.sk);
- if (sock_owned_by_user(asoc->base.sk)) {
+ bh_lock_sock(sk);
+ if (sock_owned_by_user(sk)) {
pr_debug("%s: sock is busy\n", __func__);
/* Try again later. */
transport, GFP_ATOMIC);
if (error)
- asoc->base.sk->sk_err = -error;
+ sk->sk_err = -error;
out_unlock:
- bh_unlock_sock(asoc->base.sk);
+ bh_unlock_sock(sk);
sctp_transport_put(transport);
}
static void sctp_generate_timeout_event(struct sctp_association *asoc,
sctp_event_timeout_t timeout_type)
{
- struct net *net = sock_net(asoc->base.sk);
+ struct sock *sk = asoc->base.sk;
+ struct net *net = sock_net(sk);
int error = 0;
- bh_lock_sock(asoc->base.sk);
- if (sock_owned_by_user(asoc->base.sk)) {
+ bh_lock_sock(sk);
+ if (sock_owned_by_user(sk)) {
pr_debug("%s: sock is busy: timer %d\n", __func__,
timeout_type);
(void *)timeout_type, GFP_ATOMIC);
if (error)
- asoc->base.sk->sk_err = -error;
+ sk->sk_err = -error;
out_unlock:
- bh_unlock_sock(asoc->base.sk);
+ bh_unlock_sock(sk);
sctp_association_put(asoc);
}
int error = 0;
struct sctp_transport *transport = (struct sctp_transport *) data;
struct sctp_association *asoc = transport->asoc;
- struct net *net = sock_net(asoc->base.sk);
+ struct sock *sk = asoc->base.sk;
+ struct net *net = sock_net(sk);
- bh_lock_sock(asoc->base.sk);
- if (sock_owned_by_user(asoc->base.sk)) {
+ bh_lock_sock(sk);
+ if (sock_owned_by_user(sk)) {
pr_debug("%s: sock is busy\n", __func__);
/* Try again later. */
asoc->state, asoc->ep, asoc,
transport, GFP_ATOMIC);
- if (error)
- asoc->base.sk->sk_err = -error;
+ if (error)
+ sk->sk_err = -error;
out_unlock:
- bh_unlock_sock(asoc->base.sk);
+ bh_unlock_sock(sk);
sctp_transport_put(transport);
}
{
struct sctp_transport *transport = (struct sctp_transport *) data;
struct sctp_association *asoc = transport->asoc;
- struct net *net = sock_net(asoc->base.sk);
+ struct sock *sk = asoc->base.sk;
+ struct net *net = sock_net(sk);
- bh_lock_sock(asoc->base.sk);
- if (sock_owned_by_user(asoc->base.sk)) {
+ bh_lock_sock(sk);
+ if (sock_owned_by_user(sk)) {
pr_debug("%s: sock is busy\n", __func__);
/* Try again later. */
asoc->state, asoc->ep, asoc, transport, GFP_ATOMIC);
out_unlock:
- bh_unlock_sock(asoc->base.sk);
+ bh_unlock_sock(sk);
sctp_association_put(asoc);
}
clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
ret = atomic_dec_and_test(&task->tk_count);
if (waitqueue_active(wq))
- __wake_up_locked_key(wq, TASK_NORMAL, 1, &k);
+ __wake_up_locked_key(wq, TASK_NORMAL, &k);
spin_unlock_irqrestore(&wq->lock, flags);
return ret;
}
rpc_destroy_mempool(void)
{
rpciod_stop();
- if (rpc_buffer_mempool)
- mempool_destroy(rpc_buffer_mempool);
- if (rpc_task_mempool)
- mempool_destroy(rpc_task_mempool);
- if (rpc_task_slabp)
- kmem_cache_destroy(rpc_task_slabp);
- if (rpc_buffer_slabp)
- kmem_cache_destroy(rpc_buffer_slabp);
+ mempool_destroy(rpc_buffer_mempool);
+ mempool_destroy(rpc_task_mempool);
+ kmem_cache_destroy(rpc_task_slabp);
+ kmem_cache_destroy(rpc_buffer_slabp);
rpc_destroy_wait_queue(&delay_queue);
}
clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
xprt->ops->close(xprt);
xprt_release_write(xprt, NULL);
+ wake_up_bit(&xprt->state, XPRT_LOCKED);
}
/**
xprt->ops->release_xprt(xprt, NULL);
out:
spin_unlock_bh(&xprt->transport_lock);
+ wake_up_bit(&xprt->state, XPRT_LOCKED);
}
/**
static void xprt_destroy(struct rpc_xprt *xprt)
{
dprintk("RPC: destroying transport %p\n", xprt);
+
+ /* Exclude transport connect/disconnect handlers */
+ wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE);
+
del_timer_sync(&xprt->timer);
rpc_xprt_debugfs_unregister(xprt);
fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
struct rpcrdma_create_data_internal *cdata)
{
- struct ib_device_attr *devattr = &ia->ri_devattr;
- struct ib_mr *mr;
-
- /* Obtain an lkey to use for the regbufs, which are
- * protected from remote access.
- */
- if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
- ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
- } else {
- mr = ib_get_dma_mr(ia->ri_pd, IB_ACCESS_LOCAL_WRITE);
- if (IS_ERR(mr)) {
- pr_err("%s: ib_get_dma_mr for failed with %lX\n",
- __func__, PTR_ERR(mr));
- return -ENOMEM;
- }
- ia->ri_dma_lkey = ia->ri_dma_mr->lkey;
- ia->ri_dma_mr = mr;
- }
-
return 0;
}
struct ib_device_attr *devattr = &ia->ri_devattr;
int depth, delta;
- /* Obtain an lkey to use for the regbufs, which are
- * protected from remote access.
- */
- ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
-
ia->ri_max_frmr_depth =
min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS,
devattr->max_fast_reg_page_list_len);
physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
struct rpcrdma_create_data_internal *cdata)
{
- struct ib_device_attr *devattr = &ia->ri_devattr;
struct ib_mr *mr;
/* Obtain an rkey to use for RPC data payloads.
__func__, PTR_ERR(mr));
return -ENOMEM;
}
- ia->ri_dma_mr = mr;
-
- /* Obtain an lkey to use for regbufs.
- */
- if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
- ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
- else
- ia->ri_dma_lkey = ia->ri_dma_mr->lkey;
+ ia->ri_dma_mr = mr;
return 0;
}
ctxt->direction = DMA_FROM_DEVICE;
ctxt->read_hdr = head;
pages_needed = min_t(int, pages_needed, xprt->sc_max_sge_rd);
- read = min_t(int, pages_needed << PAGE_SHIFT, rs_length);
+ read = min_t(int, (pages_needed << PAGE_SHIFT) - *page_offset,
+ rs_length);
for (pno = 0; pno < pages_needed; pno++) {
int len = min_t(int, rs_length, PAGE_SIZE - pg_off);
ctxt->direction = DMA_FROM_DEVICE;
ctxt->frmr = frmr;
pages_needed = min_t(int, pages_needed, xprt->sc_frmr_pg_list_len);
- read = min_t(int, pages_needed << PAGE_SHIFT, rs_length);
+ read = min_t(int, (pages_needed << PAGE_SHIFT) - *page_offset,
+ rs_length);
frmr->kva = page_address(rqstp->rq_arg.pages[pg_no]);
frmr->direction = DMA_FROM_DEVICE;
xprt_clear_connected(xprt);
- rpcrdma_buffer_destroy(&r_xprt->rx_buf);
rpcrdma_ep_destroy(&r_xprt->rx_ep, &r_xprt->rx_ia);
+ rpcrdma_buffer_destroy(&r_xprt->rx_buf);
rpcrdma_ia_close(&r_xprt->rx_ia);
xprt_rdma_free_addresses(xprt);
cancel_delayed_work_sync(&ep->rep_connect_worker);
- if (ia->ri_id->qp) {
+ if (ia->ri_id->qp)
rpcrdma_ep_disconnect(ep, ia);
+
+ rpcrdma_clean_cq(ep->rep_attr.recv_cq);
+ rpcrdma_clean_cq(ep->rep_attr.send_cq);
+
+ if (ia->ri_id->qp) {
rdma_destroy_qp(ia->ri_id);
ia->ri_id->qp = NULL;
}
- rpcrdma_clean_cq(ep->rep_attr.recv_cq);
rc = ib_destroy_cq(ep->rep_attr.recv_cq);
if (rc)
dprintk("RPC: %s: ib_destroy_cq returned %i\n",
__func__, rc);
- rpcrdma_clean_cq(ep->rep_attr.send_cq);
rc = ib_destroy_cq(ep->rep_attr.send_cq);
if (rc)
dprintk("RPC: %s: ib_destroy_cq returned %i\n",
goto out_free;
iov->length = size;
- iov->lkey = ia->ri_dma_lkey;
+ iov->lkey = ia->ri_pd->local_dma_lkey;
rb->rg_size = size;
rb->rg_owner = NULL;
return rb;
struct rdma_cm_id *ri_id;
struct ib_pd *ri_pd;
struct ib_mr *ri_dma_mr;
- u32 ri_dma_lkey;
struct completion ri_done;
int ri_async_rc;
unsigned int ri_max_frmr_depth;
xs_sock_reset_connection_flags(xprt);
/* Mark transport as closed and wake up all pending tasks */
xprt_disconnect_done(xprt);
- xprt_force_disconnect(xprt);
}
/**
*/
static void xs_destroy(struct rpc_xprt *xprt)
{
+ struct sock_xprt *transport = container_of(xprt,
+ struct sock_xprt, xprt);
dprintk("RPC: xs_destroy xprt %p\n", xprt);
+ cancel_delayed_work_sync(&transport->connect_worker);
xs_close(xprt);
xs_xprt_free(xprt);
module_put(THIS_MODULE);
static void xs_tcp_state_change(struct sock *sk)
{
struct rpc_xprt *xprt;
+ struct sock_xprt *transport;
read_lock_bh(&sk->sk_callback_lock);
if (!(xprt = xprt_from_sock(sk)))
sock_flag(sk, SOCK_ZAPPED),
sk->sk_shutdown);
+ transport = container_of(xprt, struct sock_xprt, xprt);
trace_rpc_socket_state_change(xprt, sk->sk_socket);
switch (sk->sk_state) {
case TCP_ESTABLISHED:
spin_lock(&xprt->transport_lock);
if (!xprt_test_and_set_connected(xprt)) {
- struct sock_xprt *transport = container_of(xprt,
- struct sock_xprt, xprt);
/* Reset TCP record info */
transport->tcp_offset = 0;
transport->tcp_flags =
TCP_RCV_COPY_FRAGHDR | TCP_RCV_COPY_XID;
xprt->connect_cookie++;
+ clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
+ xprt_clear_connecting(xprt);
xprt_wake_pending_tasks(xprt, -EAGAIN);
}
smp_mb__after_atomic();
break;
case TCP_CLOSE:
+ if (test_and_clear_bit(XPRT_SOCK_CONNECTING,
+ &transport->sock_state))
+ xprt_clear_connecting(xprt);
xs_sock_mark_closed(xprt);
}
out:
/* Tell the socket layer to start connecting... */
xprt->stat.connect_count++;
xprt->stat.connect_start = jiffies;
+ set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);
switch (ret) {
case 0:
case -EINPROGRESS:
case -EALREADY:
xprt_unlock_connect(xprt, transport);
- xprt_clear_connecting(xprt);
return;
case -EINVAL:
/* Happens, for instance, if the user specified a link
*err = -TIPC_ERR_NO_NAME;
if (skb_linearize(skb))
return false;
+ msg = buf_msg(skb);
if (msg_reroute_cnt(msg))
return false;
dnode = addr_domain(net, msg_lookup_scope(msg));
if (UNIXCB(skb).fp)
scm.fp = scm_fp_dup(UNIXCB(skb).fp);
- sk_peek_offset_fwd(sk, chunk);
+ if (skip) {
+ sk_peek_offset_fwd(sk, chunk);
+ skip -= chunk;
+ }
+ if (UNIXCB(skb).fp)
+ break;
+
+ last = skb;
+ last_len = skb->len;
+ unix_state_lock(sk);
+ skb = skb_peek_next(skb, &sk->sk_receive_queue);
+ if (skb)
+ goto again;
+ unix_state_unlock(sk);
break;
}
} while (size);
/*
* Here's a sample kernel module showing the use of jprobes to dump
- * the arguments of do_fork().
+ * the arguments of _do_fork().
*
* For more information on theory of operation of jprobes, see
* Documentation/kprobes.txt
*
* Build and insert the kernel module as done in the kprobe example.
* You will see the trace data in /var/log/messages and on the
- * console whenever do_fork() is invoked to create a new process.
+ * console whenever _do_fork() is invoked to create a new process.
* (Some messages may be suppressed if syslogd is configured to
* eliminate duplicate messages.)
*/
#include <linux/kprobes.h>
/*
- * Jumper probe for do_fork.
+ * Jumper probe for _do_fork.
* Mirror principle enables access to arguments of the probed routine
* from the probe handler.
*/
-/* Proxy routine having the same arguments as actual do_fork() routine */
-static long jdo_fork(unsigned long clone_flags, unsigned long stack_start,
+/* Proxy routine having the same arguments as actual _do_fork() routine */
+static long j_do_fork(unsigned long clone_flags, unsigned long stack_start,
unsigned long stack_size, int __user *parent_tidptr,
int __user *child_tidptr)
{
}
static struct jprobe my_jprobe = {
- .entry = jdo_fork,
+ .entry = j_do_fork,
.kp = {
- .symbol_name = "do_fork",
+ .symbol_name = "_do_fork",
},
};
/*
* NOTE: This example is works on x86 and powerpc.
* Here's a sample kernel module showing the use of kprobes to dump a
- * stack trace and selected registers when do_fork() is called.
+ * stack trace and selected registers when _do_fork() is called.
*
* For more information on theory of operation of kprobes, see
* Documentation/kprobes.txt
*
* You will see the trace data in /var/log/messages and on the console
- * whenever do_fork() is invoked to create a new process.
+ * whenever _do_fork() is invoked to create a new process.
*/
#include <linux/kernel.h>
/* For each probe you need to allocate a kprobe structure */
static struct kprobe kp = {
- .symbol_name = "do_fork",
+ .symbol_name = "_do_fork",
};
/* kprobe pre_handler: called just before the probed instruction is executed */
*
* usage: insmod kretprobe_example.ko func=<func_name>
*
- * If no func_name is specified, do_fork is instrumented
+ * If no func_name is specified, _do_fork is instrumented
*
* For more information on theory of operation of kretprobes, see
* Documentation/kprobes.txt
#include <linux/limits.h>
#include <linux/sched.h>
-static char func_name[NAME_MAX] = "do_fork";
+static char func_name[NAME_MAX] = "_do_fork";
module_param_string(func, func_name, NAME_MAX, S_IRUGO);
MODULE_PARM_DESC(func, "Function to kretprobe; this module will report the"
" function's execution time");
/* Extract X.509 certificate in DER form from PKCS#11 or PEM.
*
- * Copyright © 2014 Red Hat, Inc. All Rights Reserved.
- * Copyright © 2015 Intel Corporation.
+ * Copyright © 2014-2015 Red Hat, Inc. All Rights Reserved.
+ * Copyright © 2015 Intel Corporation.
*
* Authors: David Howells <dhowells@redhat.com>
* David Woodhouse <dwmw2@infradead.org>
*
* This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public Licence
- * as published by the Free Software Foundation; either version
- * 2 of the Licence, or (at your option) any later version.
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the licence, or (at your option) any later version.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
-#include <getopt.h>
#include <err.h>
-#include <arpa/inet.h>
#include <openssl/bio.h>
-#include <openssl/evp.h>
#include <openssl/pem.h>
-#include <openssl/pkcs7.h>
#include <openssl/err.h>
#include <openssl/engine.h>
/* Sign a module file using the given key.
*
- * Copyright (C) 2014 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
+ * Copyright © 2014-2015 Red Hat, Inc. All Rights Reserved.
+ * Copyright © 2015 Intel Corporation.
+ *
+ * Authors: David Howells <dhowells@redhat.com>
+ * David Woodhouse <dwmw2@infradead.org>
*
* This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public Licence
- * as published by the Free Software Foundation; either version
- * 2 of the Licence, or (at your option) any later version.
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the licence, or (at your option) any later version.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <getopt.h>
#include <err.h>
#include <arpa/inet.h>
+#include <openssl/opensslv.h>
#include <openssl/bio.h>
#include <openssl/evp.h>
#include <openssl/pem.h>
-#include <openssl/cms.h>
#include <openssl/err.h>
#include <openssl/engine.h>
+/*
+ * Use CMS if we have openssl-1.0.0 or newer available - otherwise we have to
+ * assume that it's not available and its header file is missing and that we
+ * should use PKCS#7 instead. Switching to the older PKCS#7 format restricts
+ * the options we have on specifying the X.509 certificate we want.
+ *
+ * Further, older versions of OpenSSL don't support manually adding signers to
+ * the PKCS#7 message so have to accept that we get a certificate included in
+ * the signature message. Nor do such older versions of OpenSSL support
+ * signing with anything other than SHA1 - so we're stuck with that if such is
+ * the case.
+ */
+#if OPENSSL_VERSION_NUMBER < 0x10000000L
+#define USE_PKCS7
+#endif
+#ifndef USE_PKCS7
+#include <openssl/cms.h>
+#else
+#include <openssl/pkcs7.h>
+#endif
+
struct module_signature {
uint8_t algo; /* Public-key crypto algorithm [0] */
uint8_t hash; /* Digest algorithm [0] */
struct module_signature sig_info = { .id_type = PKEY_ID_PKCS7 };
char *hash_algo = NULL;
char *private_key_name, *x509_name, *module_name, *dest_name;
- bool save_cms = false, replace_orig;
+ bool save_sig = false, replace_orig;
bool sign_only = false;
unsigned char buf[4096];
- unsigned long module_size, cms_size;
- unsigned int use_keyid = 0, use_signed_attrs = CMS_NOATTR;
+ unsigned long module_size, sig_size;
+ unsigned int use_signed_attrs;
const EVP_MD *digest_algo;
EVP_PKEY *private_key;
+#ifndef USE_PKCS7
CMS_ContentInfo *cms;
+ unsigned int use_keyid = 0;
+#else
+ PKCS7 *pkcs7;
+#endif
X509 *x509;
BIO *b, *bd = NULL, *bm;
int opt, n;
-
OpenSSL_add_all_algorithms();
ERR_load_crypto_strings();
ERR_clear_error();
key_pass = getenv("KBUILD_SIGN_PIN");
+#ifndef USE_PKCS7
+ use_signed_attrs = CMS_NOATTR;
+#else
+ use_signed_attrs = PKCS7_NOATTR;
+#endif
+
do {
opt = getopt(argc, argv, "dpk");
switch (opt) {
- case 'p': save_cms = true; break;
- case 'd': sign_only = true; save_cms = true; break;
+ case 'p': save_sig = true; break;
+ case 'd': sign_only = true; save_sig = true; break;
+#ifndef USE_PKCS7
case 'k': use_keyid = CMS_USE_KEYID; break;
+#endif
case -1: break;
default: format();
}
replace_orig = true;
}
+#ifdef USE_PKCS7
+ if (strcmp(hash_algo, "sha1") != 0) {
+ fprintf(stderr, "sign-file: %s only supports SHA1 signing\n",
+ OPENSSL_VERSION_TEXT);
+ exit(3);
+ }
+#endif
+
/* Read the private key and the X.509 cert the PKCS#7 message
* will point to.
*/
bm = BIO_new_file(module_name, "rb");
ERR(!bm, "%s", module_name);
- /* Load the CMS message from the digest buffer. */
+#ifndef USE_PKCS7
+ /* Load the signature message from the digest buffer. */
cms = CMS_sign(NULL, NULL, NULL, NULL,
CMS_NOCERTS | CMS_PARTIAL | CMS_BINARY | CMS_DETACHED | CMS_STREAM);
ERR(!cms, "CMS_sign");
ERR(!CMS_add1_signer(cms, x509, private_key, digest_algo,
CMS_NOCERTS | CMS_BINARY | CMS_NOSMIMECAP |
use_keyid | use_signed_attrs),
- "CMS_sign_add_signer");
+ "CMS_add1_signer");
ERR(CMS_final(cms, bm, NULL, CMS_NOCERTS | CMS_BINARY) < 0,
"CMS_final");
- if (save_cms) {
- char *cms_name;
+#else
+ pkcs7 = PKCS7_sign(x509, private_key, NULL, bm,
+ PKCS7_NOCERTS | PKCS7_BINARY |
+ PKCS7_DETACHED | use_signed_attrs);
+ ERR(!pkcs7, "PKCS7_sign");
+#endif
+
+ if (save_sig) {
+ char *sig_file_name;
- ERR(asprintf(&cms_name, "%s.p7s", module_name) < 0, "asprintf");
- b = BIO_new_file(cms_name, "wb");
- ERR(!b, "%s", cms_name);
- ERR(i2d_CMS_bio_stream(b, cms, NULL, 0) < 0, "%s", cms_name);
+ ERR(asprintf(&sig_file_name, "%s.p7s", module_name) < 0,
+ "asprintf");
+ b = BIO_new_file(sig_file_name, "wb");
+ ERR(!b, "%s", sig_file_name);
+#ifndef USE_PKCS7
+ ERR(i2d_CMS_bio_stream(b, cms, NULL, 0) < 0,
+ "%s", sig_file_name);
+#else
+ ERR(i2d_PKCS7_bio(b, pkcs7) < 0,
+ "%s", sig_file_name);
+#endif
BIO_free(b);
}
ERR(n < 0, "%s", module_name);
module_size = BIO_number_written(bd);
+#ifndef USE_PKCS7
ERR(i2d_CMS_bio_stream(bd, cms, NULL, 0) < 0, "%s", dest_name);
- cms_size = BIO_number_written(bd) - module_size;
- sig_info.sig_len = htonl(cms_size);
+#else
+ ERR(i2d_PKCS7_bio(bd, pkcs7) < 0, "%s", dest_name);
+#endif
+ sig_size = BIO_number_written(bd) - module_size;
+ sig_info.sig_len = htonl(sig_size);
ERR(BIO_write(bd, &sig_info, sizeof(sig_info)) < 0, "%s", dest_name);
ERR(BIO_write(bd, magic_number, sizeof(magic_number) - 1) < 0, "%s", dest_name);
bool match = false;
RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&
- lockdep_is_held(&devcgroup_mutex),
+ !lockdep_is_held(&devcgroup_mutex),
"device_cgroup:verify_new_ex called without proper synchronization");
if (dev_cgroup->behavior == DEVCG_DEFAULT_ALLOW) {
kdebug("- %u", key->serial);
key_check(key);
+ /* Throw away the key data */
+ if (key->type->destroy)
+ key->type->destroy(key);
+
security_key_free(key);
/* deal with the user's key tracking and quota */
if (test_bit(KEY_FLAG_INSTANTIATED, &key->flags))
atomic_dec(&key->user->nikeys);
- /* now throw away the key memory */
- if (key->type->destroy)
- key->type->destroy(key);
-
key_user_put(key->user);
kfree(key->description);
Drivers that are implemented on ASoC can be found in
"ALSA for SoC audio support" section.
+config SND_PXA2XX_LIB
+ tristate
+ select SND_AC97_CODEC if SND_PXA2XX_LIB_AC97
+ select SND_DMAENGINE_PCM
+
+config SND_PXA2XX_LIB_AC97
+ bool
+
if SND_ARM
config SND_ARMAACI
tristate
select SND_PCM
-config SND_PXA2XX_LIB
- tristate
- select SND_AC97_CODEC if SND_PXA2XX_LIB_AC97
-
-config SND_PXA2XX_LIB_AC97
- bool
-
config SND_PXA2XX_AC97
tristate "AC97 driver for the Intel PXA2xx chip"
depends on ARCH_PXA
unsigned int snoop:1;
unsigned int align_buffer_size:1;
unsigned int region_requested:1;
- unsigned int disabled:1; /* disabled by VGA-switcher */
+ unsigned int disabled:1; /* disabled by vga_switcheroo */
#ifdef CONFIG_SND_HDA_DSP_LOADER
struct azx_dev saved_azx_dev;
AZX_DCAPS_4K_BDLE_BOUNDARY | AZX_DCAPS_SNOOP_OFF)
/*
- * VGA-switcher support
+ * vga_switcheroo support
*/
#ifdef SUPPORT_VGA_SWITCHEROO
#define use_vga_switcheroo(chip) ((chip)->use_vga_switcheroo)
}
}
} else {
- dev_info(chip->card->dev, "%s via VGA-switcheroo\n",
+ dev_info(chip->card->dev, "%s via vga_switcheroo\n",
disabled ? "Disabling" : "Enabling");
if (disabled) {
pm_runtime_put_sync_suspend(card->dev);
azx_suspend(card->dev);
- /* when we get suspended by vga switcheroo we end up in D3cold,
+ /* when we get suspended by vga_switcheroo we end up in D3cold,
* however we have no ACPI handle, so pci/acpi can't put us there,
* put ourselves there */
pci->current_state = PCI_D3cold;
struct pci_dev *p = get_bound_vga(chip->pci);
if (p) {
dev_info(chip->card->dev,
- "Handle VGA-switcheroo audio client\n");
+ "Handle vga_switcheroo audio client\n");
hda->use_vga_switcheroo = 1;
pci_dev_put(p);
}
* is there any machine with two switchable HDMI audio controllers?
*/
err = vga_switcheroo_register_audio_client(chip->pci, &azx_vs_ops,
- VGA_SWITCHEROO_DIS,
- hda->probe_continued);
+ VGA_SWITCHEROO_DIS);
if (err < 0)
return err;
hda->vga_switcheroo_registered = 1;
#ifdef SUPPORT_VGA_SWITCHEROO
/*
- * Check of disabled HDMI controller by vga-switcheroo
+ * Check of disabled HDMI controller by vga_switcheroo
*/
static struct pci_dev *get_bound_vga(struct pci_dev *pci)
{
err = register_vga_switcheroo(chip);
if (err < 0) {
- dev_err(card->dev, "Error registering VGA-switcheroo client\n");
+ dev_err(card->dev, "Error registering vga_switcheroo client\n");
goto out_free;
}
unsigned int irq_pending_warned:1;
unsigned int probe_continued:1;
- /* VGA-switcheroo setup */
+ /* vga_switcheroo setup */
unsigned int use_vga_switcheroo:1;
unsigned int vga_switcheroo_registered:1;
unsigned int init_failed:1; /* delayed init failed */
struct clk *hda2codec_2x_clk;
struct clk *hda2hdmi_clk;
void __iomem *regs;
+ struct work_struct probe_work;
};
#ifdef CONFIG_PM
static int hda_tegra_dev_free(struct snd_device *device)
{
struct azx *chip = device->device_data;
+ struct hda_tegra *hda = container_of(chip, struct hda_tegra, chip);
+ cancel_work_sync(&hda->probe_work);
if (azx_bus(chip)->chip_init) {
azx_stop_all_streams(chip);
azx_stop_chip(chip);
/*
* constructor
*/
+
+static void hda_tegra_probe_work(struct work_struct *work);
+
static int hda_tegra_create(struct snd_card *card,
unsigned int driver_caps,
struct hda_tegra *hda)
chip->single_cmd = false;
chip->snoop = true;
+ INIT_WORK(&hda->probe_work, hda_tegra_probe_work);
+
err = azx_bus_init(chip, NULL, &hda_tegra_io_ops);
if (err < 0)
return err;
card->private_data = chip;
dev_set_drvdata(&pdev->dev, card);
+ schedule_work(&hda->probe_work);
+
+ return 0;
+
+out_free:
+ snd_card_free(card);
+ return err;
+}
+
+static void hda_tegra_probe_work(struct work_struct *work)
+{
+ struct hda_tegra *hda = container_of(work, struct hda_tegra, probe_work);
+ struct azx *chip = &hda->chip;
+ struct platform_device *pdev = to_platform_device(hda->dev);
+ int err;
err = hda_tegra_first_init(chip, pdev);
if (err < 0)
chip->running = 1;
snd_hda_set_power_save(&chip->bus, power_save * 1000);
- return 0;
-
-out_free:
- snd_card_free(card);
- return err;
+ out_free:
+ return; /* no error return from async probe */
}
static int hda_tegra_remove(struct platform_device *pdev)
SND_PCI_QUIRK(0x106b, 0x5e00, "MacBookPro 11,2", CS4208_MBP11),
SND_PCI_QUIRK(0x106b, 0x7100, "MacBookAir 6,1", CS4208_MBA6),
SND_PCI_QUIRK(0x106b, 0x7200, "MacBookAir 6,2", CS4208_MBA6),
+ SND_PCI_QUIRK(0x106b, 0x7b00, "MacBookPro 12,1", CS4208_MBP11),
{} /* terminator */
};
return non_pcm;
}
+/* There is a fixed mapping between audio pin node and display port
+ * on current Intel platforms:
+ * Pin Widget 5 - PORT B (port = 1 in i915 driver)
+ * Pin Widget 6 - PORT C (port = 2 in i915 driver)
+ * Pin Widget 7 - PORT D (port = 3 in i915 driver)
+ */
+static int intel_pin2port(hda_nid_t pin_nid)
+{
+ return pin_nid - 4;
+}
/*
* HDMI callbacks
int pin_idx = hinfo_to_pin_index(codec, hinfo);
struct hdmi_spec_per_pin *per_pin = get_pin(spec, pin_idx);
hda_nid_t pin_nid = per_pin->pin_nid;
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct i915_audio_component *acomp = codec->bus->core.audio_component;
bool non_pcm;
int pinctl;
intel_not_share_assigned_cvt(codec, pin_nid, per_pin->mux_idx);
}
+ /* Call sync_audio_rate to set the N/CTS/M manually if necessary */
+ /* Todo: add DP1.2 MST audio support later */
+ if (acomp && acomp->ops && acomp->ops->sync_audio_rate)
+ acomp->ops->sync_audio_rate(acomp->dev,
+ intel_pin2port(pin_nid),
+ runtime->rate);
+
non_pcm = check_non_pcm_per_cvt(codec, cvt_nid);
mutex_lock(&per_pin->lock);
per_pin->channels = substream->runtime->channels;
}
}
+/* fixup for Thinkpad docks: add dock pins, avoid HP parser fixup */
+static void alc_fixup_tpt440_dock(struct hda_codec *codec,
+ const struct hda_fixup *fix, int action)
+{
+ static const struct hda_pintbl pincfgs[] = {
+ { 0x16, 0x21211010 }, /* dock headphone */
+ { 0x19, 0x21a11010 }, /* dock mic */
+ { }
+ };
+ struct alc_spec *spec = codec->spec;
+
+ if (action == HDA_FIXUP_ACT_PRE_PROBE) {
+ spec->parse_flags = HDA_PINCFG_NO_HP_FIXUP;
+ codec->power_save_node = 0; /* avoid click noises */
+ snd_hda_apply_pincfgs(codec, pincfgs);
+ }
+}
+
static void alc_shutup_dell_xps13(struct hda_codec *codec)
{
struct alc_spec *spec = codec->spec;
ALC255_FIXUP_HEADSET_MODE_NO_HP_MIC,
ALC293_FIXUP_DELL1_MIC_NO_PRESENCE,
ALC292_FIXUP_TPT440_DOCK,
- ALC292_FIXUP_TPT440_DOCK2,
ALC283_FIXUP_BXBT2807_MIC,
ALC255_FIXUP_DELL_WMI_MIC_MUTE_LED,
ALC282_FIXUP_ASPIRE_V5_PINS,
},
[ALC292_FIXUP_TPT440_DOCK] = {
.type = HDA_FIXUP_FUNC,
- .v.func = alc269_fixup_pincfg_no_hp_to_lineout,
- .chained = true,
- .chain_id = ALC292_FIXUP_TPT440_DOCK2
- },
- [ALC292_FIXUP_TPT440_DOCK2] = {
- .type = HDA_FIXUP_PINS,
- .v.pins = (const struct hda_pintbl[]) {
- { 0x16, 0x21211010 }, /* dock headphone */
- { 0x19, 0x21a11010 }, /* dock mic */
- { }
- },
+ .v.func = alc_fixup_tpt440_dock,
.chained = true,
.chain_id = ALC269_FIXUP_LIMIT_INT_MIC_BOOST
},
SND_PCI_QUIRK(0x17aa, 0x2212, "Thinkpad T440", ALC292_FIXUP_TPT440_DOCK),
SND_PCI_QUIRK(0x17aa, 0x2214, "Thinkpad X240", ALC292_FIXUP_TPT440_DOCK),
SND_PCI_QUIRK(0x17aa, 0x2215, "Thinkpad", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
+ SND_PCI_QUIRK(0x17aa, 0x2223, "ThinkPad T550", ALC292_FIXUP_TPT440_DOCK),
SND_PCI_QUIRK(0x17aa, 0x2226, "ThinkPad X250", ALC292_FIXUP_TPT440_DOCK),
SND_PCI_QUIRK(0x17aa, 0x3977, "IdeaPad S210", ALC283_FIXUP_INT_MIC),
SND_PCI_QUIRK(0x17aa, 0x3978, "IdeaPad Y410P", ALC269_FIXUP_NO_SHUTUP),
return err;
spec = codec->spec;
- codec->power_save_node = 1;
+ /* enable power_save_node only for new 92HD89xx chips, as it causes
+ * click noises on old 92HD73xx chips.
+ */
+ if ((codec->core.vendor_id & 0xfffffff0) != 0x111d7670)
+ codec->power_save_node = 1;
spec->linear_tone_beep = 0;
spec->gen.mixer_nid = 0x1d;
spec->have_spdif_mux = 1;
.cpu_dai_name = "au1xpsc_i2s.2",
.platform_name = "au1xpsc-pcm.2",
.codec_name = "wm8731.0-001b",
+ .dai_fmt = SND_SOC_DAIFMT_LEFT_J | SND_SOC_DAIFMT_NB_NF |
+ SND_SOC_DAIFMT_CBM_CFM,
.ops = &db1200_i2s_wm8731_ops,
};
.cpu_dai_name = "au1xpsc_i2s.3",
.platform_name = "au1xpsc-pcm.3",
.codec_name = "wm8731.0-001b",
+ .dai_fmt = SND_SOC_DAIFMT_LEFT_J | SND_SOC_DAIFMT_NB_NF |
+ SND_SOC_DAIFMT_CBM_CFM,
.ops = &db1200_i2s_wm8731_ops,
};
{
struct resource *iores, *dmares;
unsigned long sel;
- int ret;
struct au1xpsc_audio_data *wd;
wd = devm_kzalloc(&pdev->dev, sizeof(struct au1xpsc_audio_data),
RT5645_L_VOL_SFT + 1, RT5645_R_VOL_SFT + 1, 63, 0, adc_vol_tlv),
/* ADC Boost Volume Control */
- SOC_DOUBLE_TLV("STO1 ADC Boost Gain", RT5645_ADC_BST_VOL1,
+ SOC_DOUBLE_TLV("ADC Boost Capture Volume", RT5645_ADC_BST_VOL1,
RT5645_STO1_ADC_L_BST_SFT, RT5645_STO1_ADC_R_BST_SFT, 3, 0,
adc_bst_tlv),
- SOC_DOUBLE_TLV("STO2 ADC Boost Gain", RT5645_ADC_BST_VOL1,
- RT5645_STO2_ADC_L_BST_SFT, RT5645_STO2_ADC_R_BST_SFT, 3, 0,
+ SOC_DOUBLE_TLV("Mono ADC Boost Capture Volume", RT5645_ADC_BST_VOL2,
+ RT5645_MONO_ADC_L_BST_SFT, RT5645_MONO_ADC_R_BST_SFT, 3, 0,
adc_bst_tlv),
/* I2S2 function select */
static const struct snd_kcontrol_new rt5645_dac_l_mix[] = {
SOC_DAPM_SINGLE("Stereo ADC Switch", RT5645_AD_DA_MIXER,
RT5645_M_ADCMIX_L_SFT, 1, 1),
- SOC_DAPM_SINGLE("DAC1 Switch", RT5645_AD_DA_MIXER,
+ SOC_DAPM_SINGLE_AUTODISABLE("DAC1 Switch", RT5645_AD_DA_MIXER,
RT5645_M_DAC1_L_SFT, 1, 1),
};
static const struct snd_kcontrol_new rt5645_dac_r_mix[] = {
SOC_DAPM_SINGLE("Stereo ADC Switch", RT5645_AD_DA_MIXER,
RT5645_M_ADCMIX_R_SFT, 1, 1),
- SOC_DAPM_SINGLE("DAC1 Switch", RT5645_AD_DA_MIXER,
+ SOC_DAPM_SINGLE_AUTODISABLE("DAC1 Switch", RT5645_AD_DA_MIXER,
RT5645_M_DAC1_R_SFT, 1, 1),
};
regmap_write(rt5645->regmap, RT5645_PR_BASE +
RT5645_MAMP_INT_REG2, 0xfc00);
snd_soc_write(codec, RT5645_DEPOP_M2, 0x1140);
- mdelay(5);
+ msleep(40);
rt5645->hp_on = true;
} else {
/* depop parameters */
snd_soc_dapm_sync(dapm);
rt5645->jack_type = SND_JACK_HEADPHONE;
}
-
- snd_soc_update_bits(codec, RT5645_CHARGE_PUMP, 0x0300, 0x0200);
- snd_soc_write(codec, RT5645_DEPOP_M1, 0x001d);
- snd_soc_write(codec, RT5645_DEPOP_M1, 0x0001);
} else { /* jack out */
rt5645->jack_type = 0;
+ regmap_update_bits(rt5645->regmap, RT5645_HP_VOL,
+ RT5645_L_MUTE | RT5645_R_MUTE,
+ RT5645_L_MUTE | RT5645_R_MUTE);
regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL2,
RT5645_CBJ_MN_JD, RT5645_CBJ_MN_JD);
regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL1,
rt5645->en_button_func = true;
regmap_update_bits(rt5645->regmap, RT5645_GPIO_CTRL1,
RT5645_GP1_PIN_IRQ, RT5645_GP1_PIN_IRQ);
- regmap_update_bits(rt5645->regmap, RT5645_DEPOP_M1,
- RT5645_HP_CB_MASK, RT5645_HP_CB_PU);
regmap_update_bits(rt5645->regmap, RT5645_GEN_CTRL1,
RT5645_DIG_GATE_CTRL, RT5645_DIG_GATE_CTRL);
}
DMI_MATCH(DMI_PRODUCT_NAME, "Celes"),
},
},
+ {
+ .ident = "Google Ultima",
+ .callback = strago_quirk_cb,
+ .matches = {
+ DMI_MATCH(DMI_PRODUCT_NAME, "Ultima"),
+ },
+ },
{ }
};
#define RT5645_STO1_ADC_DIG_VOL 0x1c
#define RT5645_MONO_ADC_DIG_VOL 0x1d
#define RT5645_ADC_BST_VOL1 0x1e
-/* Mixer - D-D */
#define RT5645_ADC_BST_VOL2 0x20
+/* Mixer - D-D */
#define RT5645_STO1_ADC_MIXER 0x27
#define RT5645_MONO_ADC_MIXER 0x28
#define RT5645_AD_DA_MIXER 0x29
#define RT5645_STO1_ADC_R_BST_SFT 12
#define RT5645_STO1_ADC_COMP_MASK (0x3 << 10)
#define RT5645_STO1_ADC_COMP_SFT 10
-#define RT5645_STO2_ADC_L_BST_MASK (0x3 << 8)
-#define RT5645_STO2_ADC_L_BST_SFT 8
-#define RT5645_STO2_ADC_R_BST_MASK (0x3 << 6)
-#define RT5645_STO2_ADC_R_BST_SFT 6
-#define RT5645_STO2_ADC_COMP_MASK (0x3 << 4)
-#define RT5645_STO2_ADC_COMP_SFT 4
+
+/* ADC Boost Volume Control (0x20) */
+#define RT5645_MONO_ADC_L_BST_MASK (0x3 << 14)
+#define RT5645_MONO_ADC_L_BST_SFT 14
+#define RT5645_MONO_ADC_R_BST_MASK (0x3 << 12)
+#define RT5645_MONO_ADC_R_BST_SFT 12
+#define RT5645_MONO_ADC_COMP_MASK (0x3 << 10)
+#define RT5645_MONO_ADC_COMP_SFT 10
/* Stereo2 ADC Mixer Control (0x26) */
#define RT5645_STO2_ADC_SRC_MASK (0x1 << 15)
sgtl5000->micbias_resistor << SGTL5000_BIAS_R_SHIFT);
snd_soc_update_bits(codec, SGTL5000_CHIP_MIC_CTRL,
- SGTL5000_BIAS_R_MASK,
- sgtl5000->micbias_voltage << SGTL5000_BIAS_R_SHIFT);
+ SGTL5000_BIAS_VOLT_MASK,
+ sgtl5000->micbias_voltage << SGTL5000_BIAS_VOLT_SHIFT);
/*
* disable DAP
* TODO:
else {
sgtl5000->micbias_voltage = 0;
dev_err(&client->dev,
- "Unsuitable MicBias resistor\n");
+ "Unsuitable MicBias voltage\n");
}
} else {
sgtl5000->micbias_voltage = 0;
/*
* DAC digital volumes. From -7 to 24 dB in 1 dB steps
*/
-static DECLARE_TLV_DB_SCALE(dac_tlv, -7, 100, 0);
+static DECLARE_TLV_DB_SCALE(dac_tlv, -700, 100, 0);
static const char * const tas2552_din_source_select[] = {
"Muted",
snd_soc_write(codec, PGAL_2_LLOPM_VOL, DEFAULT_VOL);
snd_soc_write(codec, PGAR_2_RLOPM_VOL, DEFAULT_VOL);
- /* Line2 to HP Bypass default volume, disconnect from Output Mixer */
- snd_soc_write(codec, LINE2L_2_HPLOUT_VOL, DEFAULT_VOL);
- snd_soc_write(codec, LINE2R_2_HPROUT_VOL, DEFAULT_VOL);
- snd_soc_write(codec, LINE2L_2_HPLCOM_VOL, DEFAULT_VOL);
- snd_soc_write(codec, LINE2R_2_HPRCOM_VOL, DEFAULT_VOL);
- /* Line2 Line Out default volume, disconnect from Output Mixer */
- snd_soc_write(codec, LINE2L_2_LLOPM_VOL, DEFAULT_VOL);
- snd_soc_write(codec, LINE2R_2_RLOPM_VOL, DEFAULT_VOL);
+ /* On tlv320aic3104, these registers are reserved and must not be written */
+ if (aic3x->model != AIC3X_MODEL_3104) {
+ /* Line2 to HP Bypass default volume, disconnect from Output Mixer */
+ snd_soc_write(codec, LINE2L_2_HPLOUT_VOL, DEFAULT_VOL);
+ snd_soc_write(codec, LINE2R_2_HPROUT_VOL, DEFAULT_VOL);
+ snd_soc_write(codec, LINE2L_2_HPLCOM_VOL, DEFAULT_VOL);
+ snd_soc_write(codec, LINE2R_2_HPRCOM_VOL, DEFAULT_VOL);
+ /* Line2 Line Out default volume, disconnect from Output Mixer */
+ snd_soc_write(codec, LINE2L_2_LLOPM_VOL, DEFAULT_VOL);
+ snd_soc_write(codec, LINE2R_2_RLOPM_VOL, DEFAULT_VOL);
+ }
switch (aic3x->model) {
case AIC3X_MODEL_3X:
struct wm0010_priv *wm0010 = snd_soc_codec_get_drvdata(codec);
unsigned long flags;
int ret;
- const struct firmware *fw;
struct spi_message m;
struct spi_transfer t;
struct dfw_pllrec pll_rec;
wm0010->state = WM0010_OUT_OF_RESET;
spin_unlock_irqrestore(&wm0010->irq_lock, flags);
- /* First the bootloader */
- ret = request_firmware(&fw, "wm0010_stage2.bin", codec->dev);
- if (ret != 0) {
- dev_err(codec->dev, "Failed to request stage2 loader: %d\n",
- ret);
- goto abort;
- }
-
if (!wait_for_completion_timeout(&wm0010->boot_completion,
msecs_to_jiffies(20)))
dev_err(codec->dev, "Failed to get interrupt from DSP\n");
img_swap = kzalloc(len, GFP_KERNEL | GFP_DMA);
if (!img_swap)
- goto abort;
+ goto abort_out;
/* We need to re-order for 0010 */
byte_swap_64((u64 *)&pll_rec, img_swap, len);
spi_message_add_tail(&t, &m);
ret = spi_sync(spi, &m);
- if (ret != 0) {
+ if (ret) {
dev_err(codec->dev, "First PLL write failed: %d\n", ret);
- goto abort;
+ goto abort_swap;
}
/* Use a second send of the message to get the return status */
ret = spi_sync(spi, &m);
- if (ret != 0) {
+ if (ret) {
dev_err(codec->dev, "Second PLL write failed: %d\n", ret);
- goto abort;
+ goto abort_swap;
}
p = (u32 *)out;
return 0;
+abort_swap:
+ kfree(img_swap);
+abort_out:
+ kfree(out);
abort:
/* Put the chip back into reset */
wm0010_halt(codec);
return wm8960_set_deemph(codec);
}
-static const DECLARE_TLV_DB_SCALE(adc_tlv, -9700, 50, 0);
-static const DECLARE_TLV_DB_SCALE(dac_tlv, -12700, 50, 1);
+static const DECLARE_TLV_DB_SCALE(adc_tlv, -9750, 50, 1);
+static const DECLARE_TLV_DB_SCALE(inpga_tlv, -1725, 75, 0);
+static const DECLARE_TLV_DB_SCALE(dac_tlv, -12750, 50, 1);
static const DECLARE_TLV_DB_SCALE(bypass_tlv, -2100, 300, 0);
static const DECLARE_TLV_DB_SCALE(out_tlv, -12100, 100, 1);
-static const DECLARE_TLV_DB_SCALE(boost_tlv, -1200, 300, 1);
+static const DECLARE_TLV_DB_SCALE(lineinboost_tlv, -1500, 300, 1);
+static const unsigned int micboost_tlv[] = {
+ TLV_DB_RANGE_HEAD(2),
+ 0, 1, TLV_DB_SCALE_ITEM(0, 1300, 0),
+ 2, 3, TLV_DB_SCALE_ITEM(2000, 900, 0),
+};
static const struct snd_kcontrol_new wm8960_snd_controls[] = {
SOC_DOUBLE_R_TLV("Capture Volume", WM8960_LINVOL, WM8960_RINVOL,
- 0, 63, 0, adc_tlv),
+ 0, 63, 0, inpga_tlv),
SOC_DOUBLE_R("Capture Volume ZC Switch", WM8960_LINVOL, WM8960_RINVOL,
6, 1, 0),
SOC_DOUBLE_R("Capture Switch", WM8960_LINVOL, WM8960_RINVOL,
7, 1, 0),
SOC_SINGLE_TLV("Right Input Boost Mixer RINPUT3 Volume",
- WM8960_INBMIX1, 4, 7, 0, boost_tlv),
+ WM8960_INBMIX1, 4, 7, 0, lineinboost_tlv),
SOC_SINGLE_TLV("Right Input Boost Mixer RINPUT2 Volume",
- WM8960_INBMIX1, 1, 7, 0, boost_tlv),
+ WM8960_INBMIX1, 1, 7, 0, lineinboost_tlv),
SOC_SINGLE_TLV("Left Input Boost Mixer LINPUT3 Volume",
- WM8960_INBMIX2, 4, 7, 0, boost_tlv),
+ WM8960_INBMIX2, 4, 7, 0, lineinboost_tlv),
SOC_SINGLE_TLV("Left Input Boost Mixer LINPUT2 Volume",
- WM8960_INBMIX2, 1, 7, 0, boost_tlv),
+ WM8960_INBMIX2, 1, 7, 0, lineinboost_tlv),
+SOC_SINGLE_TLV("Right Input Boost Mixer RINPUT1 Volume",
+ WM8960_RINPATH, 4, 3, 0, micboost_tlv),
+SOC_SINGLE_TLV("Left Input Boost Mixer LINPUT1 Volume",
+ WM8960_LINPATH, 4, 3, 0, micboost_tlv),
SOC_DOUBLE_R_TLV("Playback Volume", WM8960_LDAC, WM8960_RDAC,
0, 255, 0, dac_tlv),
WM8962_DAC_MUTE, val);
}
-#define WM8962_RATES SNDRV_PCM_RATE_8000_96000
+#define WM8962_RATES (SNDRV_PCM_RATE_8000_48000 |\
+ SNDRV_PCM_RATE_88200 | SNDRV_PCM_RATE_96000)
#define WM8962_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE)
ret = snd_soc_register_codec(&i2c->dev,
&soc_codec_dev_wm8962, &wm8962_dai, 1);
if (ret < 0)
- goto err_enable;
+ goto err_pm_runtime;
regcache_cache_only(wm8962->regmap, true);
return 0;
+err_pm_runtime:
+ pm_runtime_disable(&i2c->dev);
err_enable:
regulator_bulk_disable(ARRAY_SIZE(wm8962->supplies), wm8962->supplies);
err:
static int wm8962_i2c_remove(struct i2c_client *client)
{
snd_soc_unregister_codec(&client->dev);
+ pm_runtime_disable(&client->dev);
return 0;
}
u8 rx_ser = 0;
u8 slots = mcasp->tdm_slots;
u8 max_active_serializers = (channels + slots - 1) / slots;
- int active_serializers, numevt, n;
+ int active_serializers, numevt;
u32 reg;
/* Default configuration */
if (mcasp->version < MCASP_VERSION_3)
* The number of words for numevt need to be in steps of active
* serializers.
*/
- n = numevt % active_serializers;
- if (n)
- numevt += (active_serializers - n);
+ numevt = (numevt / active_serializers) * active_serializers;
+
while (period_words % numevt && numevt > 0)
numevt -= active_serializers;
if (numevt <= 0)
.ops = &davinci_mcasp_dai_ops,
.symmetric_samplebits = 1,
+ .symmetric_rates = 1,
},
{
.name = "davinci-mcasp.1",
irq = platform_get_irq_byname(pdev, "common");
if (irq >= 0) {
- irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_common\n",
+ irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_common",
dev_name(&pdev->dev));
ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
davinci_mcasp_common_irq_handler,
irq = platform_get_irq_byname(pdev, "rx");
if (irq >= 0) {
- irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_rx\n",
+ irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_rx",
dev_name(&pdev->dev));
ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
davinci_mcasp_rx_irq_handler,
irq = platform_get_irq_byname(pdev, "tx");
if (irq >= 0) {
- irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_tx\n",
+ irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_tx",
dev_name(&pdev->dev));
ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
davinci_mcasp_tx_irq_handler,
if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
for (i = 0; i < 4; i++)
- i2s_write_reg(dev->i2s_base, TOR(i), 0);
+ i2s_read_reg(dev->i2s_base, TOR(i));
} else {
for (i = 0; i < 4; i++)
- i2s_write_reg(dev->i2s_base, ROR(i), 0);
+ i2s_read_reg(dev->i2s_base, ROR(i));
}
}
static void i2s_start(struct dw_i2s_dev *dev,
struct snd_pcm_substream *substream)
{
-
+ u32 i, irq;
i2s_write_reg(dev->i2s_base, IER, 1);
- if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+ for (i = 0; i < 4; i++) {
+ irq = i2s_read_reg(dev->i2s_base, IMR(i));
+ i2s_write_reg(dev->i2s_base, IMR(i), irq & ~0x30);
+ }
i2s_write_reg(dev->i2s_base, ITER, 1);
- else
+ } else {
+ for (i = 0; i < 4; i++) {
+ irq = i2s_read_reg(dev->i2s_base, IMR(i));
+ i2s_write_reg(dev->i2s_base, IMR(i), irq & ~0x03);
+ }
i2s_write_reg(dev->i2s_base, IRER, 1);
+ }
i2s_write_reg(dev->i2s_base, CER, 1);
}
priv->dai_fmt |= SND_SOC_DAIFMT_CBM_CFM;
} else {
dev_err(&pdev->dev, "unknown Device Tree compatible\n");
- return -EINVAL;
+ ret = -EINVAL;
+ goto asrc_fail;
}
/* Common settings for corresponding Freescale CPU DAI driver */
static bool fsl_ssi_is_ac97(struct fsl_ssi_private *ssi_private)
{
- return !!(ssi_private->dai_fmt & SND_SOC_DAIFMT_AC97);
+ return (ssi_private->dai_fmt & SND_SOC_DAIFMT_FORMAT_MASK) ==
+ SND_SOC_DAIFMT_AC97;
}
static bool fsl_ssi_is_i2s_master(struct fsl_ssi_private *ssi_private)
CCSR_SSI_SCR_TCH_EN);
}
- if (fmt & SND_SOC_DAIFMT_AC97)
+ if ((fmt & SND_SOC_DAIFMT_FORMAT_MASK) == SND_SOC_DAIFMT_AC97)
fsl_ssi_setup_ac97(ssi_private);
return 0;
switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
case SND_SOC_DAIFMT_I2S:
/* data on rising edge of bclk, frame low 1clk before data */
- strcr |= SSI_STCR_TFSI | SSI_STCR_TEFS | SSI_STCR_TXBIT0;
+ strcr |= SSI_STCR_TXBIT0 | SSI_STCR_TSCKP | SSI_STCR_TFSI |
+ SSI_STCR_TEFS;
scr |= SSI_SCR_NET;
if (ssi->flags & IMX_SSI_USE_I2S_SLAVE) {
scr &= ~SSI_I2S_MODE_MASK;
break;
case SND_SOC_DAIFMT_LEFT_J:
/* data on rising edge of bclk, frame high with data */
- strcr |= SSI_STCR_TXBIT0;
+ strcr |= SSI_STCR_TXBIT0 | SSI_STCR_TSCKP;
break;
case SND_SOC_DAIFMT_DSP_B:
/* data on rising edge of bclk, frame high with data */
- strcr |= SSI_STCR_TFSL | SSI_STCR_TXBIT0;
+ strcr |= SSI_STCR_TXBIT0 | SSI_STCR_TSCKP | SSI_STCR_TFSL;
break;
case SND_SOC_DAIFMT_DSP_A:
/* data on rising edge of bclk, frame high 1clk before data */
- strcr |= SSI_STCR_TFSL | SSI_STCR_TXBIT0 | SSI_STCR_TEFS;
+ strcr |= SSI_STCR_TXBIT0 | SSI_STCR_TSCKP | SSI_STCR_TFSL |
+ SSI_STCR_TEFS;
break;
}
/* DAI clock inversion */
switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
case SND_SOC_DAIFMT_IB_IF:
- strcr |= SSI_STCR_TFSI;
- strcr &= ~SSI_STCR_TSCKP;
+ strcr ^= SSI_STCR_TSCKP | SSI_STCR_TFSI;
break;
case SND_SOC_DAIFMT_IB_NF:
- strcr &= ~(SSI_STCR_TSCKP | SSI_STCR_TFSI);
+ strcr ^= SSI_STCR_TSCKP;
break;
case SND_SOC_DAIFMT_NB_IF:
- strcr |= SSI_STCR_TFSI | SSI_STCR_TSCKP;
+ strcr ^= SSI_STCR_TFSI;
break;
case SND_SOC_DAIFMT_NB_NF:
- strcr &= ~SSI_STCR_TFSI;
- strcr |= SSI_STCR_TSCKP;
break;
}
struct sst_hsw_ipc_dx_reply dx;
void *dx_context;
dma_addr_t dx_context_paddr;
+ enum sst_hsw_device_id dx_dev;
+ enum sst_hsw_device_mclk dx_mclk;
+ enum sst_hsw_device_mode dx_mode;
+ u32 dx_clock_divider;
/* boot */
wait_queue_head_t boot_wait;
trace_ipc_request("set device config", dev);
- config.ssp_interface = dev;
- config.clock_frequency = mclk;
- config.mode = mode;
- config.clock_divider = clock_divider;
+ hsw->dx_dev = config.ssp_interface = dev;
+ hsw->dx_mclk = config.clock_frequency = mclk;
+ hsw->dx_mode = config.mode = mode;
+ hsw->dx_clock_divider = config.clock_divider = clock_divider;
if (mode == SST_HSW_DEVICE_TDM_CLOCK_MASTER)
config.channels = 4;
else
return -EIO;
}
- /* Set ADSP SSP port settings */
- ret = sst_hsw_device_set_config(hsw, SST_HSW_DEVICE_SSP_0,
- SST_HSW_DEVICE_MCLK_FREQ_24_MHZ,
- SST_HSW_DEVICE_CLOCK_MASTER, 9);
+ /* Set ADSP SSP port settings - sadly the FW does not store SSP port
+ settings as part of the PM context. */
+ ret = sst_hsw_device_set_config(hsw, hsw->dx_dev, hsw->dx_mclk,
+ hsw->dx_mode, hsw->dx_clock_divider);
if (ret < 0)
dev_err(dev, "error: SSP re-initialization failed\n");
memif->substream = substream;
snd_soc_set_runtime_hwparams(substream, &mtk_afe_hardware);
+
+ /*
+ * Capture cannot use ping-pong buffer since hw_ptr at IRQ may be
+ * smaller than period_size due to AFE's internal buffer.
+ * This easily leads to overrun when avail_min is period_size.
+ * One more period can hold the possible unread buffer.
+ */
+ if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) {
+ ret = snd_pcm_hw_constraint_minmax(runtime,
+ SNDRV_PCM_HW_PARAM_PERIODS,
+ 3,
+ mtk_afe_hardware.periods_max);
+ if (ret < 0) {
+ dev_err(afe->dev, "hw_constraint_minmax failed\n");
+ return ret;
+ }
+ }
ret = snd_pcm_hw_constraint_integer(runtime,
SNDRV_PCM_HW_PARAM_PERIODS);
if (ret < 0)
config SND_PXA2XX_SOC
tristate "SoC Audio for the Intel PXA2xx chip"
depends on ARCH_PXA
- select SND_ARM
select SND_PXA2XX_LIB
help
Say Y or M if you want to add support for codecs attached to
config SND_PXA2XX_SOC_AC97
tristate
select AC97_BUS
- select SND_ARM
select SND_PXA2XX_LIB_AC97
select SND_SOC_AC97_BUS
.reset = pxa2xx_ac97_cold_reset,
};
-static unsigned long pxa2xx_ac97_pcm_stereo_in_req = 12;
+static unsigned long pxa2xx_ac97_pcm_stereo_in_req = 11;
static struct snd_dmaengine_dai_dma_data pxa2xx_ac97_pcm_stereo_in = {
.addr = __PREG(PCDR),
.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
.filter_data = &pxa2xx_ac97_pcm_stereo_in_req,
};
-static unsigned long pxa2xx_ac97_pcm_stereo_out_req = 11;
+static unsigned long pxa2xx_ac97_pcm_stereo_out_req = 12;
static struct snd_dmaengine_dai_dma_data pxa2xx_ac97_pcm_stereo_out = {
.addr = __PREG(PCDR),
.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
default:
WARN(1, "Unknown event %d\n", event);
- return -EINVAL;
+ ret = -EINVAL;
}
out:
SNDRV_PCM_FMTBIT_S32_LE | \
SNDRV_PCM_FMTBIT_U32_LE | \
SNDRV_PCM_FMTBIT_IEC958_SUBFRAME_LE)
+/*
+ * The dummy CODEC is only meant to be used in situations where there is no
+ * actual hardware.
+ *
+ * If there is actual hardware even if it does not have a control bus
+ * the hardware will still have constraints like supported samplerates, etc.
+ * which should be modelled. And the data flow graph also should be modelled
+ * using DAPM.
+ */
static struct snd_soc_dai_driver dummy_dai = {
.name = "snd-soc-dummy-dai",
.playback = {
config SND_SPEAR_SOC
tristate
- select SND_DMAENGINE_PCM
+ select SND_SOC_GENERIC_DMAENGINE_PCM
config SND_SPEAR_SPDIF_OUT
tristate
if (!info)
return -ENOMEM;
- of_property_read_u32(pnode, "version", &player->ver);
- if (player->ver == SND_ST_UNIPERIF_VERSION_UNKNOWN) {
+ if (of_property_read_u32(pnode, "version", &player->ver) ||
+ player->ver == SND_ST_UNIPERIF_VERSION_UNKNOWN) {
dev_err(dev, "Unknown uniperipheral version ");
return -EINVAL;
}
if (player->ver >= SND_ST_UNIPERIF_VERSION_UNI_PLR_TOP_1_0)
info->underflow_enabled = 1;
- of_property_read_u32(pnode, "uniperiph-id", &info->id);
+ if (of_property_read_u32(pnode, "uniperiph-id", &info->id)) {
+ dev_err(dev, "uniperipheral id not defined");
+ return -EINVAL;
+ }
/* Read the device mode property */
- of_property_read_string(pnode, "mode", &mode);
+ if (of_property_read_string(pnode, "mode", &mode)) {
+ dev_err(dev, "uniperipheral mode not defined");
+ return -EINVAL;
+ }
if (strcasecmp(mode, "hdmi") == 0)
info->player_type = SND_ST_UNIPERIF_PLAYER_TYPE_HDMI;
if (!info)
return -ENOMEM;
- of_property_read_u32(node, "version", &reader->ver);
+ if (of_property_read_u32(node, "version", &reader->ver) ||
+ reader->ver == SND_ST_UNIPERIF_VERSION_UNKNOWN) {
+ dev_err(&pdev->dev, "Unknown uniperipheral version ");
+ return -EINVAL;
+ }
/* Save the info structure */
reader->info = info;
struct snd_seq_oss_reg *arg;
struct snd_seq_device *dev;
- if (snd_seq_device_new(emu->card, 0, SNDRV_SEQ_DEV_ID_OSS,
+ /* using device#1 here for avoiding conflicts with OPL3 */
+ if (snd_seq_device_new(emu->card, 1, SNDRV_SEQ_DEV_ID_OSS,
sizeof(struct snd_seq_oss_reg), &dev) < 0)
return;
libelf-getphdrnum \
libelf-mmap \
libnuma \
+ numa_num_possible_cpus \
libperl \
libpython \
libpython-version \
timerfd \
libdw-dwarf-unwind \
zlib \
- lzma
+ lzma \
+ get_cpuid
FEATURE_DISPLAY ?= \
dwarf \
libbfd \
libelf \
libnuma \
+ numa_num_possible_cpus \
libperl \
libpython \
libslang \
libunwind \
libdw-dwarf-unwind \
zlib \
- lzma
+ lzma \
+ get_cpuid
# Set FEATURE_CHECK_(C|LD)FLAGS-all for all FEATURE_TESTS features.
# If in the future we need per-feature checks/flags for features not
test-libelf-getphdrnum.bin \
test-libelf-mmap.bin \
test-libnuma.bin \
+ test-numa_num_possible_cpus.bin \
test-libperl.bin \
test-libpython.bin \
test-libpython-version.bin \
test-compile-x32.bin \
test-zlib.bin \
test-lzma.bin \
- test-bpf.bin
+ test-bpf.bin \
+ test-get_cpuid.bin
CC := $(CROSS_COMPILE)gcc -MD
PKG_CONFIG := $(CROSS_COMPILE)pkg-config
test-libnuma.bin:
$(BUILD) -lnuma
+test-numa_num_possible_cpus.bin:
+ $(BUILD) -lnuma
+
test-libunwind.bin:
$(BUILD) -lelf
test-lzma.bin:
$(BUILD) -llzma
+test-get_cpuid.bin:
+ $(BUILD)
+
test-bpf.bin:
$(BUILD)
# include "test-libnuma.c"
#undef main
+#define main main_test_numa_num_possible_cpus
+# include "test-numa_num_possible_cpus.c"
+#undef main
+
#define main main_test_timerfd
# include "test-timerfd.c"
#undef main
# include "test-lzma.c"
#undef main
+#define main main_test_get_cpuid
+# include "test-get_cpuid.c"
+#undef main
+
int main(int argc, char *argv[])
{
main_test_libpython();
main_test_libbfd();
main_test_backtrace();
main_test_libnuma();
+ main_test_numa_num_possible_cpus();
main_test_timerfd();
main_test_stackprotector_all();
main_test_libdw_dwarf_unwind();
main_test_zlib();
main_test_pthread_attr_setaffinity_np();
main_test_lzma();
+ main_test_get_cpuid();
return 0;
}
--- /dev/null
+#include <cpuid.h>
+
+int main(void)
+{
+ unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
+ return __get_cpuid(0x15, &eax, &ebx, &ecx, &edx);
+}
--- /dev/null
+#include <numa.h>
+
+int main(void)
+{
+ return numa_num_possible_cpus();
+}
struct format_field *field;
struct printk_map *printk;
long long val, fval;
- unsigned long addr;
+ unsigned long long addr;
char *str;
unsigned char *hex;
int print;
*/
if (!(field->flags & FIELD_IS_ARRAY) &&
field->size == pevent->long_size) {
- addr = *(unsigned long *)(data + field->offset);
+
+ /* Handle heterogeneous recording and processing
+ * architectures
+ *
+ * CASE I:
+ * Traces recorded on 32-bit devices (32-bit
+ * addressing) and processed on 64-bit devices:
+ * In this case, only 32 bits should be read.
+ *
+ * CASE II:
+ * Traces recorded on 64 bit devices and processed
+ * on 32-bit devices:
+ * In this case, 64 bits must be read.
+ */
+ addr = (pevent->long_size == 8) ?
+ *(unsigned long long *)(data + field->offset) :
+ (unsigned long long)*(unsigned int *)(data + field->offset);
+
/* Check if it matches a print format */
printk = find_printk(pevent, addr);
if (printk)
trace_seq_puts(s, printk->printk);
else
- trace_seq_printf(s, "%lx", addr);
+ trace_seq_printf(s, "%llx", addr);
break;
}
str = malloc(len + 1);
CYC packets are not requested by default.
-no_force_psb This is a driver option and is not in the IA32_RTIT_CTL MSR.
-
- It stops the driver resetting the byte count to zero whenever
- enabling the trace (for example on context switches) which in
- turn results in no PSB being forced. However some processors
- will produce a PSB anyway.
-
- In any case, there is still a PSB when the trace is enabled for
- the first time.
-
- no_force_psb can be used to slightly decrease the trace size but
- may make it harder for the decoder to recover from errors.
-
- no_force_psb is not selected by default.
-
new snapshot option
-------------------
if (!evsel->attr.sample_id_all) {
sample->cpu = 0;
sample->time = 0;
- sample->tid = event->comm.tid;
- sample->pid = event->comm.pid;
+ sample->tid = event->fork.tid;
+ sample->pid = event->fork.pid;
}
print_sample_start(sample, thread, evsel);
perf_event__fprintf(event, stdout);
msg := $(warning No numa.h found, disables 'perf bench numa mem' benchmark, please install numactl-devel/libnuma-devel/libnuma-dev);
NO_LIBNUMA := 1
else
- CFLAGS += -DHAVE_LIBNUMA_SUPPORT
- EXTLIBS += -lnuma
- $(call detected,CONFIG_NUMA)
+ ifeq ($(feature-numa_num_possible_cpus), 0)
+ msg := $(warning Old numa library found, disables 'perf bench numa mem' benchmark, please install numactl-devel/libnuma-devel/libnuma-dev >= 2.0.8);
+ NO_LIBNUMA := 1
+ else
+ CFLAGS += -DHAVE_LIBNUMA_SUPPORT
+ EXTLIBS += -lnuma
+ $(call detected,CONFIG_NUMA)
+ endif
endif
endif
endif
ifndef NO_AUXTRACE
- $(call detected,CONFIG_AUXTRACE)
- CFLAGS += -DHAVE_AUXTRACE_SUPPORT
+ ifeq ($(feature-get_cpuid), 0)
+ msg := $(warning Your gcc lacks the __get_cpuid() builtin, disables support for auxtrace/Intel PT, please install a newer gcc);
+ NO_AUXTRACE := 1
+ else
+ $(call detected,CONFIG_AUXTRACE)
+ CFLAGS += -DHAVE_AUXTRACE_SUPPORT
+ endif
endif
# Among the variables below, these:
.disabled = 1,
.freq = 1,
};
+ struct cpu_map *cpus;
+ struct thread_map *threads;
attr.sample_freq = 500;
}
perf_evlist__add(evlist, evsel);
- evlist->cpus = cpu_map__dummy_new();
- evlist->threads = thread_map__new_by_tid(getpid());
- if (!evlist->cpus || !evlist->threads) {
+ cpus = cpu_map__dummy_new();
+ threads = thread_map__new_by_tid(getpid());
+ if (!cpus || !threads) {
err = -ENOMEM;
pr_debug("Not enough memory to create thread/cpu maps\n");
- goto out_delete_evlist;
+ goto out_free_maps;
}
+ perf_evlist__set_maps(evlist, cpus, threads);
+
+ cpus = NULL;
+ threads = NULL;
+
if (perf_evlist__open(evlist)) {
const char *knob = "/proc/sys/kernel/perf_event_max_sample_rate";
err = -1;
}
+out_free_maps:
+ cpu_map__put(cpus);
+ thread_map__put(threads);
out_delete_evlist:
perf_evlist__delete(evlist);
return err;
};
const char *argv[] = { "true", NULL };
char sbuf[STRERR_BUFSIZE];
+ struct cpu_map *cpus;
+ struct thread_map *threads;
signal(SIGCHLD, sig_handler);
* perf_evlist__prepare_workload we'll fill in the only thread
* we're monitoring, the one forked there.
*/
- evlist->cpus = cpu_map__dummy_new();
- evlist->threads = thread_map__new_by_tid(-1);
- if (!evlist->cpus || !evlist->threads) {
+ cpus = cpu_map__dummy_new();
+ threads = thread_map__new_by_tid(-1);
+ if (!cpus || !threads) {
err = -ENOMEM;
pr_debug("Not enough memory to create thread/cpu maps\n");
- goto out_delete_evlist;
+ goto out_free_maps;
}
+ perf_evlist__set_maps(evlist, cpus, threads);
+
+ cpus = NULL;
+ threads = NULL;
+
err = perf_evlist__prepare_workload(evlist, &target, argv, false,
workload_exec_failed_signal);
if (err < 0) {
err = -1;
}
+out_free_maps:
+ cpu_map__put(cpus);
+ thread_map__put(threads);
out_delete_evlist:
perf_evlist__delete(evlist);
return err;
&options[nr_options], dso);
nr_options += add_map_opt(browser, &actions[nr_options],
&options[nr_options],
- browser->selection->map);
+ browser->selection ?
+ browser->selection->map : NULL);
/* perf script support */
if (browser->he_selection) {
&actions[nr_options],
&options[nr_options],
thread, NULL);
+ /*
+ * Note that browser->selection != NULL
+ * when browser->he_selection is not NULL,
+ * so we don't need to check browser->selection
+ * before fetching browser->selection->sym like what
+ * we do before fetching browser->selection->map.
+ *
+ * See hist_browser__show_entry.
+ */
nr_options += add_script_opt(browser,
&actions[nr_options],
&options[nr_options],
libperf-y += llvm-utils.o
libperf-y += parse-options.o
libperf-y += parse-events.o
+libperf-y += perf_regs.o
libperf-y += path.o
libperf-y += rbtree.o
libperf-y += bitmap.o
libperf-y += scripting-engines/
-libperf-$(CONFIG_PERF_REGS) += perf_regs.o
libperf-$(CONFIG_ZLIB) += zlib.o
libperf-$(CONFIG_LZMA) += lzma.o
free(evlist);
}
+static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,
+ struct perf_evsel *evsel)
+{
+ /*
+ * We already have cpus for evsel (via PMU sysfs) so
+ * keep it, if there's no target cpu list defined.
+ */
+ if (!evsel->own_cpus || evlist->has_user_cpus) {
+ cpu_map__put(evsel->cpus);
+ evsel->cpus = cpu_map__get(evlist->cpus);
+ } else if (evsel->cpus != evsel->own_cpus) {
+ cpu_map__put(evsel->cpus);
+ evsel->cpus = cpu_map__get(evsel->own_cpus);
+ }
+
+ thread_map__put(evsel->threads);
+ evsel->threads = thread_map__get(evlist->threads);
+}
+
+static void perf_evlist__propagate_maps(struct perf_evlist *evlist)
+{
+ struct perf_evsel *evsel;
+
+ evlist__for_each(evlist, evsel)
+ __perf_evlist__propagate_maps(evlist, evsel);
+}
+
void perf_evlist__add(struct perf_evlist *evlist, struct perf_evsel *entry)
{
entry->evlist = evlist;
if (!evlist->nr_entries++)
perf_evlist__set_id_pos(evlist);
+
+ __perf_evlist__propagate_maps(evlist, entry);
}
void perf_evlist__splice_list_tail(struct perf_evlist *evlist,
- struct list_head *list,
- int nr_entries)
+ struct list_head *list)
{
- bool set_id_pos = !evlist->nr_entries;
+ struct perf_evsel *evsel, *temp;
- list_splice_tail(list, &evlist->entries);
- evlist->nr_entries += nr_entries;
- if (set_id_pos)
- perf_evlist__set_id_pos(evlist);
+ __evlist__for_each_safe(list, temp, evsel) {
+ list_del_init(&evsel->node);
+ perf_evlist__add(evlist, evsel);
+ }
}
void __perf_evlist__set_leader(struct list_head *list)
list_add_tail(&evsel->node, &head);
}
- perf_evlist__splice_list_tail(evlist, &head, nr_attrs);
+ perf_evlist__splice_list_tail(evlist, &head);
return 0;
return perf_evlist__mmap_ex(evlist, pages, overwrite, 0, false);
}
-static int perf_evlist__propagate_maps(struct perf_evlist *evlist,
- bool has_user_cpus)
-{
- struct perf_evsel *evsel;
-
- evlist__for_each(evlist, evsel) {
- /*
- * We already have cpus for evsel (via PMU sysfs) so
- * keep it, if there's no target cpu list defined.
- */
- if (evsel->cpus && has_user_cpus)
- cpu_map__put(evsel->cpus);
-
- if (!evsel->cpus || has_user_cpus)
- evsel->cpus = cpu_map__get(evlist->cpus);
-
- evsel->threads = thread_map__get(evlist->threads);
-
- if ((evlist->cpus && !evsel->cpus) ||
- (evlist->threads && !evsel->threads))
- return -ENOMEM;
- }
-
- return 0;
-}
-
int perf_evlist__create_maps(struct perf_evlist *evlist, struct target *target)
{
- evlist->threads = thread_map__new_str(target->pid, target->tid,
- target->uid);
+ struct cpu_map *cpus;
+ struct thread_map *threads;
+
+ threads = thread_map__new_str(target->pid, target->tid, target->uid);
- if (evlist->threads == NULL)
+ if (!threads)
return -1;
if (target__uses_dummy_map(target))
- evlist->cpus = cpu_map__dummy_new();
+ cpus = cpu_map__dummy_new();
else
- evlist->cpus = cpu_map__new(target->cpu_list);
+ cpus = cpu_map__new(target->cpu_list);
- if (evlist->cpus == NULL)
+ if (!cpus)
goto out_delete_threads;
- return perf_evlist__propagate_maps(evlist, !!target->cpu_list);
+ evlist->has_user_cpus = !!target->cpu_list;
+
+ perf_evlist__set_maps(evlist, cpus, threads);
+
+ return 0;
out_delete_threads:
- thread_map__put(evlist->threads);
- evlist->threads = NULL;
+ thread_map__put(threads);
return -1;
}
-int perf_evlist__set_maps(struct perf_evlist *evlist,
- struct cpu_map *cpus,
- struct thread_map *threads)
+void perf_evlist__set_maps(struct perf_evlist *evlist, struct cpu_map *cpus,
+ struct thread_map *threads)
{
- if (evlist->cpus)
+ /*
+ * Allow for the possibility that one or another of the maps isn't being
+ * changed i.e. don't put it. Note we are assuming the maps that are
+ * being applied are brand new and evlist is taking ownership of the
+ * original reference count of 1. If that is not the case it is up to
+ * the caller to increase the reference count.
+ */
+ if (cpus != evlist->cpus) {
cpu_map__put(evlist->cpus);
+ evlist->cpus = cpus;
+ }
- evlist->cpus = cpus;
-
- if (evlist->threads)
+ if (threads != evlist->threads) {
thread_map__put(evlist->threads);
+ evlist->threads = threads;
+ }
- evlist->threads = threads;
-
- return perf_evlist__propagate_maps(evlist, false);
+ perf_evlist__propagate_maps(evlist);
}
int perf_evlist__apply_filters(struct perf_evlist *evlist, struct perf_evsel **err_evsel)
static int perf_evlist__create_syswide_maps(struct perf_evlist *evlist)
{
+ struct cpu_map *cpus;
+ struct thread_map *threads;
int err = -ENOMEM;
/*
* error, and we may not want to do that fallback to a
* default cpu identity map :-\
*/
- evlist->cpus = cpu_map__new(NULL);
- if (evlist->cpus == NULL)
+ cpus = cpu_map__new(NULL);
+ if (!cpus)
goto out;
- evlist->threads = thread_map__new_dummy();
- if (evlist->threads == NULL)
- goto out_free_cpus;
+ threads = thread_map__new_dummy();
+ if (!threads)
+ goto out_put;
- err = 0;
+ perf_evlist__set_maps(evlist, cpus, threads);
out:
return err;
-out_free_cpus:
- cpu_map__put(evlist->cpus);
- evlist->cpus = NULL;
+out_put:
+ cpu_map__put(cpus);
goto out;
}
int nr_mmaps;
bool overwrite;
bool enabled;
+ bool has_user_cpus;
size_t mmap_len;
int id_pos;
int is_pos;
void perf_evlist__set_selected(struct perf_evlist *evlist,
struct perf_evsel *evsel);
-int perf_evlist__set_maps(struct perf_evlist *evlist,
- struct cpu_map *cpus,
- struct thread_map *threads);
+void perf_evlist__set_maps(struct perf_evlist *evlist, struct cpu_map *cpus,
+ struct thread_map *threads);
int perf_evlist__create_maps(struct perf_evlist *evlist, struct target *target);
int perf_evlist__apply_filters(struct perf_evlist *evlist, struct perf_evsel **err_evsel);
bool perf_evlist__valid_read_format(struct perf_evlist *evlist);
void perf_evlist__splice_list_tail(struct perf_evlist *evlist,
- struct list_head *list,
- int nr_entries);
+ struct list_head *list);
static inline struct perf_evsel *perf_evlist__first(struct perf_evlist *evlist)
{
perf_evsel__free_config_terms(evsel);
close_cgroup(evsel->cgrp);
cpu_map__put(evsel->cpus);
+ cpu_map__put(evsel->own_cpus);
thread_map__put(evsel->threads);
zfree(&evsel->group_name);
zfree(&evsel->name);
struct cgroup_sel *cgrp;
void *handler;
struct cpu_map *cpus;
+ struct cpu_map *own_cpus;
struct thread_map *threads;
unsigned int sample_size;
int id_pos;
if (ph->needs_swap)
nr = bswap_32(nr);
- ph->env.nr_cpus_online = nr;
+ ph->env.nr_cpus_avail = nr;
ret = readn(fd, &nr, sizeof(nr));
if (ret != sizeof(nr))
if (ph->needs_swap)
nr = bswap_32(nr);
- ph->env.nr_cpus_avail = nr;
+ ph->env.nr_cpus_online = nr;
return 0;
}
if (err)
return err;
if (event->header.type == PERF_RECORD_EXIT) {
- err = intel_bts_process_tid_exit(bts, event->comm.tid);
+ err = intel_bts_process_tid_exit(bts, event->fork.tid);
if (err)
return err;
}
if (pt->timeless_decoding) {
if (event->header.type == PERF_RECORD_EXIT) {
err = intel_pt_process_timeless_queues(pt,
- event->comm.tid,
+ event->fork.tid,
sample->time);
}
} else if (timestamp) {
if (!evsel)
return NULL;
- if (cpus)
- evsel->cpus = cpu_map__get(cpus);
+ evsel->cpus = cpu_map__get(cpus);
+ evsel->own_cpus = cpu_map__get(cpus);
if (name)
evsel->name = strdup(name);
ret = parse_events__scanner(str, &data, PE_START_EVENTS);
perf_pmu__parse_cleanup();
if (!ret) {
- int entries = data.idx - evlist->nr_entries;
struct perf_evsel *last;
- perf_evlist__splice_list_tail(evlist, &data.list, entries);
+ perf_evlist__splice_list_tail(evlist, &data.list);
evlist->nr_groups += data.nr_groups;
last = perf_evlist__last(evlist);
last->cmdline_group_boundary = true;
list_add_tail(&term->list, head);
ALLOC_LIST(list);
- ABORT_ON(parse_events_add_pmu(list, &data->idx, "cpu", head));
+ ABORT_ON(parse_events_add_pmu(data, list, "cpu", head));
parse_events__free_terms(head);
$$ = list;
}
SMPL_REG_END
};
+#ifdef HAVE_PERF_REGS_SUPPORT
int perf_reg_value(u64 *valp, struct regs_dump *regs, int id)
{
int i, idx = 0;
*valp = regs->cache_regs[id];
return 0;
}
+#endif
#define __PERF_REGS_H
#include <linux/types.h>
+#include <linux/compiler.h>
struct regs_dump;
int ret = 0;
if (module) {
- list_for_each_entry(dso, &host_machine->dsos.head, node) {
- if (!dso->kernel)
- continue;
- if (strncmp(dso->short_name + 1, module,
- dso->short_name_len - 2) == 0)
- goto found;
+ char module_name[128];
+
+ snprintf(module_name, sizeof(module_name), "[%s]", module);
+ map = map_groups__find_by_name(&host_machine->kmaps, MAP__FUNCTION, module_name);
+ if (map) {
+ dso = map->dso;
+ goto found;
}
pr_debug("Failed to find module %s.\n", module);
return -ENOENT;
file_offset = page_offset;
head = data_offset - page_offset;
- if (data_size && (data_offset + data_size < file_size))
+ if (data_size == 0)
+ goto out;
+
+ if (data_offset + data_size < file_size)
file_size = data_offset + data_size;
ui_progress__init(&prog, file_size, "Processing events...");
memset(counter->per_pkg_mask, 0, MAX_NR_CPUS);
}
-static int check_per_pkg(struct perf_evsel *counter, int cpu, bool *skip)
+static int check_per_pkg(struct perf_evsel *counter,
+ struct perf_counts_values *vals, int cpu, bool *skip)
{
unsigned long *mask = counter->per_pkg_mask;
struct cpu_map *cpus = perf_evsel__cpus(counter);
counter->per_pkg_mask = mask;
}
+ /*
+ * we do not consider an event that has not run as a good
+ * instance to mark a package as used (skip=1). Otherwise
+ * we may run into a situation where the first CPU in a package
+ * is not running anything, yet the second is, and this function
+ * would mark the package as used after the first CPU and would
+ * not read the values from the second CPU.
+ */
+ if (!(vals->run && vals->ena))
+ return 0;
+
s = cpu_map__get_socket(cpus, cpu);
if (s < 0)
return -1;
static struct perf_counts_values zero;
bool skip = false;
- if (check_per_pkg(evsel, cpu, &skip)) {
+ if (check_per_pkg(evsel, count, cpu, &skip)) {
pr_err("failed to read per-pkg counter\n");
return -1;
}
#endif
#ifndef HAVE_ELF_GETPHDRNUM_SUPPORT
-int elf_getphdrnum(Elf *elf, size_t *dst)
+static int elf_getphdrnum(Elf *elf, size_t *dst)
{
GElf_Ehdr gehdr;
GElf_Ehdr *ehdr;
static int kcore__init(struct kcore *kcore, char *filename, int elfclass,
bool temp)
{
- GElf_Ehdr *ehdr;
-
kcore->elfclass = elfclass;
if (temp)
if (!gelf_newehdr(kcore->elf, elfclass))
goto out_end;
- ehdr = gelf_getehdr(kcore->elf, &kcore->ehdr);
- if (!ehdr)
- goto out_end;
+ memset(&kcore->ehdr, 0, sizeof(GElf_Ehdr));
return 0;
static int kcore__add_phdr(struct kcore *kcore, int idx, off_t offset,
u64 addr, u64 len)
{
- GElf_Phdr gphdr;
- GElf_Phdr *phdr;
-
- phdr = gelf_getphdr(kcore->elf, idx, &gphdr);
- if (!phdr)
- return -1;
-
- phdr->p_type = PT_LOAD;
- phdr->p_flags = PF_R | PF_W | PF_X;
- phdr->p_offset = offset;
- phdr->p_vaddr = addr;
- phdr->p_paddr = 0;
- phdr->p_filesz = len;
- phdr->p_memsz = len;
- phdr->p_align = page_size;
-
- if (!gelf_update_phdr(kcore->elf, idx, phdr))
+ GElf_Phdr phdr = {
+ .p_type = PT_LOAD,
+ .p_flags = PF_R | PF_W | PF_X,
+ .p_offset = offset,
+ .p_vaddr = addr,
+ .p_paddr = 0,
+ .p_filesz = len,
+ .p_memsz = len,
+ .p_align = page_size,
+ };
+
+ if (!gelf_update_phdr(kcore->elf, idx, &phdr))
return -1;
return 0;
dir = opendir(procfs__mountpoint());
if (!dir)
- return -1;
+ return false;
/* Walk through the directory. */
while (ret && (d = readdir(dir)) != NULL) {
unsigned int extra_msr_offset64;
unsigned int extra_delta_offset32;
unsigned int extra_delta_offset64;
+unsigned int aperf_mperf_multiplier = 1;
int do_smi;
double bclk;
+double base_hz;
+double tsc_tweak = 1.0;
unsigned int show_pkg;
unsigned int show_core;
unsigned int show_cpu;
/* %Busy */
if (has_aperf) {
if (!skip_c0)
- outp += sprintf(outp, "%8.2f", 100.0 * t->mperf/t->tsc);
+ outp += sprintf(outp, "%8.2f", 100.0 * t->mperf/t->tsc/tsc_tweak);
else
outp += sprintf(outp, "********");
}
/* Bzy_MHz */
if (has_aperf)
outp += sprintf(outp, "%8.0f",
- 1.0 * t->tsc / units * t->aperf / t->mperf / interval_float);
+ 1.0 * t->tsc * tsc_tweak / units * t->aperf / t->mperf / interval_float);
/* TSC_MHz */
outp += sprintf(outp, "%8.0f", 1.0 * t->tsc/units/interval_float);
return -3;
if (get_msr(cpu, MSR_IA32_MPERF, &t->mperf))
return -4;
+ t->aperf = t->aperf * aperf_mperf_multiplier;
+ t->mperf = t->mperf * aperf_mperf_multiplier;
}
if (do_smi) {
int amt_pkg_cstate_limits[16] = {PCL__0, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
int phi_pkg_cstate_limits[16] = {PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
+
+static void
+calculate_tsc_tweak()
+{
+ unsigned long long msr;
+ unsigned int base_ratio;
+
+ get_msr(base_cpu, MSR_NHM_PLATFORM_INFO, &msr);
+ base_ratio = (msr >> 8) & 0xFF;
+ base_hz = base_ratio * bclk * 1000000;
+ tsc_tweak = base_hz / tsc_hz;
+}
+
static void
dump_nhm_platform_info(void)
{
switch (model) {
case 0x3A: /* IVB */
- case 0x3E: /* IVB Xeon */
-
case 0x3C: /* HSW */
case 0x3F: /* HSX */
case 0x45: /* HSW */
return 0;
}
+unsigned int get_aperf_mperf_multiplier(unsigned int family, unsigned int model)
+{
+ if (is_knl(family, model))
+ return 1024;
+ return 1;
+}
+
#define SLM_BCLK_FREQS 5
double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0};
}
}
+ if (has_aperf)
+ aperf_mperf_multiplier = get_aperf_mperf_multiplier(family, model);
+
do_nhm_platform_info = do_nhm_cstates = do_smi = probe_nhm_msrs(family, model);
do_snb_cstates = has_snb_msrs(family, model);
do_pc2 = do_snb_cstates && (pkg_cstate_limit >= PCL__2);
if (debug)
dump_cstate_pstate_config_info();
+ if (has_skl_msrs(family, model))
+ calculate_tsc_tweak();
+
return;
}
}
void print_version() {
- fprintf(stderr, "turbostat version 4.7 17-June, 2015"
+ fprintf(stderr, "turbostat version 4.8 26-Sep, 2015"
" - Len Brown <lenb@kernel.org>\n");
}
TARGETS += ptrace
TARGETS += seccomp
TARGETS += size
+TARGETS += static_keys
TARGETS += sysctl
ifneq (1, $(quicktest))
TARGETS += timers
endif
TARGETS += user
-TARGETS += jumplabel
TARGETS += vm
TARGETS += x86
TARGETS += zram
CFLAGS = -Wall
BINARIES = execveat
-DEPS = execveat.symlink execveat.denatured script
+DEPS = execveat.symlink execveat.denatured script subdir
all: $(BINARIES) $(DEPS)
subdir:
include ../lib.mk
-override EMIT_TESTS := echo "mkdir -p subdir; (./execveat && echo \"selftests: execveat [PASS]\") || echo \"selftests: execveat [FAIL]\""
-
clean:
rm -rf $(BINARIES) $(DEPS) subdir.moved execveat.moved xxxxx*
all:
TEST_PROGS := ftracetest
-TEST_DIRS := test.d/
+TEST_DIRS := test.d
include ../lib.mk
$(RUN_TESTS)
define INSTALL_RULE
- @if [ "X$(TEST_PROGS)$(TEST_PROGS_EXTENDED)$(TEST_FILES)" != "X" ]; then \
- mkdir -p $(INSTALL_PATH); \
- for TEST_DIR in $(TEST_DIRS); do \
- cp -r $$TEST_DIR $(INSTALL_PATH); \
- done; \
- echo "install -t $(INSTALL_PATH) $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES)"; \
- install -t $(INSTALL_PATH) $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES); \
+ @if [ "X$(TEST_PROGS)$(TEST_PROGS_EXTENDED)$(TEST_FILES)" != "X" ]; then \
+ mkdir -p ${INSTALL_PATH}; \
+ echo "rsync -a $(TEST_DIRS) $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES) $(INSTALL_PATH)/"; \
+ rsync -a $(TEST_DIRS) $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES) $(INSTALL_PATH)/; \
fi
endef
CFLAGS += -g -I../../../../usr/include/
-all:
- $(CC) $(CFLAGS) membarrier_test.c -o membarrier_test
-
TEST_PROGS := membarrier_test
+all: $(TEST_PROGS)
+
include ../lib.mk
clean:
- $(RM) membarrier_test
+ $(RM) $(TEST_PROGS)
#define _GNU_SOURCE
-#define __EXPORTED_HEADERS__
-
#include <linux/membarrier.h>
-#include <asm-generic/unistd.h>
-#include <sys/syscall.h>
+#include <syscall.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
-CFLAGS = -O2
+CFLAGS += -O2
+LDLIBS = -lrt -lpthread -lpopt
+TEST_PROGS := mq_open_tests mq_perf_tests
-all:
- $(CC) $(CFLAGS) mq_open_tests.c -o mq_open_tests -lrt
- $(CC) $(CFLAGS) -o mq_perf_tests mq_perf_tests.c -lrt -lpthread -lpopt
+all: $(TEST_PROGS)
include ../lib.mk
@./mq_perf_tests || echo "selftests: mq_perf_tests [FAIL]"
endef
-TEST_PROGS := mq_open_tests mq_perf_tests
-
override define EMIT_TESTS
echo "./mq_open_tests /test1 || echo \"selftests: mq_open_tests [FAIL]\""
echo "./mq_perf_tests || echo \"selftests: mq_perf_tests [FAIL]\""
# define ARCH_REGS struct pt_regs
# define SYSCALL_NUM gpr[0]
# define SYSCALL_RET gpr[3]
+#elif defined(__s390__)
+# define ARCH_REGS s390_regs
+# define SYSCALL_NUM gprs[2]
+# define SYSCALL_RET gprs[2]
#else
# error "Do not know how to find your architecture's registers and syscalls"
#endif
ret = ptrace(PTRACE_GETREGSET, tracee, NT_PRSTATUS, &iov);
EXPECT_EQ(0, ret);
-#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || defined(__powerpc__)
+#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || \
+ defined(__powerpc__) || defined(__s390__)
{
regs.SYSCALL_NUM = syscall;
}
ret = ptrace(PTRACE_GETEVENTMSG, tracee, NULL, &msg);
EXPECT_EQ(0, ret);
+ /* Validate and take action on expected syscalls. */
switch (msg) {
case 0x1002:
/* change getpid to getppid. */
+ EXPECT_EQ(__NR_getpid, get_syscall(_metadata, tracee));
change_syscall(_metadata, tracee, __NR_getppid);
break;
case 0x1003:
/* skip gettid. */
+ EXPECT_EQ(__NR_gettid, get_syscall(_metadata, tracee));
change_syscall(_metadata, tracee, -1);
break;
case 0x1004:
/* do nothing (allow getppid) */
+ EXPECT_EQ(__NR_getppid, get_syscall(_metadata, tracee));
break;
default:
EXPECT_EQ(0, msg) {
# define __NR_seccomp 277
# elif defined(__powerpc__)
# define __NR_seccomp 358
+# elif defined(__s390__)
+# define __NR_seccomp 348
# else
# warning "seccomp syscall number unknown for this architecture"
# define __NR_seccomp 0xffff
/* Reject insane operation. */
ret = seccomp(-1, 0, &prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
EXPECT_EQ(EINVAL, errno) {
TH_LOG("Did not reject crazy op value!");
}
}
ret = seccomp(SECCOMP_SET_MODE_FILTER, 0, &prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
EXPECT_EQ(0, ret) {
TH_LOG("Could not install filter!");
}
ret = seccomp(SECCOMP_SET_MODE_FILTER, SECCOMP_FLAG_FILTER_TSYNC,
&prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
EXPECT_EQ(0, ret) {
TH_LOG("Could not install initial filter with TSYNC!");
}
/* Check prctl failure detection by requesting sib 0 diverge. */
ret = seccomp(SECCOMP_SET_MODE_FILTER, 0, &prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
ASSERT_EQ(0, ret) {
TH_LOG("setting filter failed");
}
}
ret = seccomp(SECCOMP_SET_MODE_FILTER, 0, &self->root_prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
ASSERT_EQ(0, ret) {
TH_LOG("Kernel does not support SECCOMP_SET_MODE_FILTER!");
}
ret = seccomp(SECCOMP_SET_MODE_FILTER, SECCOMP_FLAG_FILTER_TSYNC,
&self->apply_prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
ASSERT_EQ(0, ret) {
TH_LOG("Could install filter on all threads!");
}
}
ret = seccomp(SECCOMP_SET_MODE_FILTER, 0, &self->root_prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
ASSERT_EQ(0, ret) {
TH_LOG("Kernel does not support SECCOMP_SET_MODE_FILTER!");
}
}
ret = seccomp(SECCOMP_SET_MODE_FILTER, 0, &self->root_prog);
+ ASSERT_NE(ENOSYS, errno) {
+ TH_LOG("Kernel does not support seccomp syscall!");
+ }
ASSERT_EQ(0, ret) {
TH_LOG("Kernel does not support SECCOMP_SET_MODE_FILTER!");
}
__typeof__(_expected) __exp = (_expected); \
__typeof__(_seen) __seen = (_seen); \
if (!(__exp _t __seen)) { \
- unsigned long long __exp_print = 0; \
- unsigned long long __seen_print = 0; \
- /* Avoid casting complaints the scariest way we can. */ \
- memcpy(&__exp_print, &__exp, sizeof(__exp)); \
- memcpy(&__seen_print, &__seen, sizeof(__seen)); \
+ unsigned long long __exp_print = (unsigned long long)__exp; \
+ unsigned long long __seen_print = (unsigned long long)__seen; \
__TH_LOG("Expected %s (%llu) %s %s (%llu)", \
#_expected, __exp_print, #_t, \
#_seen, __seen_print); \
# Makefile for vm selftests
-CFLAGS = -Wall
+CFLAGS = -Wall -I ../../../../usr/include $(EXTRA_CFLAGS)
BINARIES = compaction_test
BINARIES += hugepage-mmap
BINARIES += hugepage-shm
all: $(BINARIES)
%: %.c
$(CC) $(CFLAGS) -o $@ $^ -lrt
-userfaultfd: userfaultfd.c
- $(CC) $(CFLAGS) -O2 -o $@ $^ -lpthread
+userfaultfd: userfaultfd.c ../../../../usr/include/linux/kernel.h
+ $(CC) $(CFLAGS) -O2 -o $@ $< -lpthread
+
+../../../../usr/include/linux/kernel.h:
+ make -C ../../../.. headers_install
TEST_PROGS := run_vmtests
TEST_FILES := $(BINARIES)
#include <sys/syscall.h>
#include <sys/ioctl.h>
#include <pthread.h>
-#include "../../../../include/uapi/linux/userfaultfd.h"
-
-#ifdef __x86_64__
-#define __NR_userfaultfd 323
-#elif defined(__i386__)
-#define __NR_userfaultfd 374
-#elif defined(__powewrpc__)
-#define __NR_userfaultfd 364
-#else
-#error "missing __NR_userfaultfd definition"
-#endif
+#include <linux/userfaultfd.h>
+
+#ifdef __NR_userfaultfd
static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
struct uffdio_register uffdio_register;
struct uffdio_api uffdio_api;
unsigned long cpu;
- int uffd_flags;
+ int uffd_flags, err;
unsigned long userfaults[nr_cpus];
if (posix_memalign(&area, page_size, nr_pages * page_size)) {
*area_mutex(area_src, nr) = (pthread_mutex_t)
PTHREAD_MUTEX_INITIALIZER;
count_verify[nr] = *area_count(area_src, nr) = 1;
+ /*
+ * In the transition between 255 to 256, powerpc will
+ * read out of order in my_bcmp and see both bytes as
+ * zero, so leave a placeholder below always non-zero
+ * after the count, to avoid my_bcmp to trigger false
+ * positives.
+ */
+ *(area_count(area_src, nr) + 1) = 1;
}
pipefd = malloc(sizeof(int) * nr_cpus * 2);
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, 16*1024*1024);
+ err = 0;
while (bounces--) {
unsigned long expected_ioctls;
/* verification */
if (bounces & BOUNCE_VERIFY) {
for (nr = 0; nr < nr_pages; nr++) {
- if (my_bcmp(area_dst,
- area_dst + nr * page_size,
- sizeof(pthread_mutex_t))) {
- fprintf(stderr,
- "error mutex 2 %lu\n",
- nr);
- bounces = 0;
- }
if (*area_count(area_dst, nr) != count_verify[nr]) {
fprintf(stderr,
"error area_count %Lu %Lu %lu\n",
*area_count(area_src, nr),
count_verify[nr],
nr);
+ err = 1;
bounces = 0;
}
}
printf("\n");
}
- return 0;
+ return err;
}
int main(int argc, char **argv)
fprintf(stderr, "Usage: <MiB> <bounces>\n"), exit(1);
nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
page_size = sysconf(_SC_PAGE_SIZE);
- if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) >
- page_size)
+ if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
+ > page_size)
fprintf(stderr, "Impossible to run this test\n"), exit(2);
nr_pages_per_cpu = atol(argv[1]) * 1024*1024 / page_size /
nr_cpus;
nr_pages, nr_pages_per_cpu);
return userfaultfd_stress();
}
+
+#else /* __NR_userfaultfd */
+
+#warning "missing __NR_userfaultfd definition"
+
+int main(void)
+{
+ printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
+ return 0;
+}
+
+#endif /* __NR_userfaultfd */
v86->regs.eip = eip;
ret = vm86(VM86_ENTER, v86);
- if (ret == -1 && errno == ENOSYS) {
- printf("[SKIP]\tvm86 not supported\n");
+ if (ret == -1 && (errno == ENOSYS || errno == EPERM)) {
+ printf("[SKIP]\tvm86 %s\n",
+ errno == ENOSYS ? "not supported" : "not allowed");
return false;
}
#!/bin/bash
TCID="zram.sh"
-check_prereqs()
-{
- local msg="skip all tests:"
-
- if [ $UID != 0 ]; then
- echo $msg must be run as root >&2
- exit 0
- fi
-}
+. ./zram_lib.sh
run_zram () {
echo "--------------------"
check_prereqs()
{
local msg="skip all tests:"
+ local uid=$(id -u)
- if [ $UID != 0 ]; then
+ if [ $uid -ne 0 ]; then
echo $msg must be run as root >&2
exit 0
fi
CFLAGS += -g -O2 -Werror -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE
vpath %.c ../../drivers/virtio ../../drivers/vhost
mod:
- ${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test
+ ${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V}
.PHONY: all test mod clean
clean:
${RM} *.o vringh_test virtio_test vhost_test/*.o vhost_test/.*.cmd \
#define mb() __sync_synchronize()
#define smp_mb() mb()
+# define dma_rmb() barrier()
+# define dma_wmb() barrier()
# define smp_rmb() barrier()
# define smp_wmb() barrier()
/* Weak barriers should be used. If not - it's a bug */
--- /dev/null
+#define EXPORT_SYMBOL_GPL(sym) extern typeof(sym) sym
+#define EXPORT_SYMBOL(sym) extern typeof(sym) sym
+
typedef unsigned long long dma_addr_t;
typedef size_t __kernel_size_t;
+typedef unsigned int __wsum;
struct page {
unsigned long long dummy;
return __kmalloc_fake;
return malloc(s);
}
+static inline void *kzalloc(size_t s, gfp_t gfp)
+{
+ void *p = kmalloc(s, gfp);
+
+ memset(p, 0, s);
+ return p;
+}
static inline void kfree(void *p)
{
*/
timer->irq = irq;
+ /*
+ * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
+ * and to 0 for ARMv7. We provide an implementation that always
+ * resets the timer to be disabled and unmasked and is compliant with
+ * the ARMv7 architecture.
+ */
+ timer->cntv_ctl = 0;
+
/*
* Tell the VGIC that the virtual interrupt is tied to a
* physical interrupt. We do that once per VCPU.
vgic->vctrl_base = NULL;
vgic->type = VGIC_V3;
- vgic->max_gic_vcpus = KVM_MAX_VCPUS;
+ vgic->max_gic_vcpus = VGIC_V3_MAX_CPUS;
kvm_info("%s@%llx IRQ%d\n", vgic_node->name,
vcpu_res.start, vgic->maint_irq);
struct irq_phys_map *map;
map = vgic_irq_map_search(vcpu, irq);
- /*
- * If we have a mapping, and the virtual interrupt is
- * being injected, then we must set the state to
- * active in the physical world. Otherwise the
- * physical interrupt will fire and the guest will
- * exit before processing the virtual interrupt.
- */
if (map) {
- int ret;
-
- BUG_ON(!map->active);
vlr.hwirq = map->phys_irq;
vlr.state |= LR_HW;
vlr.state &= ~LR_EOI_INT;
- ret = irq_set_irqchip_state(map->irq,
- IRQCHIP_STATE_ACTIVE,
- true);
- WARN_ON(ret);
-
/*
* Make sure we're not going to sample this
* again, as a HW-backed interrupt cannot be
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
unsigned long *pa_percpu, *pa_shared;
- int i, vcpu_id;
+ int i, vcpu_id, lr, ret;
int overflow = 0;
int nr_shared = vgic_nr_shared_irqs(dist);
*/
clear_bit(vcpu_id, dist->irq_pending_on_cpu);
}
+
+ for (lr = 0; lr < vgic->nr_lr; lr++) {
+ struct vgic_lr vlr;
+
+ if (!test_bit(lr, vgic_cpu->lr_used))
+ continue;
+
+ vlr = vgic_get_lr(vcpu, lr);
+
+ /*
+ * If we have a mapping, and the virtual interrupt is
+ * presented to the guest (as pending or active), then we must
+ * set the state to active in the physical world. See
+ * Documentation/virtual/kvm/arm/vgic-mapped-irqs.txt.
+ */
+ if (vlr.state & LR_HW) {
+ struct irq_phys_map *map;
+ map = vgic_irq_map_search(vcpu, vlr.irq);
+
+ ret = irq_set_irqchip_state(map->irq,
+ IRQCHIP_STATE_ACTIVE,
+ true);
+ WARN_ON(ret);
+ }
+ }
}
static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
int kvm_coalesced_mmio_init(struct kvm *kvm);
void kvm_coalesced_mmio_free(struct kvm *kvm);
int kvm_vm_ioctl_register_coalesced_mmio(struct kvm *kvm,
- struct kvm_coalesced_mmio_zone *zone);
+ struct kvm_coalesced_mmio_zone *zone);
int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm,
- struct kvm_coalesced_mmio_zone *zone);
+ struct kvm_coalesced_mmio_zone *zone);
#else
return KVM_MMIO_BUS;
}
-static int
-kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
+static int kvm_assign_ioeventfd_idx(struct kvm *kvm,
+ enum kvm_bus bus_idx,
+ struct kvm_ioeventfd *args)
{
- enum kvm_bus bus_idx;
- struct _ioeventfd *p;
- struct eventfd_ctx *eventfd;
- int ret;
-
- bus_idx = ioeventfd_bus_from_flags(args->flags);
- /* must be natural-word sized, or 0 to ignore length */
- switch (args->len) {
- case 0:
- case 1:
- case 2:
- case 4:
- case 8:
- break;
- default:
- return -EINVAL;
- }
-
- /* check for range overflow */
- if (args->addr + args->len < args->addr)
- return -EINVAL;
- /* check for extra flags that we don't understand */
- if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)
- return -EINVAL;
-
- /* ioeventfd with no length can't be combined with DATAMATCH */
- if (!args->len &&
- args->flags & (KVM_IOEVENTFD_FLAG_PIO |
- KVM_IOEVENTFD_FLAG_DATAMATCH))
- return -EINVAL;
+ struct eventfd_ctx *eventfd;
+ struct _ioeventfd *p;
+ int ret;
eventfd = eventfd_ctx_fdget(args->fd);
if (IS_ERR(eventfd))
if (ret < 0)
goto unlock_fail;
- /* When length is ignored, MMIO is also put on a separate bus, for
- * faster lookups.
- */
- if (!args->len && !(args->flags & KVM_IOEVENTFD_FLAG_PIO)) {
- ret = kvm_io_bus_register_dev(kvm, KVM_FAST_MMIO_BUS,
- p->addr, 0, &p->dev);
- if (ret < 0)
- goto register_fail;
- }
-
kvm->buses[bus_idx]->ioeventfd_count++;
list_add_tail(&p->list, &kvm->ioeventfds);
return 0;
-register_fail:
- kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
unlock_fail:
mutex_unlock(&kvm->slots_lock);
}
static int
-kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
+kvm_deassign_ioeventfd_idx(struct kvm *kvm, enum kvm_bus bus_idx,
+ struct kvm_ioeventfd *args)
{
- enum kvm_bus bus_idx;
struct _ioeventfd *p, *tmp;
struct eventfd_ctx *eventfd;
int ret = -ENOENT;
- bus_idx = ioeventfd_bus_from_flags(args->flags);
eventfd = eventfd_ctx_fdget(args->fd);
if (IS_ERR(eventfd))
return PTR_ERR(eventfd);
continue;
kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
- if (!p->length) {
- kvm_io_bus_unregister_dev(kvm, KVM_FAST_MMIO_BUS,
- &p->dev);
- }
kvm->buses[bus_idx]->ioeventfd_count--;
ioeventfd_release(p);
ret = 0;
return ret;
}
+static int kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
+{
+ enum kvm_bus bus_idx = ioeventfd_bus_from_flags(args->flags);
+ int ret = kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
+
+ if (!args->len && bus_idx == KVM_MMIO_BUS)
+ kvm_deassign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
+
+ return ret;
+}
+
+static int
+kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
+{
+ enum kvm_bus bus_idx;
+ int ret;
+
+ bus_idx = ioeventfd_bus_from_flags(args->flags);
+ /* must be natural-word sized, or 0 to ignore length */
+ switch (args->len) {
+ case 0:
+ case 1:
+ case 2:
+ case 4:
+ case 8:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ /* check for range overflow */
+ if (args->addr + args->len < args->addr)
+ return -EINVAL;
+
+ /* check for extra flags that we don't understand */
+ if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)
+ return -EINVAL;
+
+ /* ioeventfd with no length can't be combined with DATAMATCH */
+ if (!args->len &&
+ args->flags & (KVM_IOEVENTFD_FLAG_PIO |
+ KVM_IOEVENTFD_FLAG_DATAMATCH))
+ return -EINVAL;
+
+ ret = kvm_assign_ioeventfd_idx(kvm, bus_idx, args);
+ if (ret)
+ goto fail;
+
+ /* When length is ignored, MMIO is also put on a separate bus, for
+ * faster lookups.
+ */
+ if (!args->len && bus_idx == KVM_MMIO_BUS) {
+ ret = kvm_assign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
+ if (ret < 0)
+ goto fast_fail;
+ }
+
+ return 0;
+
+fast_fail:
+ kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
+fail:
+ return ret;
+}
+
int
kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
{
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");
-/* halt polling only reduces halt latency by 5-7 us, 500us is enough */
-static unsigned int halt_poll_ns = 500000;
+/* Architectures should define their poll value according to the halt latency */
+static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
/* Default doubles per-vcpu halt_poll_ns. */
if (vcpu->halt_poll_ns) {
ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
+ ++vcpu->stat.halt_attempted_poll;
do {
/*
* This sets KVM_REQ_UNHALT if an interrupt
else if (vcpu->halt_poll_ns < halt_poll_ns &&
block_ns < halt_poll_ns)
grow_halt_poll_ns(vcpu);
- }
+ } else
+ vcpu->halt_poll_ns = 0;
trace_kvm_vcpu_wakeup(block_ns, waited);
}
static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
const struct kvm_io_range *r2)
{
- if (r1->addr < r2->addr)
+ gpa_t addr1 = r1->addr;
+ gpa_t addr2 = r2->addr;
+
+ if (addr1 < addr2)
return -1;
- if (r1->addr + r1->len > r2->addr + r2->len)
+
+ /* If r2->len == 0, match the exact address. If r2->len != 0,
+ * accept any overlapping write. Any order is acceptable for
+ * overlapping ranges, because kvm_io_bus_get_first_dev ensures
+ * we process all of them.
+ */
+ if (r2->len) {
+ addr1 += r1->len;
+ addr2 += r2->len;
+ }
+
+ if (addr1 > addr2)
return 1;
+
return 0;
}
projects / firefly-linux-kernel-4.4.55.git / commitdiff