2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 #include <linux/kernel.h>
48 #include <linux/module.h>
49 #include <linux/pci.h>
50 #include <linux/init.h>
51 #include <linux/types.h>
53 #include <linux/slab.h>
54 #include <linux/errno.h>
55 #include <linux/device.h>
56 #include <linux/dma-mapping.h>
57 #include <linux/platform_device.h>
58 #include <linux/workqueue.h>
60 #include <adf_accel_devices.h>
61 #include <adf_common_drv.h>
63 #include <adf_transport_access_macros.h>
64 #include "adf_dh895xcc_hw_data.h"
67 static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;
69 #define ADF_SYSTEM_DEVICE(device_id) \
70 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
72 static const struct pci_device_id adf_pci_tbl[] = {
73 ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
76 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
78 static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
79 static void adf_remove(struct pci_dev *dev);
81 static struct pci_driver adf_driver = {
82 .id_table = adf_pci_tbl,
83 .name = adf_driver_name,
88 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
90 struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
93 adf_exit_admin_comms(accel_dev);
94 adf_exit_arb(accel_dev);
95 adf_cleanup_etr_data(accel_dev);
97 for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
98 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
101 pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
104 if (accel_dev->hw_device) {
105 switch (accel_dev->hw_device->pci_dev_id) {
106 case ADF_DH895XCC_PCI_DEVICE_ID:
107 adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
112 kfree(accel_dev->hw_device);
114 adf_cfg_dev_remove(accel_dev);
115 debugfs_remove(accel_dev->debugfs_dir);
116 adf_devmgr_rm_dev(accel_dev);
117 pci_release_regions(accel_pci_dev->pci_dev);
118 pci_disable_device(accel_pci_dev->pci_dev);
122 static uint8_t adf_get_dev_node_id(struct pci_dev *pdev)
124 unsigned int bus_per_cpu = 0;
125 struct cpuinfo_x86 *c = &cpu_data(num_online_cpus() - 1);
127 if (!c->phys_proc_id)
130 bus_per_cpu = 256 / (c->phys_proc_id + 1);
132 if (bus_per_cpu != 0)
133 return pdev->bus->number / bus_per_cpu;
137 static int qat_dev_start(struct adf_accel_dev *accel_dev)
139 int cpus = num_online_cpus();
140 int banks = GET_MAX_BANKS(accel_dev);
141 int instances = min(cpus, banks);
142 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
146 if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
148 if (adf_cfg_section_add(accel_dev, "Accelerator0"))
150 for (i = 0; i < instances; i++) {
152 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
153 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
154 key, (void *)&val, ADF_DEC))
157 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
159 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
160 key, (void *)&val, ADF_DEC))
163 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
165 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
166 key, (void *)&val, ADF_DEC))
170 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
171 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
172 key, (void *)&val, ADF_DEC))
176 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
177 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
178 key, (void *)&val, ADF_DEC))
182 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
183 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
184 key, (void *)&val, ADF_DEC))
188 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i);
189 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
190 key, (void *)&val, ADF_DEC))
194 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
195 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
196 key, (void *)&val, ADF_DEC))
200 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
201 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
202 key, (void *)&val, ADF_DEC))
206 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i);
207 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
208 key, (void *)&val, ADF_DEC))
211 val = ADF_COALESCING_DEF_TIME;
212 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
213 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
214 key, (void *)&val, ADF_DEC))
219 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
220 ADF_NUM_CY, (void *)&val, ADF_DEC))
223 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
224 return adf_dev_start(accel_dev);
226 dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
230 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
232 struct adf_accel_dev *accel_dev;
233 struct adf_accel_pci *accel_pci_dev;
234 struct adf_hw_device_data *hw_data;
235 void __iomem *pmisc_bar_addr = NULL;
236 char name[ADF_DEVICE_NAME_LENGTH];
237 unsigned int i, bar_nr;
241 switch (ent->device) {
242 case ADF_DH895XCC_PCI_DEVICE_ID:
245 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
249 node = adf_get_dev_node_id(pdev);
250 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, node);
254 accel_dev->numa_node = node;
255 INIT_LIST_HEAD(&accel_dev->crypto_list);
257 /* Add accel device to accel table.
258 * This should be called before adf_cleanup_accel is called */
259 if (adf_devmgr_add_dev(accel_dev)) {
260 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
265 accel_dev->owner = THIS_MODULE;
266 /* Allocate and configure device configuration structure */
267 hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, node);
273 accel_dev->hw_device = hw_data;
274 switch (ent->device) {
275 case ADF_DH895XCC_PCI_DEVICE_ID:
276 adf_init_hw_data_dh895xcc(accel_dev->hw_device);
281 accel_pci_dev = &accel_dev->accel_pci_dev;
282 pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
283 pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
286 /* Get Accelerators and Accelerators Engines masks */
287 hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
288 hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
289 accel_pci_dev->sku = hw_data->get_sku(hw_data);
290 accel_pci_dev->pci_dev = pdev;
291 /* If the device has no acceleration engines then ignore it. */
292 if (!hw_data->accel_mask || !hw_data->ae_mask ||
293 ((~hw_data->ae_mask) & 0x01)) {
294 dev_err(&pdev->dev, "No acceleration units found");
299 /* Create dev top level debugfs entry */
300 snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX,
301 hw_data->dev_class->name, hw_data->instance_id);
302 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
303 if (!accel_dev->debugfs_dir) {
304 dev_err(&pdev->dev, "Could not create debugfs dir\n");
309 /* Create device configuration table */
310 ret = adf_cfg_dev_add(accel_dev);
314 /* enable PCI device */
315 if (pci_enable_device(pdev)) {
320 /* set dma identifier */
321 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
322 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
323 dev_err(&pdev->dev, "No usable DMA configuration\n");
327 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
331 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
334 if (pci_request_regions(pdev, adf_driver_name)) {
339 /* Read accelerator capabilities mask */
340 pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
341 &hw_data->accel_capabilities_mask);
343 /* Find and map all the device's BARS */
344 for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
345 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
348 bar->base_addr = pci_resource_start(pdev, bar_nr);
351 bar->size = pci_resource_len(pdev, bar_nr);
352 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
353 if (!bar->virt_addr) {
354 dev_err(&pdev->dev, "Failed to map BAR %d\n", i);
358 if (i == ADF_DH895XCC_PMISC_BAR)
359 pmisc_bar_addr = bar->virt_addr;
361 pci_set_master(pdev);
363 if (adf_enable_aer(accel_dev, &adf_driver)) {
364 dev_err(&pdev->dev, "Failed to enable aer\n");
369 if (adf_init_etr_data(accel_dev)) {
370 dev_err(&pdev->dev, "Failed initialize etr\n");
375 if (adf_init_admin_comms(accel_dev)) {
376 dev_err(&pdev->dev, "Failed initialize admin comms\n");
381 if (adf_init_arb(accel_dev)) {
382 dev_err(&pdev->dev, "Failed initialize hw arbiter\n");
386 if (pci_save_state(pdev)) {
387 dev_err(&pdev->dev, "Failed to save pci state\n");
392 /* Enable bundle and misc interrupts */
393 ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET,
394 ADF_DH895XCC_SMIA0_MASK);
395 ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET,
396 ADF_DH895XCC_SMIA1_MASK);
398 ret = qat_dev_start(accel_dev);
400 adf_dev_stop(accel_dev);
406 adf_cleanup_accel(accel_dev);
410 static void __exit adf_remove(struct pci_dev *pdev)
412 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
415 pr_err("QAT: Driver removal failed\n");
418 if (adf_dev_stop(accel_dev))
419 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
420 adf_disable_aer(accel_dev);
421 adf_cleanup_accel(accel_dev);
424 static int __init adfdrv_init(void)
426 request_module("intel_qat");
427 if (qat_admin_register())
430 if (pci_register_driver(&adf_driver)) {
431 pr_err("QAT: Driver initialization failed\n");
437 static void __exit adfdrv_release(void)
439 pci_unregister_driver(&adf_driver);
440 qat_admin_unregister();
443 module_init(adfdrv_init);
444 module_exit(adfdrv_release);
446 MODULE_LICENSE("Dual BSD/GPL");
447 MODULE_AUTHOR("Intel");
448 MODULE_FIRMWARE("qat_895xcc.bin");
449 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");