1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/slab.h>
4 #include <linux/ctype.h>
5 #include <linux/kernel.h>
6 #include <linux/delay.h>
7 #include <linux/pci_ids.h>
8 #include "adf_accel_devices.h"
9 #include "adf_common_drv.h"
10 #include "icp_qat_uclo.h"
11 #include "icp_qat_hal.h"
12 #include "icp_qat_fw_loader_handle.h"
14 #define UWORD_CPYBUF_SIZE 1024
15 #define INVLD_UWORD 0xffffffffffull
16 #define PID_MINOR_REV 0xf
17 #define PID_MAJOR_REV (0xf << 4)
19 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle,
20 unsigned int ae, unsigned int image_num)
22 struct icp_qat_uclo_aedata *ae_data;
23 struct icp_qat_uclo_encapme *encap_image;
24 struct icp_qat_uclo_page *page = NULL;
25 struct icp_qat_uclo_aeslice *ae_slice = NULL;
27 ae_data = &obj_handle->ae_data[ae];
28 encap_image = &obj_handle->ae_uimage[image_num];
29 ae_slice = &ae_data->ae_slices[ae_data->slice_num];
30 ae_slice->encap_image = encap_image;
32 if (encap_image->img_ptr) {
33 ae_slice->ctx_mask_assigned =
34 encap_image->img_ptr->ctx_assigned;
35 ae_data->eff_ustore_size = obj_handle->ustore_phy_size;
37 ae_slice->ctx_mask_assigned = 0;
39 ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL);
40 if (!ae_slice->region)
42 ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL);
45 page = ae_slice->page;
46 page->encap_page = encap_image->page;
47 ae_slice->page->region = ae_slice->region;
51 kfree(ae_slice->region);
52 ae_slice->region = NULL;
56 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data)
61 pr_err("QAT: bad argument, ae_data is NULL\n ");
65 for (i = 0; i < ae_data->slice_num; i++) {
66 kfree(ae_data->ae_slices[i].region);
67 ae_data->ae_slices[i].region = NULL;
68 kfree(ae_data->ae_slices[i].page);
69 ae_data->ae_slices[i].page = NULL;
74 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table,
75 unsigned int str_offset)
77 if ((!str_table->table_len) || (str_offset > str_table->table_len))
79 return (char *)(((uintptr_t)(str_table->strings)) + str_offset);
82 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr)
84 int maj = hdr->maj_ver & 0xff;
85 int min = hdr->min_ver & 0xff;
87 if (hdr->file_id != ICP_QAT_UOF_FID) {
88 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id);
91 if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) {
92 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n",
99 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr)
101 int maj = suof_hdr->maj_ver & 0xff;
102 int min = suof_hdr->min_ver & 0xff;
104 if (suof_hdr->file_id != ICP_QAT_SUOF_FID) {
105 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id);
108 if (suof_hdr->fw_type != 0) {
109 pr_err("QAT: unsupported firmware type\n");
112 if (suof_hdr->num_chunks <= 0x1) {
113 pr_err("QAT: SUOF chunk amount is incorrect\n");
116 if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) {
117 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n",
124 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle,
125 unsigned int addr, unsigned int *val,
126 unsigned int num_in_bytes)
129 unsigned char *ptr = (unsigned char *)val;
131 while (num_in_bytes) {
132 memcpy(&outval, ptr, 4);
133 SRAM_WRITE(handle, addr, outval);
140 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle,
141 unsigned char ae, unsigned int addr,
143 unsigned int num_in_bytes)
146 unsigned char *ptr = (unsigned char *)val;
148 addr >>= 0x2; /* convert to uword address */
150 while (num_in_bytes) {
151 memcpy(&outval, ptr, 4);
152 qat_hal_wr_umem(handle, ae, addr++, 1, &outval);
158 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle,
160 struct icp_qat_uof_batch_init
163 struct icp_qat_uof_batch_init *umem_init;
165 if (!umem_init_header)
167 umem_init = umem_init_header->next;
169 unsigned int addr, *value, size;
172 addr = umem_init->addr;
173 value = umem_init->value;
174 size = umem_init->size;
175 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size);
176 umem_init = umem_init->next;
181 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle,
182 struct icp_qat_uof_batch_init **base)
184 struct icp_qat_uof_batch_init *umem_init;
188 struct icp_qat_uof_batch_init *pre;
191 umem_init = umem_init->next;
197 static int qat_uclo_parse_num(char *str, unsigned int *num)
200 unsigned long ae = 0;
203 strncpy(buf, str, 15);
204 for (i = 0; i < 16; i++) {
205 if (!isdigit(buf[i])) {
210 if ((kstrtoul(buf, 10, &ae)))
213 *num = (unsigned int)ae;
217 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle,
218 struct icp_qat_uof_initmem *init_mem,
219 unsigned int size_range, unsigned int *ae)
221 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
224 if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) {
225 pr_err("QAT: initmem is out of range");
228 if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) {
229 pr_err("QAT: Memory scope for init_mem error\n");
232 str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name);
234 pr_err("QAT: AE name assigned in UOF init table is NULL\n");
237 if (qat_uclo_parse_num(str, ae)) {
238 pr_err("QAT: Parse num for AE number failed\n");
241 if (*ae >= ICP_QAT_UCLO_MAX_AE) {
242 pr_err("QAT: ae %d out of range\n", *ae);
248 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle
249 *handle, struct icp_qat_uof_initmem
250 *init_mem, unsigned int ae,
251 struct icp_qat_uof_batch_init
254 struct icp_qat_uof_batch_init *init_header, *tail;
255 struct icp_qat_uof_batch_init *mem_init, *tail_old;
256 struct icp_qat_uof_memvar_attr *mem_val_attr;
257 unsigned int i, flag = 0;
260 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem +
261 sizeof(struct icp_qat_uof_initmem));
263 init_header = *init_tab_base;
265 init_header = kzalloc(sizeof(*init_header), GFP_KERNEL);
268 init_header->size = 1;
269 *init_tab_base = init_header;
272 tail_old = init_header;
273 while (tail_old->next)
274 tail_old = tail_old->next;
276 for (i = 0; i < init_mem->val_attr_num; i++) {
277 mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL);
281 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte;
282 mem_init->value = &mem_val_attr->value;
284 mem_init->next = NULL;
285 tail->next = mem_init;
287 init_header->size += qat_hal_get_ins_num();
292 /* Do not free the list head unless we allocated it. */
293 tail_old = tail_old->next;
295 kfree(*init_tab_base);
296 *init_tab_base = NULL;
300 mem_init = tail_old->next;
307 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle,
308 struct icp_qat_uof_initmem *init_mem)
310 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
313 if (qat_uclo_fetch_initmem_ae(handle, init_mem,
314 ICP_QAT_UCLO_MAX_LMEM_REG, &ae))
316 if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
317 &obj_handle->lm_init_tab[ae]))
322 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle,
323 struct icp_qat_uof_initmem *init_mem)
325 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
326 unsigned int ae, ustore_size, uaddr, i;
328 ustore_size = obj_handle->ustore_phy_size;
329 if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae))
331 if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
332 &obj_handle->umem_init_tab[ae]))
334 /* set the highest ustore address referenced */
335 uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2;
336 for (i = 0; i < obj_handle->ae_data[ae].slice_num; i++) {
337 if (obj_handle->ae_data[ae].ae_slices[i].
338 encap_image->uwords_num < uaddr)
339 obj_handle->ae_data[ae].ae_slices[i].
340 encap_image->uwords_num = uaddr;
345 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle,
346 struct icp_qat_uof_initmem *init_mem)
348 switch (init_mem->region) {
349 case ICP_QAT_UOF_LMEM_REGION:
350 if (qat_uclo_init_lmem_seg(handle, init_mem))
353 case ICP_QAT_UOF_UMEM_REGION:
354 if (qat_uclo_init_umem_seg(handle, init_mem))
358 pr_err("QAT: initmem region error. region type=0x%x\n",
365 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle,
366 struct icp_qat_uclo_encapme *image)
369 struct icp_qat_uclo_encap_page *page;
370 struct icp_qat_uof_image *uof_image;
372 unsigned int ustore_size;
373 unsigned int patt_pos;
374 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
377 uof_image = image->img_ptr;
378 fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64),
382 for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++)
383 memcpy(&fill_data[i], &uof_image->fill_pattern,
387 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
388 if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned))
390 ustore_size = obj_handle->ae_data[ae].eff_ustore_size;
391 patt_pos = page->beg_addr_p + page->micro_words_num;
393 qat_hal_wr_uwords(handle, (unsigned char)ae, 0,
394 page->beg_addr_p, &fill_data[0]);
395 qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos,
396 ustore_size - patt_pos + 1,
397 &fill_data[page->beg_addr_p]);
403 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle)
406 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
407 struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem;
409 for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) {
410 if (initmem->num_in_bytes) {
411 if (qat_uclo_init_ae_memory(handle, initmem))
414 initmem = (struct icp_qat_uof_initmem *)((uintptr_t)(
416 sizeof(struct icp_qat_uof_initmem)) +
417 (sizeof(struct icp_qat_uof_memvar_attr) *
418 initmem->val_attr_num));
420 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
421 if (qat_hal_batch_wr_lm(handle, ae,
422 obj_handle->lm_init_tab[ae])) {
423 pr_err("QAT: fail to batch init lmem for AE %d\n", ae);
426 qat_uclo_cleanup_batch_init_list(handle,
427 &obj_handle->lm_init_tab[ae]);
428 qat_uclo_batch_wr_umem(handle, ae,
429 obj_handle->umem_init_tab[ae]);
430 qat_uclo_cleanup_batch_init_list(handle,
437 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr,
438 char *chunk_id, void *cur)
441 struct icp_qat_uof_chunkhdr *chunk_hdr =
442 (struct icp_qat_uof_chunkhdr *)
443 ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr));
445 for (i = 0; i < obj_hdr->num_chunks; i++) {
446 if ((cur < (void *)&chunk_hdr[i]) &&
447 !strncmp(chunk_hdr[i].chunk_id, chunk_id,
448 ICP_QAT_UOF_OBJID_LEN)) {
449 return &chunk_hdr[i];
455 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch)
458 unsigned int topbit = 1 << 0xF;
459 unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch);
461 reg ^= inbyte << 0x8;
462 for (i = 0; i < 0x8; i++) {
464 reg = (reg << 1) ^ 0x1021;
471 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num)
473 unsigned int chksum = 0;
477 chksum = qat_uclo_calc_checksum(chksum, *ptr++);
481 static struct icp_qat_uclo_objhdr *
482 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr,
485 struct icp_qat_uof_filechunkhdr *file_chunk;
486 struct icp_qat_uclo_objhdr *obj_hdr;
490 file_chunk = (struct icp_qat_uof_filechunkhdr *)
491 (buf + sizeof(struct icp_qat_uof_filehdr));
492 for (i = 0; i < file_hdr->num_chunks; i++) {
493 if (!strncmp(file_chunk->chunk_id, chunk_id,
494 ICP_QAT_UOF_OBJID_LEN)) {
495 chunk = buf + file_chunk->offset;
496 if (file_chunk->checksum != qat_uclo_calc_str_checksum(
497 chunk, file_chunk->size))
499 obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL);
502 obj_hdr->file_buff = chunk;
503 obj_hdr->checksum = file_chunk->checksum;
504 obj_hdr->size = file_chunk->size;
513 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj,
514 struct icp_qat_uof_image *image)
516 struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab;
517 struct icp_qat_uof_objtable *neigh_reg_tab;
518 struct icp_qat_uof_code_page *code_page;
520 code_page = (struct icp_qat_uof_code_page *)
521 ((char *)image + sizeof(struct icp_qat_uof_image));
522 uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
523 code_page->uc_var_tab_offset);
524 imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
525 code_page->imp_var_tab_offset);
526 imp_expr_tab = (struct icp_qat_uof_objtable *)
527 (encap_uof_obj->beg_uof +
528 code_page->imp_expr_tab_offset);
529 if (uc_var_tab->entry_num || imp_var_tab->entry_num ||
530 imp_expr_tab->entry_num) {
531 pr_err("QAT: UOF can't contain imported variable to be parsed\n");
534 neigh_reg_tab = (struct icp_qat_uof_objtable *)
535 (encap_uof_obj->beg_uof +
536 code_page->neigh_reg_tab_offset);
537 if (neigh_reg_tab->entry_num) {
538 pr_err("QAT: UOF can't contain shared control store feature\n");
541 if (image->numpages > 1) {
542 pr_err("QAT: UOF can't contain multiple pages\n");
545 if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) {
546 pr_err("QAT: UOF can't use shared control store feature\n");
549 if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) {
550 pr_err("QAT: UOF can't use reloadable feature\n");
556 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj
558 struct icp_qat_uof_image *img,
559 struct icp_qat_uclo_encap_page *page)
561 struct icp_qat_uof_code_page *code_page;
562 struct icp_qat_uof_code_area *code_area;
563 struct icp_qat_uof_objtable *uword_block_tab;
564 struct icp_qat_uof_uword_block *uwblock;
567 code_page = (struct icp_qat_uof_code_page *)
568 ((char *)img + sizeof(struct icp_qat_uof_image));
569 page->def_page = code_page->def_page;
570 page->page_region = code_page->page_region;
571 page->beg_addr_v = code_page->beg_addr_v;
572 page->beg_addr_p = code_page->beg_addr_p;
573 code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof +
574 code_page->code_area_offset);
575 page->micro_words_num = code_area->micro_words_num;
576 uword_block_tab = (struct icp_qat_uof_objtable *)
577 (encap_uof_obj->beg_uof +
578 code_area->uword_block_tab);
579 page->uwblock_num = uword_block_tab->entry_num;
580 uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab +
581 sizeof(struct icp_qat_uof_objtable));
582 page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock;
583 for (i = 0; i < uword_block_tab->entry_num; i++)
584 page->uwblock[i].micro_words =
585 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset;
588 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle,
589 struct icp_qat_uclo_encapme *ae_uimage,
593 struct icp_qat_uof_chunkhdr *chunk_hdr = NULL;
594 struct icp_qat_uof_image *image;
595 struct icp_qat_uof_objtable *ae_regtab;
596 struct icp_qat_uof_objtable *init_reg_sym_tab;
597 struct icp_qat_uof_objtable *sbreak_tab;
598 struct icp_qat_uof_encap_obj *encap_uof_obj =
599 &obj_handle->encap_uof_obj;
601 for (j = 0; j < max_image; j++) {
602 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
603 ICP_QAT_UOF_IMAG, chunk_hdr);
606 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof +
608 ae_regtab = (struct icp_qat_uof_objtable *)
609 (image->reg_tab_offset +
610 obj_handle->obj_hdr->file_buff);
611 ae_uimage[j].ae_reg_num = ae_regtab->entry_num;
612 ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *)
613 (((char *)ae_regtab) +
614 sizeof(struct icp_qat_uof_objtable));
615 init_reg_sym_tab = (struct icp_qat_uof_objtable *)
616 (image->init_reg_sym_tab +
617 obj_handle->obj_hdr->file_buff);
618 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num;
619 ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *)
620 (((char *)init_reg_sym_tab) +
621 sizeof(struct icp_qat_uof_objtable));
622 sbreak_tab = (struct icp_qat_uof_objtable *)
623 (image->sbreak_tab + obj_handle->obj_hdr->file_buff);
624 ae_uimage[j].sbreak_num = sbreak_tab->entry_num;
625 ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *)
626 (((char *)sbreak_tab) +
627 sizeof(struct icp_qat_uof_objtable));
628 ae_uimage[j].img_ptr = image;
629 if (qat_uclo_check_image_compat(encap_uof_obj, image))
632 kzalloc(sizeof(struct icp_qat_uclo_encap_page),
634 if (!ae_uimage[j].page)
636 qat_uclo_map_image_page(encap_uof_obj, image,
641 for (i = 0; i < j; i++)
642 kfree(ae_uimage[i].page);
646 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae)
650 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
652 for (ae = 0; ae < max_ae; ae++) {
654 (unsigned long *)&handle->hal_handle->ae_mask))
656 for (i = 0; i < obj_handle->uimage_num; i++) {
657 if (!test_bit(ae, (unsigned long *)
658 &obj_handle->ae_uimage[i].img_ptr->ae_assigned))
661 if (qat_uclo_init_ae_data(obj_handle, ae, i))
666 pr_err("QAT: uimage uses AE not set\n");
672 static struct icp_qat_uof_strtable *
673 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr,
674 char *tab_name, struct icp_qat_uof_strtable *str_table)
676 struct icp_qat_uof_chunkhdr *chunk_hdr;
678 chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)
679 obj_hdr->file_buff, tab_name, NULL);
683 memcpy(&str_table->table_len, obj_hdr->file_buff +
684 chunk_hdr->offset, sizeof(str_table->table_len));
685 hdr_size = (char *)&str_table->strings - (char *)str_table;
686 str_table->strings = (uintptr_t)obj_hdr->file_buff +
687 chunk_hdr->offset + hdr_size;
694 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj,
695 struct icp_qat_uclo_init_mem_table *init_mem_tab)
697 struct icp_qat_uof_chunkhdr *chunk_hdr;
699 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
700 ICP_QAT_UOF_IMEM, NULL);
702 memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof +
703 chunk_hdr->offset, sizeof(unsigned int));
704 init_mem_tab->init_mem = (struct icp_qat_uof_initmem *)
705 (encap_uof_obj->beg_uof + chunk_hdr->offset +
706 sizeof(unsigned int));
711 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
713 switch (handle->pci_dev->device) {
714 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
715 return ICP_QAT_AC_895XCC_DEV_TYPE;
716 case PCI_DEVICE_ID_INTEL_QAT_C62X:
717 return ICP_QAT_AC_C62X_DEV_TYPE;
718 case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
719 return ICP_QAT_AC_C3XXX_DEV_TYPE;
721 pr_err("QAT: unsupported device 0x%x\n",
722 handle->pci_dev->device);
727 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle)
729 unsigned int maj_ver, prod_type = obj_handle->prod_type;
731 if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) {
732 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n",
733 obj_handle->encap_uof_obj.obj_hdr->ac_dev_type,
737 maj_ver = obj_handle->prod_rev & 0xff;
738 if ((obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver) ||
739 (obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver)) {
740 pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver);
746 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle,
747 unsigned char ae, unsigned char ctx_mask,
748 enum icp_qat_uof_regtype reg_type,
749 unsigned short reg_addr, unsigned int value)
758 return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type,
770 return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type,
778 return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type,
781 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value);
783 pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type);
789 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle,
791 struct icp_qat_uclo_encapme *encap_ae)
794 unsigned char ctx_mask;
795 struct icp_qat_uof_init_regsym *init_regsym;
797 if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) ==
798 ICP_QAT_UCLO_MAX_CTX)
803 for (i = 0; i < encap_ae->init_regsym_num; i++) {
804 unsigned int exp_res;
806 init_regsym = &encap_ae->init_regsym[i];
807 exp_res = init_regsym->value;
808 switch (init_regsym->init_type) {
809 case ICP_QAT_UOF_INIT_REG:
810 qat_uclo_init_reg(handle, ae, ctx_mask,
811 (enum icp_qat_uof_regtype)
812 init_regsym->reg_type,
813 (unsigned short)init_regsym->reg_addr,
816 case ICP_QAT_UOF_INIT_REG_CTX:
817 /* check if ctx is appropriate for the ctxMode */
818 if (!((1 << init_regsym->ctx) & ctx_mask)) {
819 pr_err("QAT: invalid ctx num = 0x%x\n",
823 qat_uclo_init_reg(handle, ae,
825 (1 << init_regsym->ctx),
826 (enum icp_qat_uof_regtype)
827 init_regsym->reg_type,
828 (unsigned short)init_regsym->reg_addr,
831 case ICP_QAT_UOF_INIT_EXPR:
832 pr_err("QAT: INIT_EXPR feature not supported\n");
834 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP:
835 pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n");
844 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle)
846 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
849 if (obj_handle->global_inited)
851 if (obj_handle->init_mem_tab.entry_num) {
852 if (qat_uclo_init_memory(handle)) {
853 pr_err("QAT: initialize memory failed\n");
857 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
858 for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) {
859 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
861 if (qat_uclo_init_reg_sym(handle, ae,
862 obj_handle->ae_data[ae].
863 ae_slices[s].encap_image))
867 obj_handle->global_inited = 1;
871 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle)
873 unsigned char ae, nn_mode, s;
874 struct icp_qat_uof_image *uof_image;
875 struct icp_qat_uclo_aedata *ae_data;
876 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
878 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
880 (unsigned long *)&handle->hal_handle->ae_mask))
882 ae_data = &obj_handle->ae_data[ae];
883 for (s = 0; s < min_t(unsigned int, ae_data->slice_num,
884 ICP_QAT_UCLO_MAX_CTX); s++) {
885 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
887 uof_image = ae_data->ae_slices[s].encap_image->img_ptr;
888 if (qat_hal_set_ae_ctx_mode(handle, ae,
889 (char)ICP_QAT_CTX_MODE
890 (uof_image->ae_mode))) {
891 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n");
894 nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
895 if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) {
896 pr_err("QAT: qat_hal_set_ae_nn_mode error\n");
899 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0,
900 (char)ICP_QAT_LOC_MEM0_MODE
901 (uof_image->ae_mode))) {
902 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n");
905 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1,
906 (char)ICP_QAT_LOC_MEM1_MODE
907 (uof_image->ae_mode))) {
908 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n");
916 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle)
918 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
919 struct icp_qat_uclo_encapme *image;
922 for (a = 0; a < obj_handle->uimage_num; a++) {
923 image = &obj_handle->ae_uimage[a];
924 image->uwords_num = image->page->beg_addr_p +
925 image->page->micro_words_num;
929 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
931 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
934 obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff;
935 obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *)
936 obj_handle->obj_hdr->file_buff;
937 obj_handle->uword_in_bytes = 6;
938 obj_handle->prod_type = qat_uclo_get_dev_type(handle);
939 obj_handle->prod_rev = PID_MAJOR_REV |
940 (PID_MINOR_REV & handle->hal_handle->revision_id);
941 if (qat_uclo_check_uof_compat(obj_handle)) {
942 pr_err("QAT: UOF incompatible\n");
945 obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64),
947 if (!obj_handle->uword_buf)
949 obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE;
950 if (!obj_handle->obj_hdr->file_buff ||
951 !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT,
952 &obj_handle->str_table)) {
953 pr_err("QAT: UOF doesn't have effective images\n");
956 obj_handle->uimage_num =
957 qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage,
958 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX);
959 if (!obj_handle->uimage_num)
961 if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) {
962 pr_err("QAT: Bad object\n");
963 goto out_check_uof_aemask_err;
965 qat_uclo_init_uword_num(handle);
966 qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj,
967 &obj_handle->init_mem_tab);
968 if (qat_uclo_set_ae_mode(handle))
969 goto out_check_uof_aemask_err;
971 out_check_uof_aemask_err:
972 for (ae = 0; ae < obj_handle->uimage_num; ae++)
973 kfree(obj_handle->ae_uimage[ae].page);
975 kfree(obj_handle->uword_buf);
979 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle,
980 struct icp_qat_suof_filehdr *suof_ptr,
983 unsigned int check_sum = 0;
984 unsigned int min_ver_offset = 0;
985 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
987 suof_handle->file_id = ICP_QAT_SUOF_FID;
988 suof_handle->suof_buf = (char *)suof_ptr;
989 suof_handle->suof_size = suof_size;
990 min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr,
992 check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver,
994 if (check_sum != suof_ptr->check_sum) {
995 pr_err("QAT: incorrect SUOF checksum\n");
998 suof_handle->check_sum = suof_ptr->check_sum;
999 suof_handle->min_ver = suof_ptr->min_ver;
1000 suof_handle->maj_ver = suof_ptr->maj_ver;
1001 suof_handle->fw_type = suof_ptr->fw_type;
1005 static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle,
1006 struct icp_qat_suof_img_hdr *suof_img_hdr,
1007 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1009 struct icp_qat_simg_ae_mode *ae_mode;
1010 struct icp_qat_suof_objhdr *suof_objhdr;
1012 suof_img_hdr->simg_buf = (suof_handle->suof_buf +
1013 suof_chunk_hdr->offset +
1014 sizeof(*suof_objhdr));
1015 suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t)
1016 (suof_handle->suof_buf +
1017 suof_chunk_hdr->offset))->img_length;
1019 suof_img_hdr->css_header = suof_img_hdr->simg_buf;
1020 suof_img_hdr->css_key = (suof_img_hdr->css_header +
1021 sizeof(struct icp_qat_css_hdr));
1022 suof_img_hdr->css_signature = suof_img_hdr->css_key +
1023 ICP_QAT_CSS_FWSK_MODULUS_LEN +
1024 ICP_QAT_CSS_FWSK_EXPONENT_LEN;
1025 suof_img_hdr->css_simg = suof_img_hdr->css_signature +
1026 ICP_QAT_CSS_SIGNATURE_LEN;
1028 ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg);
1029 suof_img_hdr->ae_mask = ae_mode->ae_mask;
1030 suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name;
1031 suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data;
1032 suof_img_hdr->fw_type = ae_mode->fw_type;
1036 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle,
1037 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1039 char **sym_str = (char **)&suof_handle->sym_str;
1040 unsigned int *sym_size = &suof_handle->sym_size;
1041 struct icp_qat_suof_strtable *str_table_obj;
1043 *sym_size = *(unsigned int *)(uintptr_t)
1044 (suof_chunk_hdr->offset + suof_handle->suof_buf);
1045 *sym_str = (char *)(uintptr_t)
1046 (suof_handle->suof_buf + suof_chunk_hdr->offset +
1047 sizeof(str_table_obj->tab_length));
1050 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle,
1051 struct icp_qat_suof_img_hdr *img_hdr)
1053 struct icp_qat_simg_ae_mode *img_ae_mode = NULL;
1054 unsigned int prod_rev, maj_ver, prod_type;
1056 prod_type = qat_uclo_get_dev_type(handle);
1057 img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg;
1058 prod_rev = PID_MAJOR_REV |
1059 (PID_MINOR_REV & handle->hal_handle->revision_id);
1060 if (img_ae_mode->dev_type != prod_type) {
1061 pr_err("QAT: incompatible product type %x\n",
1062 img_ae_mode->dev_type);
1065 maj_ver = prod_rev & 0xff;
1066 if ((maj_ver > img_ae_mode->devmax_ver) ||
1067 (maj_ver < img_ae_mode->devmin_ver)) {
1068 pr_err("QAT: incompatible device majver 0x%x\n", maj_ver);
1074 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle)
1076 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
1078 kfree(sobj_handle->img_table.simg_hdr);
1079 sobj_handle->img_table.simg_hdr = NULL;
1080 kfree(handle->sobj_handle);
1081 handle->sobj_handle = NULL;
1084 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr,
1085 unsigned int img_id, unsigned int num_simgs)
1087 struct icp_qat_suof_img_hdr img_header;
1089 if (img_id != num_simgs - 1) {
1090 memcpy(&img_header, &suof_img_hdr[num_simgs - 1],
1091 sizeof(*suof_img_hdr));
1092 memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id],
1093 sizeof(*suof_img_hdr));
1094 memcpy(&suof_img_hdr[img_id], &img_header,
1095 sizeof(*suof_img_hdr));
1099 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle,
1100 struct icp_qat_suof_filehdr *suof_ptr,
1103 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1104 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL;
1105 struct icp_qat_suof_img_hdr *suof_img_hdr = NULL;
1106 int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE;
1108 struct icp_qat_suof_img_hdr img_header;
1110 if (!suof_ptr || (suof_size == 0)) {
1111 pr_err("QAT: input parameter SUOF pointer/size is NULL\n");
1114 if (qat_uclo_check_suof_format(suof_ptr))
1116 ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size);
1119 suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)
1120 ((uintptr_t)suof_ptr + sizeof(*suof_ptr));
1122 qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr);
1123 suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1;
1125 if (suof_handle->img_table.num_simgs != 0) {
1126 suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs,
1131 suof_handle->img_table.simg_hdr = suof_img_hdr;
1134 for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
1135 qat_uclo_map_simg(handle->sobj_handle, &suof_img_hdr[i],
1136 &suof_chunk_hdr[1 + i]);
1137 ret = qat_uclo_check_simg_compat(handle,
1141 if ((suof_img_hdr[i].ae_mask & 0x1) != 0)
1144 qat_uclo_tail_img(suof_img_hdr, ae0_img,
1145 suof_handle->img_table.num_simgs);
1149 #define ADD_ADDR(high, low) ((((u64)high) << 32) + low)
1150 #define BITS_IN_DWORD 32
1152 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle,
1153 struct icp_qat_fw_auth_desc *desc)
1155 unsigned int fcu_sts, retry = 0;
1158 bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low)
1159 - sizeof(struct icp_qat_auth_chunk);
1160 SET_CAP_CSR(handle, FCU_DRAM_ADDR_HI, (bus_addr >> BITS_IN_DWORD));
1161 SET_CAP_CSR(handle, FCU_DRAM_ADDR_LO, bus_addr);
1162 SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_AUTH);
1165 msleep(FW_AUTH_WAIT_PERIOD);
1166 fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
1167 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL)
1169 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1))
1170 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE)
1172 } while (retry++ < FW_AUTH_MAX_RETRY);
1174 pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n",
1175 fcu_sts & FCU_AUTH_STS_MASK, retry);
1179 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
1180 struct icp_firml_dram_desc *dram_desc,
1186 vptr = dma_alloc_coherent(&handle->pci_dev->dev,
1187 size, &ptr, GFP_KERNEL);
1190 dram_desc->dram_base_addr_v = vptr;
1191 dram_desc->dram_bus_addr = ptr;
1192 dram_desc->dram_size = size;
1196 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle,
1197 struct icp_firml_dram_desc *dram_desc)
1199 dma_free_coherent(&handle->pci_dev->dev,
1200 (size_t)(dram_desc->dram_size),
1201 (dram_desc->dram_base_addr_v),
1202 dram_desc->dram_bus_addr);
1203 memset(dram_desc, 0, sizeof(*dram_desc));
1206 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle,
1207 struct icp_qat_fw_auth_desc **desc)
1209 struct icp_firml_dram_desc dram_desc;
1211 dram_desc.dram_base_addr_v = *desc;
1212 dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *)
1213 (*desc))->chunk_bus_addr;
1214 dram_desc.dram_size = ((struct icp_qat_auth_chunk *)
1215 (*desc))->chunk_size;
1216 qat_uclo_simg_free(handle, &dram_desc);
1219 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle,
1220 char *image, unsigned int size,
1221 struct icp_qat_fw_auth_desc **desc)
1223 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image;
1224 struct icp_qat_fw_auth_desc *auth_desc;
1225 struct icp_qat_auth_chunk *auth_chunk;
1226 u64 virt_addr, bus_addr, virt_base;
1227 unsigned int length, simg_offset = sizeof(*auth_chunk);
1228 struct icp_firml_dram_desc img_desc;
1230 if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) {
1231 pr_err("QAT: error, input image size overflow %d\n", size);
1234 length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ?
1235 ICP_QAT_CSS_AE_SIMG_LEN + simg_offset :
1236 size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset;
1237 if (qat_uclo_simg_alloc(handle, &img_desc, length)) {
1238 pr_err("QAT: error, allocate continuous dram fail\n");
1242 auth_chunk = img_desc.dram_base_addr_v;
1243 auth_chunk->chunk_size = img_desc.dram_size;
1244 auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr;
1245 virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset;
1246 bus_addr = img_desc.dram_bus_addr + simg_offset;
1247 auth_desc = img_desc.dram_base_addr_v;
1248 auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1249 auth_desc->css_hdr_low = (unsigned int)bus_addr;
1250 virt_addr = virt_base;
1252 memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr));
1254 bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) +
1256 virt_addr = virt_addr + sizeof(*css_hdr);
1258 auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1259 auth_desc->fwsk_pub_low = (unsigned int)bus_addr;
1261 memcpy((void *)(uintptr_t)virt_addr,
1262 (void *)(image + sizeof(*css_hdr)),
1263 ICP_QAT_CSS_FWSK_MODULUS_LEN);
1265 memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN),
1266 0, ICP_QAT_CSS_FWSK_PAD_LEN);
1269 memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN +
1270 ICP_QAT_CSS_FWSK_PAD_LEN),
1271 (void *)(image + sizeof(*css_hdr) +
1272 ICP_QAT_CSS_FWSK_MODULUS_LEN),
1273 sizeof(unsigned int));
1276 bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high,
1277 auth_desc->fwsk_pub_low) +
1278 ICP_QAT_CSS_FWSK_PUB_LEN;
1279 virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN;
1280 auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1281 auth_desc->signature_low = (unsigned int)bus_addr;
1283 memcpy((void *)(uintptr_t)virt_addr,
1284 (void *)(image + sizeof(*css_hdr) +
1285 ICP_QAT_CSS_FWSK_MODULUS_LEN +
1286 ICP_QAT_CSS_FWSK_EXPONENT_LEN),
1287 ICP_QAT_CSS_SIGNATURE_LEN);
1289 bus_addr = ADD_ADDR(auth_desc->signature_high,
1290 auth_desc->signature_low) +
1291 ICP_QAT_CSS_SIGNATURE_LEN;
1292 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN;
1294 auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1295 auth_desc->img_low = (unsigned int)bus_addr;
1296 auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET;
1297 memcpy((void *)(uintptr_t)virt_addr,
1298 (void *)(image + ICP_QAT_AE_IMG_OFFSET),
1299 auth_desc->img_len);
1300 virt_addr = virt_base;
1302 if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type ==
1304 auth_desc->img_ae_mode_data_high = auth_desc->img_high;
1305 auth_desc->img_ae_mode_data_low = auth_desc->img_low;
1306 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high,
1307 auth_desc->img_ae_mode_data_low) +
1308 sizeof(struct icp_qat_simg_ae_mode);
1310 auth_desc->img_ae_init_data_high = (unsigned int)
1311 (bus_addr >> BITS_IN_DWORD);
1312 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr;
1313 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN;
1314 auth_desc->img_ae_insts_high = (unsigned int)
1315 (bus_addr >> BITS_IN_DWORD);
1316 auth_desc->img_ae_insts_low = (unsigned int)bus_addr;
1318 auth_desc->img_ae_insts_high = auth_desc->img_high;
1319 auth_desc->img_ae_insts_low = auth_desc->img_low;
1325 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle,
1326 struct icp_qat_fw_auth_desc *desc)
1329 unsigned int fcu_sts;
1330 struct icp_qat_simg_ae_mode *virt_addr;
1331 unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
1333 virt_addr = (void *)((uintptr_t)desc +
1334 sizeof(struct icp_qat_auth_chunk) +
1335 sizeof(struct icp_qat_css_hdr) +
1336 ICP_QAT_CSS_FWSK_PUB_LEN +
1337 ICP_QAT_CSS_SIGNATURE_LEN);
1338 for (i = 0; i < handle->hal_handle->ae_max_num; i++) {
1341 if (!((virt_addr->ae_mask >> i) & 0x1))
1343 if (qat_hal_check_ae_active(handle, i)) {
1344 pr_err("QAT: AE %d is active\n", i);
1347 SET_CAP_CSR(handle, FCU_CONTROL,
1348 (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS)));
1351 msleep(FW_AUTH_WAIT_PERIOD);
1352 fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
1353 if (((fcu_sts & FCU_AUTH_STS_MASK) ==
1354 FCU_STS_LOAD_DONE) &&
1355 ((fcu_sts >> fcu_loaded_ae_pos) & (1 << i)))
1357 } while (retry++ < FW_AUTH_MAX_RETRY);
1358 if (retry > FW_AUTH_MAX_RETRY) {
1359 pr_err("QAT: firmware load failed timeout %x\n", retry);
1366 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle,
1367 void *addr_ptr, int mem_size)
1369 struct icp_qat_suof_handle *suof_handle;
1371 suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL);
1374 handle->sobj_handle = suof_handle;
1375 if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) {
1376 qat_uclo_del_suof(handle);
1377 pr_err("QAT: map SUOF failed\n");
1383 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
1384 void *addr_ptr, int mem_size)
1386 struct icp_qat_fw_auth_desc *desc = NULL;
1389 if (handle->fw_auth) {
1390 if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc))
1391 status = qat_uclo_auth_fw(handle, desc);
1392 qat_uclo_ummap_auth_fw(handle, &desc);
1394 if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_C3XXX) {
1395 pr_err("QAT: C3XXX doesn't support unsigned MMP\n");
1398 qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size);
1403 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle,
1404 void *addr_ptr, int mem_size)
1406 struct icp_qat_uof_filehdr *filehdr;
1407 struct icp_qat_uclo_objhandle *objhdl;
1409 objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL);
1412 objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL);
1413 if (!objhdl->obj_buf)
1414 goto out_objbuf_err;
1415 filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf;
1416 if (qat_uclo_check_uof_format(filehdr))
1417 goto out_objhdr_err;
1418 objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr,
1420 if (!objhdl->obj_hdr) {
1421 pr_err("QAT: object file chunk is null\n");
1422 goto out_objhdr_err;
1424 handle->obj_handle = objhdl;
1425 if (qat_uclo_parse_uof_obj(handle))
1426 goto out_overlay_obj_err;
1429 out_overlay_obj_err:
1430 handle->obj_handle = NULL;
1431 kfree(objhdl->obj_hdr);
1433 kfree(objhdl->obj_buf);
1439 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle,
1440 void *addr_ptr, int mem_size)
1442 BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >=
1443 (sizeof(handle->hal_handle->ae_mask) * 8));
1445 if (!handle || !addr_ptr || mem_size < 24)
1448 return (handle->fw_auth) ?
1449 qat_uclo_map_suof_obj(handle, addr_ptr, mem_size) :
1450 qat_uclo_map_uof_obj(handle, addr_ptr, mem_size);
1453 void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle)
1455 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1458 if (handle->sobj_handle)
1459 qat_uclo_del_suof(handle);
1463 kfree(obj_handle->uword_buf);
1464 for (a = 0; a < obj_handle->uimage_num; a++)
1465 kfree(obj_handle->ae_uimage[a].page);
1467 for (a = 0; a < handle->hal_handle->ae_max_num; a++)
1468 qat_uclo_free_ae_data(&obj_handle->ae_data[a]);
1470 kfree(obj_handle->obj_hdr);
1471 kfree(obj_handle->obj_buf);
1473 handle->obj_handle = NULL;
1476 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle,
1477 struct icp_qat_uclo_encap_page *encap_page,
1478 u64 *uword, unsigned int addr_p,
1479 unsigned int raddr, u64 fill)
1488 for (i = 0; i < encap_page->uwblock_num; i++) {
1489 if (raddr >= encap_page->uwblock[i].start_addr &&
1490 raddr <= encap_page->uwblock[i].start_addr +
1491 encap_page->uwblock[i].words_num - 1) {
1492 raddr -= encap_page->uwblock[i].start_addr;
1493 raddr *= obj_handle->uword_in_bytes;
1494 memcpy(&uwrd, (void *)(((uintptr_t)
1495 encap_page->uwblock[i].micro_words) + raddr),
1496 obj_handle->uword_in_bytes);
1497 uwrd = uwrd & 0xbffffffffffull;
1501 if (*uword == INVLD_UWORD)
1505 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle,
1506 struct icp_qat_uclo_encap_page
1507 *encap_page, unsigned int ae)
1509 unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen;
1510 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1513 /* load the page starting at appropriate ustore address */
1514 /* get fill-pattern from an image -- they are all the same */
1515 memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern,
1517 uw_physical_addr = encap_page->beg_addr_p;
1518 uw_relative_addr = 0;
1519 words_num = encap_page->micro_words_num;
1521 if (words_num < UWORD_CPYBUF_SIZE)
1524 cpylen = UWORD_CPYBUF_SIZE;
1526 /* load the buffer */
1527 for (i = 0; i < cpylen; i++)
1528 qat_uclo_fill_uwords(obj_handle, encap_page,
1529 &obj_handle->uword_buf[i],
1530 uw_physical_addr + i,
1531 uw_relative_addr + i, fill_pat);
1533 /* copy the buffer to ustore */
1534 qat_hal_wr_uwords(handle, (unsigned char)ae,
1535 uw_physical_addr, cpylen,
1536 obj_handle->uword_buf);
1538 uw_physical_addr += cpylen;
1539 uw_relative_addr += cpylen;
1540 words_num -= cpylen;
1544 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle,
1545 struct icp_qat_uof_image *image)
1547 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1548 unsigned int ctx_mask, s;
1549 struct icp_qat_uclo_page *page;
1553 if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX)
1557 /* load the default page and set assigned CTX PC
1558 * to the entrypoint address */
1559 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
1560 if (!test_bit(ae, (unsigned long *)&image->ae_assigned))
1562 /* find the slice to which this image is assigned */
1563 for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) {
1564 if (image->ctx_assigned & obj_handle->ae_data[ae].
1565 ae_slices[s].ctx_mask_assigned)
1568 if (s >= obj_handle->ae_data[ae].slice_num)
1570 page = obj_handle->ae_data[ae].ae_slices[s].page;
1571 if (!page->encap_page->def_page)
1573 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae);
1575 page = obj_handle->ae_data[ae].ae_slices[s].page;
1576 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++)
1577 obj_handle->ae_data[ae].ae_slices[s].cur_page[ctx] =
1578 (ctx_mask & (1 << ctx)) ? page : NULL;
1579 qat_hal_set_live_ctx(handle, (unsigned char)ae,
1580 image->ctx_assigned);
1581 qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned,
1582 image->entry_address);
1586 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle)
1589 struct icp_qat_fw_auth_desc *desc = NULL;
1590 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
1591 struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr;
1593 for (i = 0; i < sobj_handle->img_table.num_simgs; i++) {
1594 if (qat_uclo_map_auth_fw(handle,
1595 (char *)simg_hdr[i].simg_buf,
1597 (simg_hdr[i].simg_len),
1600 if (qat_uclo_auth_fw(handle, desc))
1602 if (qat_uclo_load_fw(handle, desc))
1604 qat_uclo_ummap_auth_fw(handle, &desc);
1608 qat_uclo_ummap_auth_fw(handle, &desc);
1612 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle)
1614 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1617 if (qat_uclo_init_globals(handle))
1619 for (i = 0; i < obj_handle->uimage_num; i++) {
1620 if (!obj_handle->ae_uimage[i].img_ptr)
1622 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i]))
1624 qat_uclo_wr_uimage_page(handle,
1625 obj_handle->ae_uimage[i].img_ptr);
1630 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle)
1632 return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) :
1633 qat_uclo_wr_uof_img(handle);