/* Qualcomm Crypto Engine driver. * * Copyright (c) 2010-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and * only version 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "qce.h" #include "qcryptohw_30.h" #include "qce_ota.h" /* ADM definitions */ #define LI_SG_CMD (1 << 31) /* last index in the scatter gather cmd */ #define SRC_INDEX_SG_CMD(index) ((index & 0x3fff) << 16) #define DST_INDEX_SG_CMD(index) (index & 0x3fff) #define ADM_DESC_LAST (1 << 31) /* Data xfer between DM and CE in blocks of 16 bytes */ #define ADM_CE_BLOCK_SIZE 16 #define QCE_FIFO_SIZE 0x8000 /* Data xfer between DM and CE in blocks of 64 bytes */ #define ADM_SHA_BLOCK_SIZE 64 #define ADM_DESC_LENGTH_MASK 0xffff #define ADM_DESC_LENGTH(x) (x & ADM_DESC_LENGTH_MASK) struct dmov_desc { uint32_t addr; uint32_t len; }; #define ADM_STATUS_OK 0x80000002 /* Misc definitions */ /* QCE max number of descriptor in a descriptor list */ #define QCE_MAX_NUM_DESC 128 /* State of DM channel */ enum qce_chan_st_enum { QCE_CHAN_STATE_IDLE = 0, QCE_CHAN_STATE_IN_PROG = 1, QCE_CHAN_STATE_COMP = 2, QCE_CHAN_STATE_LAST }; /* * CE HW device structure. * Each engine has an instance of the structure. * Each engine can only handle one crypto operation at one time. It is up to * the sw above to ensure single threading of operation on an engine. */ struct qce_device { struct device *pdev; /* Handle to platform_device structure */ unsigned char *coh_vmem; /* Allocated coherent virtual memory */ dma_addr_t coh_pmem; /* Allocated coherent physical memory */ void __iomem *iobase; /* Virtual io base of CE HW */ unsigned int phy_iobase; /* Physical io base of CE HW */ struct clk *ce_clk; /* Handle to CE clk */ unsigned int crci_in; /* CRCI for CE DM IN Channel */ unsigned int crci_out; /* CRCI for CE DM OUT Channel */ unsigned int crci_hash; /* CRCI for CE HASH */ unsigned int chan_ce_in; /* ADM channel used for CE input * and auth result if authentication * only operation. */ unsigned int chan_ce_out; /* ADM channel used for CE output, and icv for esp */ unsigned int *cmd_pointer_list_ce_in; dma_addr_t phy_cmd_pointer_list_ce_in; unsigned int *cmd_pointer_list_ce_out; dma_addr_t phy_cmd_pointer_list_ce_out; unsigned char *cmd_list_ce_in; dma_addr_t phy_cmd_list_ce_in; unsigned char *cmd_list_ce_out; dma_addr_t phy_cmd_list_ce_out; struct dmov_desc *ce_out_src_desc; dma_addr_t phy_ce_out_src_desc; struct dmov_desc *ce_out_dst_desc; dma_addr_t phy_ce_out_dst_desc; struct dmov_desc *ce_in_src_desc; dma_addr_t phy_ce_in_src_desc; struct dmov_desc *ce_in_dst_desc; dma_addr_t phy_ce_in_dst_desc; unsigned char *ce_out_ignore; dma_addr_t phy_ce_out_ignore; unsigned char *ce_pad; dma_addr_t phy_ce_pad; struct msm_dmov_cmd *chan_ce_in_cmd; struct msm_dmov_cmd *chan_ce_out_cmd; uint32_t ce_out_ignore_size; int ce_out_dst_desc_index; int ce_in_dst_desc_index; int ce_out_src_desc_index; int ce_in_src_desc_index; enum qce_chan_st_enum chan_ce_in_state; /* chan ce_in state */ enum qce_chan_st_enum chan_ce_out_state; /* chan ce_out state */ int chan_ce_in_status; /* chan ce_in status */ int chan_ce_out_status; /* chan ce_out status */ unsigned char *dig_result; dma_addr_t phy_dig_result; /* cached aes key */ uint32_t aeskey[AES256_KEY_SIZE/sizeof(uint32_t)]; uint32_t aes_key_size; /* cached aes key size in bytes */ int fastaes; /* ce supports fast aes */ int hmac; /* ce support hmac-sha1 */ bool ota; /* ce support ota */ qce_comp_func_ptr_t qce_cb; /* qce callback function pointer */ int assoc_nents; int src_nents; int dst_nents; void *areq; enum qce_cipher_mode_enum mode; dma_addr_t phy_iv_in; dma_addr_t phy_ota_src; dma_addr_t phy_ota_dst; unsigned int ota_size; int err; }; /* Standard initialization vector for SHA-1, source: FIPS 180-2 */ static uint32_t _std_init_vector_sha1[] = { 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0 }; /* Standard initialization vector for SHA-256, source: FIPS 180-2 */ static uint32_t _std_init_vector_sha256[] = { 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 }; /* Source: FIPS 197, Figure 7. S-box: substitution values for the byte xy */ static const uint32_t _s_box[256] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 }; /* * Source: FIPS 197, Sec 5.2 Key Expansion, Figure 11. Pseudo Code for Key * Expansion. */ static void _aes_expand_key_schedule(uint32_t keysize, uint32_t *AES_KEY, uint32_t *AES_RND_KEY) { uint32_t i; uint32_t Nk; uint32_t Nr, rot_data; uint32_t Rcon = 0x01000000; uint32_t temp; uint32_t data_in; uint32_t MSB_store; uint32_t byte_for_sub; uint32_t word_sub[4]; switch (keysize) { case 192: Nk = 6; Nr = 12; break; case 256: Nk = 8; Nr = 14; break; case 128: default: /* default to AES128 */ Nk = 4; Nr = 10; break; } /* key expansion */ i = 0; while (i < Nk) { AES_RND_KEY[i] = AES_KEY[i]; i = i + 1; } i = Nk; while (i < (4 * (Nr + 1))) { temp = AES_RND_KEY[i-1]; if (Nr == 14) { switch (i) { case 8: Rcon = 0x01000000; break; case 16: Rcon = 0x02000000; break; case 24: Rcon = 0x04000000; break; case 32: Rcon = 0x08000000; break; case 40: Rcon = 0x10000000; break; case 48: Rcon = 0x20000000; break; case 56: Rcon = 0x40000000; break; } } else if (Nr == 12) { switch (i) { case 6: Rcon = 0x01000000; break; case 12: Rcon = 0x02000000; break; case 18: Rcon = 0x04000000; break; case 24: Rcon = 0x08000000; break; case 30: Rcon = 0x10000000; break; case 36: Rcon = 0x20000000; break; case 42: Rcon = 0x40000000; break; case 48: Rcon = 0x80000000; break; } } else if (Nr == 10) { switch (i) { case 4: Rcon = 0x01000000; break; case 8: Rcon = 0x02000000; break; case 12: Rcon = 0x04000000; break; case 16: Rcon = 0x08000000; break; case 20: Rcon = 0x10000000; break; case 24: Rcon = 0x20000000; break; case 28: Rcon = 0x40000000; break; case 32: Rcon = 0x80000000; break; case 36: Rcon = 0x1b000000; break; case 40: Rcon = 0x36000000; break; } } if ((i % Nk) == 0) { data_in = temp; MSB_store = (data_in >> 24 & 0xff); rot_data = (data_in << 8) | MSB_store; byte_for_sub = rot_data; word_sub[0] = _s_box[(byte_for_sub & 0xff)]; word_sub[1] = (_s_box[((byte_for_sub & 0xff00) >> 8)] << 8); word_sub[2] = (_s_box[((byte_for_sub & 0xff0000) >> 16)] << 16); word_sub[3] = (_s_box[((byte_for_sub & 0xff000000) >> 24)] << 24); word_sub[0] = word_sub[0] | word_sub[1] | word_sub[2] | word_sub[3]; temp = word_sub[0] ^ Rcon; } else if ((Nk > 6) && ((i % Nk) == 4)) { byte_for_sub = temp; word_sub[0] = _s_box[(byte_for_sub & 0xff)]; word_sub[1] = (_s_box[((byte_for_sub & 0xff00) >> 8)] << 8); word_sub[2] = (_s_box[((byte_for_sub & 0xff0000) >> 16)] << 16); word_sub[3] = (_s_box[((byte_for_sub & 0xff000000) >> 24)] << 24); word_sub[0] = word_sub[0] | word_sub[1] | word_sub[2] | word_sub[3]; temp = word_sub[0]; } AES_RND_KEY[i] = AES_RND_KEY[i-Nk]^temp; i = i+1; } } static void _byte_stream_to_net_words(uint32_t *iv, unsigned char *b, unsigned int len) { unsigned n; n = len / sizeof(uint32_t); for (; n > 0; n--) { *iv = ((*b << 24) & 0xff000000) | (((*(b+1)) << 16) & 0xff0000) | (((*(b+2)) << 8) & 0xff00) | (*(b+3) & 0xff); b += sizeof(uint32_t); iv++; } n = len % sizeof(uint32_t); if (n == 3) { *iv = ((*b << 24) & 0xff000000) | (((*(b+1)) << 16) & 0xff0000) | (((*(b+2)) << 8) & 0xff00); } else if (n == 2) { *iv = ((*b << 24) & 0xff000000) | (((*(b+1)) << 16) & 0xff0000); } else if (n == 1) { *iv = ((*b << 24) & 0xff000000); } } static void _net_words_to_byte_stream(uint32_t *iv, unsigned char *b, unsigned int len) { unsigned n = len / sizeof(uint32_t); for (; n > 0; n--) { *b++ = (unsigned char) ((*iv >> 24) & 0xff); *b++ = (unsigned char) ((*iv >> 16) & 0xff); *b++ = (unsigned char) ((*iv >> 8) & 0xff); *b++ = (unsigned char) (*iv & 0xff); iv++; } n = len % sizeof(uint32_t); if (n == 3) { *b++ = (unsigned char) ((*iv >> 24) & 0xff); *b++ = (unsigned char) ((*iv >> 16) & 0xff); *b = (unsigned char) ((*iv >> 8) & 0xff); } else if (n == 2) { *b++ = (unsigned char) ((*iv >> 24) & 0xff); *b = (unsigned char) ((*iv >> 16) & 0xff); } else if (n == 1) { *b = (unsigned char) ((*iv >> 24) & 0xff); } } static int count_sg(struct scatterlist *sg, int nbytes) { int i; for (i = 0; nbytes > 0; i++, sg = scatterwalk_sg_next(sg)) nbytes -= sg->length; return i; } static int qce_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { int i; for (i = 0; i < nents; ++i) { dma_map_sg(dev, sg, 1, direction); sg = scatterwalk_sg_next(sg); } return nents; } static int qce_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { int i; for (i = 0; i < nents; ++i) { dma_unmap_sg(dev, sg, 1, direction); sg = scatterwalk_sg_next(sg); } return nents; } static int _probe_ce_engine(struct qce_device *pce_dev) { unsigned int val; unsigned int rev; unsigned int eng_availability; /* engine available functions */ val = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG); if ((val & 0xfffffff) != 0x0200004) { dev_err(pce_dev->pdev, "unknown Qualcomm crypto device at 0x%x 0x%x\n", pce_dev->phy_iobase, val); return -EIO; }; rev = (val & CRYPTO_CORE_REV_MASK) >> CRYPTO_CORE_REV; if (rev == 0x2) { dev_info(pce_dev->pdev, "Qualcomm Crypto 3e device found at 0x%x\n", pce_dev->phy_iobase); } else if (rev == 0x1) { dev_info(pce_dev->pdev, "Qualcomm Crypto 3 device found at 0x%x\n", pce_dev->phy_iobase); } else if (rev == 0x0) { dev_info(pce_dev->pdev, "Qualcomm Crypto 2 device found at 0x%x\n", pce_dev->phy_iobase); } else { dev_err(pce_dev->pdev, "unknown Qualcomm crypto device at 0x%x\n", pce_dev->phy_iobase); return -EIO; } eng_availability = readl_relaxed(pce_dev->iobase + CRYPTO_ENGINES_AVAIL); if (((eng_availability & CRYPTO_AES_SEL_MASK) >> CRYPTO_AES_SEL) == CRYPTO_AES_SEL_FAST) pce_dev->fastaes = 1; else pce_dev->fastaes = 0; if (eng_availability & (1 << CRYPTO_HMAC_SEL)) pce_dev->hmac = 1; else pce_dev->hmac = 0; if ((eng_availability & (1 << CRYPTO_F9_SEL)) && (eng_availability & (1 << CRYPTO_F8_SEL))) pce_dev->ota = true; else pce_dev->ota = false; pce_dev->aes_key_size = 0; return 0; }; static int _init_ce_engine(struct qce_device *pce_dev) { unsigned int val; /* reset qce */ writel_relaxed(1 << CRYPTO_SW_RST, pce_dev->iobase + CRYPTO_CONFIG_REG); /* Ensure previous instruction (write to reset bit) * was completed. */ mb(); /* configure ce */ val = (1 << CRYPTO_MASK_DOUT_INTR) | (1 << CRYPTO_MASK_DIN_INTR) | (1 << CRYPTO_MASK_AUTH_DONE_INTR) | (1 << CRYPTO_MASK_ERR_INTR); writel_relaxed(val, pce_dev->iobase + CRYPTO_CONFIG_REG); if (_probe_ce_engine(pce_dev) < 0) return -EIO; if (readl_relaxed(pce_dev->iobase + CRYPTO_CONFIG_REG) != val) { dev_err(pce_dev->pdev, "unknown Qualcomm crypto device at 0x%x\n", pce_dev->phy_iobase); return -EIO; }; return 0; }; static int _sha_ce_setup(struct qce_device *pce_dev, struct qce_sha_req *sreq) { uint32_t auth32[SHA256_DIGEST_SIZE / sizeof(uint32_t)]; uint32_t diglen; int rc; int i; uint32_t cfg = 0; /* if not the last, the size has to be on the block boundary */ if (sreq->last_blk == 0 && (sreq->size % SHA256_BLOCK_SIZE)) return -EIO; switch (sreq->alg) { case QCE_HASH_SHA1: diglen = SHA1_DIGEST_SIZE; break; case QCE_HASH_SHA256: diglen = SHA256_DIGEST_SIZE; break; default: return -EINVAL; } /* * write 20/32 bytes, 5/8 words into auth_iv * for SHA1/SHA256 */ if (sreq->first_blk) { if (sreq->alg == QCE_HASH_SHA1) { for (i = 0; i < 5; i++) auth32[i] = _std_init_vector_sha1[i]; } else { for (i = 0; i < 8; i++) auth32[i] = _std_init_vector_sha256[i]; } } else _byte_stream_to_net_words(auth32, sreq->digest, diglen); rc = clk_enable(pce_dev->ce_clk); if (rc) return rc; writel_relaxed(auth32[0], pce_dev->iobase + CRYPTO_AUTH_IV0_REG); writel_relaxed(auth32[1], pce_dev->iobase + CRYPTO_AUTH_IV1_REG); writel_relaxed(auth32[2], pce_dev->iobase + CRYPTO_AUTH_IV2_REG); writel_relaxed(auth32[3], pce_dev->iobase + CRYPTO_AUTH_IV3_REG); writel_relaxed(auth32[4], pce_dev->iobase + CRYPTO_AUTH_IV4_REG); if (sreq->alg == QCE_HASH_SHA256) { writel_relaxed(auth32[5], pce_dev->iobase + CRYPTO_AUTH_IV5_REG); writel_relaxed(auth32[6], pce_dev->iobase + CRYPTO_AUTH_IV6_REG); writel_relaxed(auth32[7], pce_dev->iobase + CRYPTO_AUTH_IV7_REG); } /* write auth_bytecnt 0/1, start with 0 */ writel_relaxed(sreq->auth_data[0], pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG); writel_relaxed(sreq->auth_data[1], pce_dev->iobase + CRYPTO_AUTH_BYTECNT1_REG); /* write auth_seg_cfg */ writel_relaxed(sreq->size << CRYPTO_AUTH_SEG_SIZE, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* * write seg_cfg */ if (sreq->alg == QCE_HASH_SHA1) cfg |= (CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE); else cfg = (CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE); if (sreq->first_blk) cfg |= 1 << CRYPTO_FIRST; if (sreq->last_blk) cfg |= 1 << CRYPTO_LAST; cfg |= CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG; writel_relaxed(cfg, pce_dev->iobase + CRYPTO_SEG_CFG_REG); /* write seg_size */ writel_relaxed(sreq->size, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); /* issue go to crypto */ writel_relaxed(1 << CRYPTO_GO, pce_dev->iobase + CRYPTO_GOPROC_REG); /* Ensure previous instructions (setting the GO register) * was completed before issuing a DMA transfer request */ mb(); return 0; } static int _ce_setup(struct qce_device *pce_dev, struct qce_req *q_req, uint32_t totallen, uint32_t coffset) { uint32_t hmackey[HMAC_KEY_SIZE/sizeof(uint32_t)] = { 0, 0, 0, 0, 0}; uint32_t enckey32[MAX_CIPHER_KEY_SIZE/sizeof(uint32_t)] = { 0, 0, 0, 0, 0, 0, 0, 0}; uint32_t enciv32[MAX_IV_LENGTH / sizeof(uint32_t)] = { 0, 0, 0, 0}; uint32_t enck_size_in_word = q_req->encklen / sizeof(uint32_t); int aes_key_chg; int i, rc; uint32_t aes_round_key[CRYPTO_AES_RNDKEYS]; uint32_t cfg; uint32_t ivsize = q_req->ivsize; rc = clk_enable(pce_dev->ce_clk); if (rc) return rc; cfg = (1 << CRYPTO_FIRST) | (1 << CRYPTO_LAST); if (q_req->op == QCE_REQ_AEAD) { /* do authentication setup */ cfg |= (CRYPTO_AUTH_SIZE_HMAC_SHA1 << CRYPTO_AUTH_SIZE)| (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG); /* write sha1 init vector */ writel_relaxed(_std_init_vector_sha1[0], pce_dev->iobase + CRYPTO_AUTH_IV0_REG); writel_relaxed(_std_init_vector_sha1[1], pce_dev->iobase + CRYPTO_AUTH_IV1_REG); writel_relaxed(_std_init_vector_sha1[2], pce_dev->iobase + CRYPTO_AUTH_IV2_REG); writel_relaxed(_std_init_vector_sha1[3], pce_dev->iobase + CRYPTO_AUTH_IV3_REG); writel_relaxed(_std_init_vector_sha1[4], pce_dev->iobase + CRYPTO_AUTH_IV4_REG); /* write hmac key */ _byte_stream_to_net_words(hmackey, q_req->authkey, q_req->authklen); writel_relaxed(hmackey[0], pce_dev->iobase + CRYPTO_AUTH_IV5_REG); writel_relaxed(hmackey[1], pce_dev->iobase + CRYPTO_AUTH_IV6_REG); writel_relaxed(hmackey[2], pce_dev->iobase + CRYPTO_AUTH_IV7_REG); writel_relaxed(hmackey[3], pce_dev->iobase + CRYPTO_AUTH_IV8_REG); writel_relaxed(hmackey[4], pce_dev->iobase + CRYPTO_AUTH_IV9_REG); writel_relaxed(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG); writel_relaxed(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT1_REG); /* write auth_seg_cfg */ writel_relaxed((totallen << CRYPTO_AUTH_SEG_SIZE) & 0xffff0000, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); } _byte_stream_to_net_words(enckey32, q_req->enckey, q_req->encklen); switch (q_req->mode) { case QCE_MODE_ECB: cfg |= (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE); break; case QCE_MODE_CBC: cfg |= (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE); break; case QCE_MODE_CTR: default: cfg |= (CRYPTO_ENCR_MODE_CTR << CRYPTO_ENCR_MODE); break; } pce_dev->mode = q_req->mode; switch (q_req->alg) { case CIPHER_ALG_DES: if (q_req->mode != QCE_MODE_ECB) { _byte_stream_to_net_words(enciv32, q_req->iv, ivsize); writel_relaxed(enciv32[0], pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); writel_relaxed(enciv32[1], pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); } writel_relaxed(enckey32[0], pce_dev->iobase + CRYPTO_DES_KEY0_REG); writel_relaxed(enckey32[1], pce_dev->iobase + CRYPTO_DES_KEY1_REG); cfg |= ((CRYPTO_ENCR_KEY_SZ_DES << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG)); break; case CIPHER_ALG_3DES: if (q_req->mode != QCE_MODE_ECB) { _byte_stream_to_net_words(enciv32, q_req->iv, ivsize); writel_relaxed(enciv32[0], pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); writel_relaxed(enciv32[1], pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); } writel_relaxed(enckey32[0], pce_dev->iobase + CRYPTO_DES_KEY0_REG); writel_relaxed(enckey32[1], pce_dev->iobase + CRYPTO_DES_KEY1_REG); writel_relaxed(enckey32[2], pce_dev->iobase + CRYPTO_DES_KEY2_REG); writel_relaxed(enckey32[3], pce_dev->iobase + CRYPTO_DES_KEY3_REG); writel_relaxed(enckey32[4], pce_dev->iobase + CRYPTO_DES_KEY4_REG); writel_relaxed(enckey32[5], pce_dev->iobase + CRYPTO_DES_KEY5_REG); cfg |= ((CRYPTO_ENCR_KEY_SZ_3DES << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG)); break; case CIPHER_ALG_AES: default: if (q_req->mode != QCE_MODE_ECB) { _byte_stream_to_net_words(enciv32, q_req->iv, ivsize); writel_relaxed(enciv32[0], pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); writel_relaxed(enciv32[1], pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); writel_relaxed(enciv32[2], pce_dev->iobase + CRYPTO_CNTR2_IV2_REG); writel_relaxed(enciv32[3], pce_dev->iobase + CRYPTO_CNTR3_IV3_REG); } /* set number of counter bits */ writel_relaxed(0xffff, pce_dev->iobase + CRYPTO_CNTR_MASK_REG); if (q_req->op == QCE_REQ_ABLK_CIPHER_NO_KEY) { cfg |= (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ); cfg |= CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG; } else { switch (q_req->encklen) { case AES128_KEY_SIZE: cfg |= (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ); break; case AES192_KEY_SIZE: cfg |= (CRYPTO_ENCR_KEY_SZ_AES192 << CRYPTO_ENCR_KEY_SZ); break; case AES256_KEY_SIZE: default: cfg |= (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ); /* check for null key. If null, use hw key*/ for (i = 0; i < enck_size_in_word; i++) { if (enckey32[i] != 0) break; } if (i == enck_size_in_word) cfg |= 1 << CRYPTO_USE_HW_KEY; break; } /* end of switch (q_req->encklen) */ cfg |= CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG; if (pce_dev->aes_key_size != q_req->encklen) aes_key_chg = 1; else { for (i = 0; i < enck_size_in_word; i++) { if (enckey32[i] != pce_dev->aeskey[i]) break; } aes_key_chg = (i == enck_size_in_word) ? 0 : 1; } if (aes_key_chg) { if (pce_dev->fastaes) { for (i = 0; i < enck_size_in_word; i++) { writel_relaxed(enckey32[i], pce_dev->iobase + CRYPTO_AES_RNDKEY0 + (i * sizeof(uint32_t))); } } else { /* size in bit */ _aes_expand_key_schedule( q_req->encklen * 8, enckey32, aes_round_key); for (i = 0; i < CRYPTO_AES_RNDKEYS; i++) { writel_relaxed(aes_round_key[i], pce_dev->iobase + CRYPTO_AES_RNDKEY0 + (i * sizeof(uint32_t))); } } pce_dev->aes_key_size = q_req->encklen; for (i = 0; i < enck_size_in_word; i++) pce_dev->aeskey[i] = enckey32[i]; } /*if (aes_key_chg) { */ } /* else of if (q_req->op == QCE_REQ_ABLK_CIPHER_NO_KEY) */ break; } /* end of switch (q_req->mode) */ if (q_req->dir == QCE_ENCRYPT) cfg |= (1 << CRYPTO_AUTH_POS); cfg |= ((q_req->dir == QCE_ENCRYPT) ? 1 : 0) << CRYPTO_ENCODE; /* write encr seg cfg */ writel_relaxed((q_req->cryptlen << CRYPTO_ENCR_SEG_SIZE) | (coffset & 0xffff), /* cipher offset */ pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* write seg cfg and size */ writel_relaxed(cfg, pce_dev->iobase + CRYPTO_SEG_CFG_REG); writel_relaxed(totallen, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); /* issue go to crypto */ writel_relaxed(1 << CRYPTO_GO, pce_dev->iobase + CRYPTO_GOPROC_REG); /* Ensure previous instructions (setting the GO register) * was completed before issuing a DMA transfer request */ mb(); return 0; }; static int _aead_complete(struct qce_device *pce_dev) { struct aead_request *areq; struct crypto_aead *aead; uint32_t ivsize; uint32_t iv_out[4]; unsigned char iv[4 * sizeof(uint32_t)]; uint32_t status; areq = (struct aead_request *) pce_dev->areq; aead = crypto_aead_reqtfm(areq); ivsize = crypto_aead_ivsize(aead); if (areq->src != areq->dst) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents, DMA_FROM_DEVICE); } qce_dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); dma_unmap_single(pce_dev->pdev, pce_dev->phy_iv_in, ivsize, DMA_TO_DEVICE); qce_dma_unmap_sg(pce_dev->pdev, areq->assoc, pce_dev->assoc_nents, DMA_TO_DEVICE); /* check ce error status */ status = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG); if (status & (1 << CRYPTO_SW_ERR)) { pce_dev->err++; dev_err(pce_dev->pdev, "Qualcomm Crypto Error at 0x%x, status%x\n", pce_dev->phy_iobase, status); _init_ce_engine(pce_dev); clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, pce_dev->dig_result, NULL, -ENXIO); return 0; }; /* get iv out */ if (pce_dev->mode == QCE_MODE_ECB) { clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, pce_dev->dig_result, NULL, pce_dev->chan_ce_in_status | pce_dev->chan_ce_out_status); } else { iv_out[0] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); iv_out[1] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); iv_out[2] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR2_IV2_REG); iv_out[3] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR3_IV3_REG); _net_words_to_byte_stream(iv_out, iv, sizeof(iv)); clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, pce_dev->dig_result, iv, pce_dev->chan_ce_in_status | pce_dev->chan_ce_out_status); }; return 0; }; static void _sha_complete(struct qce_device *pce_dev) { struct ahash_request *areq; uint32_t auth_data[2]; uint32_t status; areq = (struct ahash_request *) pce_dev->areq; qce_dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents, DMA_TO_DEVICE); /* check ce error status */ status = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG); if (status & (1 << CRYPTO_SW_ERR)) { pce_dev->err++; dev_err(pce_dev->pdev, "Qualcomm Crypto Error at 0x%x, status%x\n", pce_dev->phy_iobase, status); _init_ce_engine(pce_dev); clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, pce_dev->dig_result, NULL, -ENXIO); return; }; auth_data[0] = readl_relaxed(pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG); auth_data[1] = readl_relaxed(pce_dev->iobase + CRYPTO_AUTH_BYTECNT1_REG); /* Ensure previous instruction (retriving byte count information) * was completed before disabling the clk. */ mb(); clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, pce_dev->dig_result, (unsigned char *)auth_data, pce_dev->chan_ce_in_status); }; static int _ablk_cipher_complete(struct qce_device *pce_dev) { struct ablkcipher_request *areq; uint32_t iv_out[4]; unsigned char iv[4 * sizeof(uint32_t)]; uint32_t status; areq = (struct ablkcipher_request *) pce_dev->areq; if (areq->src != areq->dst) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents, DMA_FROM_DEVICE); } qce_dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* check ce error status */ status = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG); if (status & (1 << CRYPTO_SW_ERR)) { pce_dev->err++; dev_err(pce_dev->pdev, "Qualcomm Crypto Error at 0x%x, status%x\n", pce_dev->phy_iobase, status); _init_ce_engine(pce_dev); clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, NULL, NULL, -ENXIO); return 0; }; /* get iv out */ if (pce_dev->mode == QCE_MODE_ECB) { clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, NULL, NULL, pce_dev->chan_ce_in_status | pce_dev->chan_ce_out_status); } else { iv_out[0] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); iv_out[1] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); iv_out[2] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR2_IV2_REG); iv_out[3] = readl_relaxed(pce_dev->iobase + CRYPTO_CNTR3_IV3_REG); _net_words_to_byte_stream(iv_out, iv, sizeof(iv)); clk_disable(pce_dev->ce_clk); pce_dev->qce_cb(areq, NULL, iv, pce_dev->chan_ce_in_status | pce_dev->chan_ce_out_status); } return 0; }; static int qce_split_and_insert_dm_desc(struct dmov_desc *pdesc, unsigned int plen, unsigned int paddr, int *index) { while (plen > QCE_FIFO_SIZE) { pdesc->len = QCE_FIFO_SIZE; if (paddr > 0) { pdesc->addr = paddr; paddr += QCE_FIFO_SIZE; } plen -= pdesc->len; if (plen > 0) { *index = (*index) + 1; if ((*index) >= QCE_MAX_NUM_DESC) return -ENOMEM; pdesc++; } } if ((plen > 0) && (plen <= QCE_FIFO_SIZE)) { pdesc->len = plen; if (paddr > 0) pdesc->addr = paddr; } return 0; } static int _chain_sg_buffer_in(struct qce_device *pce_dev, struct scatterlist *sg, unsigned int nbytes) { unsigned int len; unsigned int dlen; struct dmov_desc *pdesc; pdesc = pce_dev->ce_in_src_desc + pce_dev->ce_in_src_desc_index; /* * Two consective chunks may be handled by the old * buffer descriptor. */ while (nbytes > 0) { len = min(nbytes, sg_dma_len(sg)); dlen = pdesc->len & ADM_DESC_LENGTH_MASK; nbytes -= len; if (dlen == 0) { pdesc->addr = sg_dma_address(sg); pdesc->len = len; if (pdesc->len > QCE_FIFO_SIZE) { if (qce_split_and_insert_dm_desc(pdesc, pdesc->len, sg_dma_address(sg), &pce_dev->ce_in_src_desc_index)) return -EIO; } } else if (sg_dma_address(sg) == (pdesc->addr + dlen)) { pdesc->len = dlen + len; if (pdesc->len > QCE_FIFO_SIZE) { if (qce_split_and_insert_dm_desc(pdesc, pdesc->len, pdesc->addr, &pce_dev->ce_in_src_desc_index)) return -EIO; } } else { pce_dev->ce_in_src_desc_index++; if (pce_dev->ce_in_src_desc_index >= QCE_MAX_NUM_DESC) return -ENOMEM; pdesc++; pdesc->len = len; pdesc->addr = sg_dma_address(sg); if (pdesc->len > QCE_FIFO_SIZE) { if (qce_split_and_insert_dm_desc(pdesc, pdesc->len, sg_dma_address(sg), &pce_dev->ce_in_src_desc_index)) return -EIO; } } if (nbytes > 0) sg = scatterwalk_sg_next(sg); } return 0; } static int _chain_pm_buffer_in(struct qce_device *pce_dev, unsigned int pmem, unsigned int nbytes) { unsigned int dlen; struct dmov_desc *pdesc; pdesc = pce_dev->ce_in_src_desc + pce_dev->ce_in_src_desc_index; dlen = pdesc->len & ADM_DESC_LENGTH_MASK; if (dlen == 0) { pdesc->addr = pmem; pdesc->len = nbytes; } else if (pmem == (pdesc->addr + dlen)) { pdesc->len = dlen + nbytes; } else { pce_dev->ce_in_src_desc_index++; if (pce_dev->ce_in_src_desc_index >= QCE_MAX_NUM_DESC) return -ENOMEM; pdesc++; pdesc->len = nbytes; pdesc->addr = pmem; } return 0; } static void _chain_buffer_in_init(struct qce_device *pce_dev) { struct dmov_desc *pdesc; pce_dev->ce_in_src_desc_index = 0; pce_dev->ce_in_dst_desc_index = 0; pdesc = pce_dev->ce_in_src_desc; pdesc->len = 0; } static void _ce_in_final(struct qce_device *pce_dev, int ncmd, unsigned total) { struct dmov_desc *pdesc; dmov_sg *pcmd; pdesc = pce_dev->ce_in_src_desc + pce_dev->ce_in_src_desc_index; pdesc->len |= ADM_DESC_LAST; pdesc = pce_dev->ce_in_dst_desc; if (total > QCE_FIFO_SIZE) { qce_split_and_insert_dm_desc(pdesc, total, 0, &pce_dev->ce_in_dst_desc_index); pdesc = pce_dev->ce_in_dst_desc + pce_dev->ce_in_dst_desc_index; pdesc->len |= ADM_DESC_LAST; } else pdesc->len = ADM_DESC_LAST | total; pcmd = (dmov_sg *) pce_dev->cmd_list_ce_in; if (ncmd == 1) pcmd->cmd |= CMD_LC; else { dmov_s *pscmd; pcmd->cmd &= ~CMD_LC; pcmd++; pscmd = (dmov_s *)pcmd; pscmd->cmd |= CMD_LC; } #ifdef QCE_DEBUG dev_info(pce_dev->pdev, "_ce_in_final %d\n", pce_dev->ce_in_src_desc_index); #endif } #ifdef QCE_DEBUG static void _ce_in_dump(struct qce_device *pce_dev) { int i; struct dmov_desc *pdesc; dev_info(pce_dev->pdev, "_ce_in_dump: src\n"); for (i = 0; i <= pce_dev->ce_in_src_desc_index; i++) { pdesc = pce_dev->ce_in_src_desc + i; dev_info(pce_dev->pdev, "%x , %x\n", pdesc->addr, pdesc->len); } dev_info(pce_dev->pdev, "_ce_in_dump: dst\n"); for (i = 0; i <= pce_dev->ce_in_dst_desc_index; i++) { pdesc = pce_dev->ce_in_dst_desc + i; dev_info(pce_dev->pdev, "%x , %x\n", pdesc->addr, pdesc->len); } }; static void _ce_out_dump(struct qce_device *pce_dev) { int i; struct dmov_desc *pdesc; dev_info(pce_dev->pdev, "_ce_out_dump: src\n"); for (i = 0; i <= pce_dev->ce_out_src_desc_index; i++) { pdesc = pce_dev->ce_out_src_desc + i; dev_info(pce_dev->pdev, "%x , %x\n", pdesc->addr, pdesc->len); } dev_info(pce_dev->pdev, "_ce_out_dump: dst\n"); for (i = 0; i <= pce_dev->ce_out_dst_desc_index; i++) { pdesc = pce_dev->ce_out_dst_desc + i; dev_info(pce_dev->pdev, "%x , %x\n", pdesc->addr, pdesc->len); } }; #endif static int _chain_sg_buffer_out(struct qce_device *pce_dev, struct scatterlist *sg, unsigned int nbytes) { unsigned int len; unsigned int dlen; struct dmov_desc *pdesc; pdesc = pce_dev->ce_out_dst_desc + pce_dev->ce_out_dst_desc_index; /* * Two consective chunks may be handled by the old * buffer descriptor. */ while (nbytes > 0) { len = min(nbytes, sg_dma_len(sg)); dlen = pdesc->len & ADM_DESC_LENGTH_MASK; nbytes -= len; if (dlen == 0) { pdesc->addr = sg_dma_address(sg); pdesc->len = len; if (pdesc->len > QCE_FIFO_SIZE) { if (qce_split_and_insert_dm_desc(pdesc, pdesc->len, sg_dma_address(sg), &pce_dev->ce_out_dst_desc_index)) return -EIO; } } else if (sg_dma_address(sg) == (pdesc->addr + dlen)) { pdesc->len = dlen + len; if (pdesc->len > QCE_FIFO_SIZE) { if (qce_split_and_insert_dm_desc(pdesc, pdesc->len, pdesc->addr, &pce_dev->ce_out_dst_desc_index)) return -EIO; } } else { pce_dev->ce_out_dst_desc_index++; if (pce_dev->ce_out_dst_desc_index >= QCE_MAX_NUM_DESC) return -EIO; pdesc++; pdesc->len = len; pdesc->addr = sg_dma_address(sg); if (pdesc->len > QCE_FIFO_SIZE) { if (qce_split_and_insert_dm_desc(pdesc, pdesc->len, sg_dma_address(sg), &pce_dev->ce_out_dst_desc_index)) return -EIO; } } if (nbytes > 0) sg = scatterwalk_sg_next(sg); } return 0; } static int _chain_pm_buffer_out(struct qce_device *pce_dev, unsigned int pmem, unsigned int nbytes) { unsigned int dlen; struct dmov_desc *pdesc; pdesc = pce_dev->ce_out_dst_desc + pce_dev->ce_out_dst_desc_index; dlen = pdesc->len & ADM_DESC_LENGTH_MASK; if (dlen == 0) { pdesc->addr = pmem; pdesc->len = nbytes; } else if (pmem == (pdesc->addr + dlen)) { pdesc->len = dlen + nbytes; } else { pce_dev->ce_out_dst_desc_index++; if (pce_dev->ce_out_dst_desc_index >= QCE_MAX_NUM_DESC) return -EIO; pdesc++; pdesc->len = nbytes; pdesc->addr = pmem; } return 0; }; static void _chain_buffer_out_init(struct qce_device *pce_dev) { struct dmov_desc *pdesc; pce_dev->ce_out_dst_desc_index = 0; pce_dev->ce_out_src_desc_index = 0; pdesc = pce_dev->ce_out_dst_desc; pdesc->len = 0; }; static void _ce_out_final(struct qce_device *pce_dev, int ncmd, unsigned total) { struct dmov_desc *pdesc; dmov_sg *pcmd; pdesc = pce_dev->ce_out_dst_desc + pce_dev->ce_out_dst_desc_index; pdesc->len |= ADM_DESC_LAST; pdesc = pce_dev->ce_out_src_desc; if (total > QCE_FIFO_SIZE) { qce_split_and_insert_dm_desc(pdesc, total, 0, &pce_dev->ce_out_src_desc_index); pdesc = pce_dev->ce_out_src_desc + pce_dev->ce_out_src_desc_index; pdesc->len |= ADM_DESC_LAST; } else pdesc->len = ADM_DESC_LAST | total; pcmd = (dmov_sg *) pce_dev->cmd_list_ce_out; if (ncmd == 1) pcmd->cmd |= CMD_LC; else { dmov_s *pscmd; pcmd->cmd &= ~CMD_LC; pcmd++; pscmd = (dmov_s *)pcmd; pscmd->cmd |= CMD_LC; } #ifdef QCE_DEBUG dev_info(pce_dev->pdev, "_ce_out_final %d\n", pce_dev->ce_out_dst_desc_index); #endif }; static void _aead_ce_in_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_in_status = -1; } else pce_dev->chan_ce_in_status = 0; pce_dev->chan_ce_in_state = QCE_CHAN_STATE_COMP; if (pce_dev->chan_ce_out_state == QCE_CHAN_STATE_COMP) { pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IDLE; /* done */ _aead_complete(pce_dev); } }; static void _aead_ce_out_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_out_status = -1; } else { pce_dev->chan_ce_out_status = 0; }; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_COMP; if (pce_dev->chan_ce_in_state == QCE_CHAN_STATE_COMP) { pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IDLE; /* done */ _aead_complete(pce_dev); } }; static void _sha_ce_in_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_in_status = -1; } else pce_dev->chan_ce_in_status = 0; pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; _sha_complete(pce_dev); }; static void _ablk_cipher_ce_in_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_in_status = -1; } else pce_dev->chan_ce_in_status = 0; pce_dev->chan_ce_in_state = QCE_CHAN_STATE_COMP; if (pce_dev->chan_ce_out_state == QCE_CHAN_STATE_COMP) { pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IDLE; /* done */ _ablk_cipher_complete(pce_dev); } }; static void _ablk_cipher_ce_out_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_out_status = -1; } else { pce_dev->chan_ce_out_status = 0; }; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_COMP; if (pce_dev->chan_ce_in_state == QCE_CHAN_STATE_COMP) { pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IDLE; /* done */ _ablk_cipher_complete(pce_dev); } }; static int _setup_cmd_template(struct qce_device *pce_dev) { dmov_sg *pcmd; dmov_s *pscmd; struct dmov_desc *pdesc; unsigned char *vaddr; int i = 0; /* Divide up the 4K coherent memory */ /* 1. ce_in channel 1st command src descriptors, 128 entries */ vaddr = pce_dev->coh_vmem; vaddr = (unsigned char *) ALIGN(((unsigned int)vaddr), 16); pce_dev->ce_in_src_desc = (struct dmov_desc *) vaddr; pce_dev->phy_ce_in_src_desc = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + (sizeof(struct dmov_desc) * QCE_MAX_NUM_DESC); /* 2. ce_in channel 1st command dst descriptor, 1 entry */ vaddr = (unsigned char *) ALIGN(((unsigned int)vaddr), 16); pce_dev->ce_in_dst_desc = (struct dmov_desc *) vaddr; pce_dev->phy_ce_in_dst_desc = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + (sizeof(struct dmov_desc) * QCE_MAX_NUM_DESC); /* * 3. ce_in channel command list of one scatter gather command * and one simple command. */ pce_dev->cmd_list_ce_in = vaddr; pce_dev->phy_cmd_list_ce_in = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + sizeof(dmov_s) + sizeof(dmov_sg); /* 4. authentication result. */ pce_dev->dig_result = vaddr; pce_dev->phy_dig_result = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + SHA256_DIGESTSIZE; /* * 5. ce_out channel command list of one scatter gather command * and one simple command. */ pce_dev->cmd_list_ce_out = vaddr; pce_dev->phy_cmd_list_ce_out = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + sizeof(dmov_s) + sizeof(dmov_sg); /* 6. ce_out channel command src descriptors, 1 entry */ vaddr = (unsigned char *) ALIGN(((unsigned int)vaddr), 16); pce_dev->ce_out_src_desc = (struct dmov_desc *) vaddr; pce_dev->phy_ce_out_src_desc = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + (sizeof(struct dmov_desc) * QCE_MAX_NUM_DESC); /* 7. ce_out channel command dst descriptors, 128 entries. */ vaddr = (unsigned char *) ALIGN(((unsigned int)vaddr), 16); pce_dev->ce_out_dst_desc = (struct dmov_desc *) vaddr; pce_dev->phy_ce_out_dst_desc = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + (sizeof(struct dmov_desc) * QCE_MAX_NUM_DESC); /* 8. pad area. */ pce_dev->ce_pad = vaddr; pce_dev->phy_ce_pad = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + ADM_CE_BLOCK_SIZE; /* 9. ce_in channel command pointer list. */ vaddr = (unsigned char *) ALIGN(((unsigned int) vaddr), 16); pce_dev->cmd_pointer_list_ce_in = (unsigned int *) vaddr; pce_dev->phy_cmd_pointer_list_ce_in = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + sizeof(unsigned char *); /* 10. ce_ou channel command pointer list. */ vaddr = (unsigned char *) ALIGN(((unsigned int) vaddr), 16); pce_dev->cmd_pointer_list_ce_out = (unsigned int *) vaddr; pce_dev->phy_cmd_pointer_list_ce_out = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); vaddr = vaddr + sizeof(unsigned char *); /* 11. throw away area to store by-pass data from ce_out. */ pce_dev->ce_out_ignore = (unsigned char *) vaddr; pce_dev->phy_ce_out_ignore = pce_dev->coh_pmem + (vaddr - pce_dev->coh_vmem); pce_dev->ce_out_ignore_size = (2 * PAGE_SIZE) - (vaddr - pce_dev->coh_vmem); /* at least 1.5 K of space */ /* * The first command of command list ce_in is for the input of * concurrent operation of encrypt/decrypt or for the input * of authentication. */ pcmd = (dmov_sg *) pce_dev->cmd_list_ce_in; /* swap byte and half word , dst crci , scatter gather */ pcmd->cmd = CMD_DST_SWAP_BYTES | CMD_DST_SWAP_SHORTS | CMD_DST_CRCI(pce_dev->crci_in) | CMD_MODE_SG; pdesc = pce_dev->ce_in_src_desc; pdesc->addr = 0; /* to be filled in each operation */ pdesc->len = 0; /* to be filled in each operation */ pcmd->src_dscr = (unsigned) pce_dev->phy_ce_in_src_desc; pdesc = pce_dev->ce_in_dst_desc; for (i = 0; i < QCE_MAX_NUM_DESC; i++) { pdesc->addr = (CRYPTO_DATA_SHADOW0 + pce_dev->phy_iobase); pdesc->len = 0; /* to be filled in each operation */ pdesc++; } pcmd->dst_dscr = (unsigned) pce_dev->phy_ce_in_dst_desc; pcmd->_reserved = LI_SG_CMD | SRC_INDEX_SG_CMD(0) | DST_INDEX_SG_CMD(0); pcmd++; /* * The second command is for the digested data of * hashing operation only. For others, this command is not used. */ pscmd = (dmov_s *) pcmd; /* last command, swap byte, half word, src crci, single */ pscmd->cmd = CMD_LC | CMD_SRC_SWAP_BYTES | CMD_SRC_SWAP_SHORTS | CMD_SRC_CRCI(pce_dev->crci_hash) | CMD_MODE_SINGLE; pscmd->src = (unsigned) (CRYPTO_AUTH_IV0_REG + pce_dev->phy_iobase); pscmd->len = SHA256_DIGESTSIZE; /* to be filled. */ pscmd->dst = (unsigned) pce_dev->phy_dig_result; /* setup command pointer list */ *(pce_dev->cmd_pointer_list_ce_in) = (CMD_PTR_LP | DMOV_CMD_LIST | DMOV_CMD_ADDR((unsigned int) pce_dev->phy_cmd_list_ce_in)); pce_dev->chan_ce_in_cmd->user = (void *) pce_dev; pce_dev->chan_ce_in_cmd->exec_func = NULL; pce_dev->chan_ce_in_cmd->cmdptr = DMOV_CMD_ADDR( (unsigned int) pce_dev->phy_cmd_pointer_list_ce_in); /* * The first command in the command list ce_out. * It is for encry/decryp output. * If hashing only, ce_out is not used. */ pcmd = (dmov_sg *) pce_dev->cmd_list_ce_out; /* swap byte, half word, source crci, scatter gather */ pcmd->cmd = CMD_SRC_SWAP_BYTES | CMD_SRC_SWAP_SHORTS | CMD_SRC_CRCI(pce_dev->crci_out) | CMD_MODE_SG; pdesc = pce_dev->ce_out_src_desc; for (i = 0; i < QCE_MAX_NUM_DESC; i++) { pdesc->addr = (CRYPTO_DATA_SHADOW0 + pce_dev->phy_iobase); pdesc->len = 0; /* to be filled in each operation */ pdesc++; } pcmd->src_dscr = (unsigned) pce_dev->phy_ce_out_src_desc; pdesc = pce_dev->ce_out_dst_desc; pdesc->addr = 0; /* to be filled in each operation */ pdesc->len = 0; /* to be filled in each operation */ pcmd->dst_dscr = (unsigned) pce_dev->phy_ce_out_dst_desc; pcmd->_reserved = LI_SG_CMD | SRC_INDEX_SG_CMD(0) | DST_INDEX_SG_CMD(0); pcmd++; /* * The second command is for digested data of esp operation. * For ciphering, this command is not used. */ pscmd = (dmov_s *) pcmd; /* last command, swap byte, half word, src crci, single */ pscmd->cmd = CMD_LC | CMD_SRC_SWAP_BYTES | CMD_SRC_SWAP_SHORTS | CMD_SRC_CRCI(pce_dev->crci_hash) | CMD_MODE_SINGLE; pscmd->src = (CRYPTO_AUTH_IV0_REG + pce_dev->phy_iobase); pscmd->len = SHA1_DIGESTSIZE; /* we only support hmac(sha1) */ pscmd->dst = (unsigned) pce_dev->phy_dig_result; /* setup command pointer list */ *(pce_dev->cmd_pointer_list_ce_out) = (CMD_PTR_LP | DMOV_CMD_LIST | DMOV_CMD_ADDR((unsigned int)pce_dev-> phy_cmd_list_ce_out)); pce_dev->chan_ce_out_cmd->user = pce_dev; pce_dev->chan_ce_out_cmd->exec_func = NULL; pce_dev->chan_ce_out_cmd->cmdptr = DMOV_CMD_ADDR( (unsigned int) pce_dev->phy_cmd_pointer_list_ce_out); return 0; }; static int _qce_start_dma(struct qce_device *pce_dev, bool ce_in, bool ce_out) { if (ce_in) pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IN_PROG; else pce_dev->chan_ce_in_state = QCE_CHAN_STATE_COMP; if (ce_out) pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IN_PROG; else pce_dev->chan_ce_out_state = QCE_CHAN_STATE_COMP; if (ce_in) msm_dmov_enqueue_cmd(pce_dev->chan_ce_in, pce_dev->chan_ce_in_cmd); if (ce_out) msm_dmov_enqueue_cmd(pce_dev->chan_ce_out, pce_dev->chan_ce_out_cmd); return 0; }; static void _f9_complete(struct qce_device *pce_dev) { uint32_t mac_i; uint32_t status; dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_src, pce_dev->ota_size, DMA_TO_DEVICE); /* check ce error status */ status = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG); if (status & (1 << CRYPTO_SW_ERR)) { pce_dev->err++; dev_err(pce_dev->pdev, "Qualcomm Crypto Error at 0x%x, status%x\n", pce_dev->phy_iobase, status); _init_ce_engine(pce_dev); pce_dev->qce_cb(pce_dev->areq, NULL, NULL, -ENXIO); return; }; mac_i = readl_relaxed(pce_dev->iobase + CRYPTO_AUTH_IV0_REG); pce_dev->qce_cb(pce_dev->areq, (void *) mac_i, NULL, pce_dev->chan_ce_in_status); }; static void _f8_complete(struct qce_device *pce_dev) { uint32_t status; if (pce_dev->phy_ota_dst != 0) dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_dst, pce_dev->ota_size, DMA_FROM_DEVICE); if (pce_dev->phy_ota_src != 0) dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_src, pce_dev->ota_size, (pce_dev->phy_ota_dst) ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); /* check ce error status */ status = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG); if (status & (1 << CRYPTO_SW_ERR)) { pce_dev->err++; dev_err(pce_dev->pdev, "Qualcomm Crypto Error at 0x%x, status%x\n", pce_dev->phy_iobase, status); _init_ce_engine(pce_dev); pce_dev->qce_cb(pce_dev->areq, NULL, NULL, -ENXIO); return; }; pce_dev->qce_cb(pce_dev->areq, NULL, NULL, pce_dev->chan_ce_in_status | pce_dev->chan_ce_out_status); }; static void _f9_ce_in_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_in_status = -1; } else pce_dev->chan_ce_in_status = 0; pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; _f9_complete(pce_dev); }; static void _f8_ce_in_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_in_status = -1; } else pce_dev->chan_ce_in_status = 0; pce_dev->chan_ce_in_state = QCE_CHAN_STATE_COMP; if (pce_dev->chan_ce_out_state == QCE_CHAN_STATE_COMP) { pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IDLE; /* done */ _f8_complete(pce_dev); } }; static void _f8_ce_out_call_back(struct msm_dmov_cmd *cmd_ptr, unsigned int result, struct msm_dmov_errdata *err) { struct qce_device *pce_dev; pce_dev = (struct qce_device *) cmd_ptr->user; if (result != ADM_STATUS_OK) { dev_err(pce_dev->pdev, "Qualcomm ADM status error %x\n", result); pce_dev->chan_ce_out_status = -1; } else { pce_dev->chan_ce_out_status = 0; }; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_COMP; if (pce_dev->chan_ce_in_state == QCE_CHAN_STATE_COMP) { pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IDLE; /* done */ _f8_complete(pce_dev); } }; static int _ce_f9_setup(struct qce_device *pce_dev, struct qce_f9_req *req) { uint32_t cfg; uint32_t ikey[OTA_KEY_SIZE/sizeof(uint32_t)]; _byte_stream_to_net_words(ikey, &req->ikey[0], OTA_KEY_SIZE); writel_relaxed(ikey[0], pce_dev->iobase + CRYPTO_AUTH_IV0_REG); writel_relaxed(ikey[1], pce_dev->iobase + CRYPTO_AUTH_IV1_REG); writel_relaxed(ikey[2], pce_dev->iobase + CRYPTO_AUTH_IV2_REG); writel_relaxed(ikey[3], pce_dev->iobase + CRYPTO_AUTH_IV3_REG); writel_relaxed(req->last_bits, pce_dev->iobase + CRYPTO_AUTH_IV4_REG); writel_relaxed(req->fresh, pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG); writel_relaxed(req->count_i, pce_dev->iobase + CRYPTO_AUTH_BYTECNT1_REG); /* write auth_seg_cfg */ writel_relaxed((uint32_t)req->msize << CRYPTO_AUTH_SEG_SIZE, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* write seg_cfg */ cfg = (CRYPTO_AUTH_ALG_F9 << CRYPTO_AUTH_ALG) | (1 << CRYPTO_FIRST) | (1 << CRYPTO_LAST); if (req->algorithm == QCE_OTA_ALGO_KASUMI) cfg |= (CRYPTO_AUTH_SIZE_UIA1 << CRYPTO_AUTH_SIZE); else cfg |= (CRYPTO_AUTH_SIZE_UIA2 << CRYPTO_AUTH_SIZE); if (req->direction == QCE_OTA_DIR_DOWNLINK) cfg |= 1 << CRYPTO_F9_DIRECTION; writel_relaxed(cfg, pce_dev->iobase + CRYPTO_SEG_CFG_REG); /* write seg_size */ writel_relaxed(req->msize, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); /* issue go to crypto */ writel_relaxed(1 << CRYPTO_GO, pce_dev->iobase + CRYPTO_GOPROC_REG); /* * barrier to ensure previous instructions * (including GO) to CE finish before issue DMA transfer * request. */ mb(); return 0; }; static int _ce_f8_setup(struct qce_device *pce_dev, struct qce_f8_req *req, bool key_stream_mode, uint16_t npkts, uint16_t cipher_offset, uint16_t cipher_size) { uint32_t cfg; uint32_t ckey[OTA_KEY_SIZE/sizeof(uint32_t)]; if ((key_stream_mode && (req->data_len & 0xf || npkts > 1)) || (req->bearer >= QCE_OTA_MAX_BEARER)) return -EINVAL; /* write seg_cfg */ cfg = (CRYPTO_ENCR_ALG_F8 << CRYPTO_ENCR_ALG) | (1 << CRYPTO_FIRST) | (1 << CRYPTO_LAST); if (req->algorithm == QCE_OTA_ALGO_KASUMI) cfg |= (CRYPTO_ENCR_KEY_SZ_UEA1 << CRYPTO_ENCR_KEY_SZ); else cfg |= (CRYPTO_ENCR_KEY_SZ_UEA2 << CRYPTO_ENCR_KEY_SZ); if (key_stream_mode) cfg |= 1 << CRYPTO_F8_KEYSTREAM_ENABLE; if (req->direction == QCE_OTA_DIR_DOWNLINK) cfg |= 1 << CRYPTO_F8_DIRECTION; writel_relaxed(cfg, pce_dev->iobase + CRYPTO_SEG_CFG_REG); /* write seg_size */ writel_relaxed(req->data_len, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); /* write 0 to auth_size, auth_offset */ writel_relaxed(0, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* write encr_seg_cfg seg_size, seg_offset */ writel_relaxed((((uint32_t) cipher_size) << CRYPTO_ENCR_SEG_SIZE) | (cipher_offset & 0xffff), pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* write keys */ _byte_stream_to_net_words(ckey, &req->ckey[0], OTA_KEY_SIZE); writel_relaxed(ckey[0], pce_dev->iobase + CRYPTO_DES_KEY0_REG); writel_relaxed(ckey[1], pce_dev->iobase + CRYPTO_DES_KEY1_REG); writel_relaxed(ckey[2], pce_dev->iobase + CRYPTO_DES_KEY2_REG); writel_relaxed(ckey[3], pce_dev->iobase + CRYPTO_DES_KEY3_REG); /* write cntr0_iv0 for countC */ writel_relaxed(req->count_c, pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); /* write cntr1_iv1 for nPkts, and bearer */ if (npkts == 1) npkts = 0; writel_relaxed(req->bearer << CRYPTO_CNTR1_IV1_REG_F8_BEARER | npkts << CRYPTO_CNTR1_IV1_REG_F8_PKT_CNT, pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); /* issue go to crypto */ writel_relaxed(1 << CRYPTO_GO, pce_dev->iobase + CRYPTO_GOPROC_REG); /* * barrier to ensure previous instructions * (including GO) to CE finish before issue DMA transfer * request. */ mb(); return 0; }; struct qce_pm_table qce_pm_table = {NULL, NULL}; EXPORT_SYMBOL(qce_pm_table); void qce_get_driver_stats(void *handle) { } EXPORT_SYMBOL(qce_get_driver_stats); void qce_clear_driver_stats(void *handle) { } EXPORT_SYMBOL(qce_clear_driver_stats); int qce_aead_req(void *handle, struct qce_req *q_req) { struct qce_device *pce_dev = (struct qce_device *) handle; struct aead_request *areq = (struct aead_request *) q_req->areq; struct crypto_aead *aead = crypto_aead_reqtfm(areq); uint32_t ivsize = crypto_aead_ivsize(aead); uint32_t totallen; uint32_t pad_len; uint32_t authsize = crypto_aead_authsize(aead); int rc = 0; q_req->ivsize = ivsize; if (q_req->dir == QCE_ENCRYPT) q_req->cryptlen = areq->cryptlen; else q_req->cryptlen = areq->cryptlen - authsize; if ((q_req->cryptlen > UINT_MAX - ivsize) || (q_req->cryptlen + ivsize > UINT_MAX - areq->assoclen)) { pr_err("Integer overflow on total aead req length.\n"); return -EINVAL; } totallen = q_req->cryptlen + ivsize + areq->assoclen; pad_len = ALIGN(totallen, ADM_CE_BLOCK_SIZE) - totallen; _chain_buffer_in_init(pce_dev); _chain_buffer_out_init(pce_dev); pce_dev->assoc_nents = 0; pce_dev->phy_iv_in = 0; pce_dev->src_nents = 0; pce_dev->dst_nents = 0; pce_dev->assoc_nents = count_sg(areq->assoc, areq->assoclen); qce_dma_map_sg(pce_dev->pdev, areq->assoc, pce_dev->assoc_nents, DMA_TO_DEVICE); if (_chain_sg_buffer_in(pce_dev, areq->assoc, areq->assoclen) < 0) { rc = -ENOMEM; goto bad; } /* cipher iv for input */ pce_dev->phy_iv_in = dma_map_single(pce_dev->pdev, q_req->iv, ivsize, DMA_TO_DEVICE); if (_chain_pm_buffer_in(pce_dev, pce_dev->phy_iv_in, ivsize) < 0) { rc = -ENOMEM; goto bad; } /* for output, ignore associated data and cipher iv */ if (_chain_pm_buffer_out(pce_dev, pce_dev->phy_ce_out_ignore, ivsize + areq->assoclen) < 0) { rc = -ENOMEM; goto bad; } /* cipher input */ pce_dev->src_nents = count_sg(areq->src, q_req->cryptlen); qce_dma_map_sg(pce_dev->pdev, areq->src, pce_dev->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (_chain_sg_buffer_in(pce_dev, areq->src, q_req->cryptlen) < 0) { rc = -ENOMEM; goto bad; } /* cipher output */ if (areq->src != areq->dst) { pce_dev->dst_nents = count_sg(areq->dst, q_req->cryptlen); qce_dma_map_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents, DMA_FROM_DEVICE); }; if (_chain_sg_buffer_out(pce_dev, areq->dst, q_req->cryptlen) < 0) { rc = -ENOMEM; goto bad; } /* pad data */ if (pad_len) { if (_chain_pm_buffer_in(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } if (_chain_pm_buffer_out(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } } /* finalize the ce_in and ce_out channels command lists */ _ce_in_final(pce_dev, 1, ALIGN(totallen, ADM_CE_BLOCK_SIZE)); _ce_out_final(pce_dev, 2, ALIGN(totallen, ADM_CE_BLOCK_SIZE)); /* set up crypto device */ rc = _ce_setup(pce_dev, q_req, totallen, ivsize + areq->assoclen); if (rc < 0) goto bad; /* setup for callback, and issue command to adm */ pce_dev->areq = q_req->areq; pce_dev->qce_cb = q_req->qce_cb; pce_dev->chan_ce_in_cmd->complete_func = _aead_ce_in_call_back; pce_dev->chan_ce_out_cmd->complete_func = _aead_ce_out_call_back; rc = _qce_start_dma(pce_dev, true, true); if (rc == 0) return 0; bad: if (pce_dev->assoc_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->assoc, pce_dev->assoc_nents, DMA_TO_DEVICE); } if (pce_dev->phy_iv_in) { dma_unmap_single(pce_dev->pdev, pce_dev->phy_iv_in, ivsize, DMA_TO_DEVICE); } if (pce_dev->src_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); } if (pce_dev->dst_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents, DMA_FROM_DEVICE); } return rc; } EXPORT_SYMBOL(qce_aead_req); int qce_ablk_cipher_req(void *handle, struct qce_req *c_req) { int rc = 0; struct qce_device *pce_dev = (struct qce_device *) handle; struct ablkcipher_request *areq = (struct ablkcipher_request *) c_req->areq; uint32_t pad_len = ALIGN(areq->nbytes, ADM_CE_BLOCK_SIZE) - areq->nbytes; _chain_buffer_in_init(pce_dev); _chain_buffer_out_init(pce_dev); pce_dev->src_nents = 0; pce_dev->dst_nents = 0; /* cipher input */ pce_dev->src_nents = count_sg(areq->src, areq->nbytes); qce_dma_map_sg(pce_dev->pdev, areq->src, pce_dev->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (_chain_sg_buffer_in(pce_dev, areq->src, areq->nbytes) < 0) { rc = -ENOMEM; goto bad; } /* cipher output */ if (areq->src != areq->dst) { pce_dev->dst_nents = count_sg(areq->dst, areq->nbytes); qce_dma_map_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents, DMA_FROM_DEVICE); }; if (_chain_sg_buffer_out(pce_dev, areq->dst, areq->nbytes) < 0) { rc = -ENOMEM; goto bad; } /* pad data */ if (pad_len) { if (_chain_pm_buffer_in(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } if (_chain_pm_buffer_out(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } } /* finalize the ce_in and ce_out channels command lists */ _ce_in_final(pce_dev, 1, areq->nbytes + pad_len); _ce_out_final(pce_dev, 1, areq->nbytes + pad_len); #ifdef QCE_DEBUG _ce_in_dump(pce_dev); _ce_out_dump(pce_dev); #endif /* set up crypto device */ rc = _ce_setup(pce_dev, c_req, areq->nbytes, 0); if (rc < 0) goto bad; /* setup for callback, and issue command to adm */ pce_dev->areq = areq; pce_dev->qce_cb = c_req->qce_cb; pce_dev->chan_ce_in_cmd->complete_func = _ablk_cipher_ce_in_call_back; pce_dev->chan_ce_out_cmd->complete_func = _ablk_cipher_ce_out_call_back; rc = _qce_start_dma(pce_dev, true, true); if (rc == 0) return 0; bad: if (pce_dev->dst_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents, DMA_FROM_DEVICE); } if (pce_dev->src_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); } return rc; } EXPORT_SYMBOL(qce_ablk_cipher_req); int qce_process_sha_req(void *handle, struct qce_sha_req *sreq) { struct qce_device *pce_dev = (struct qce_device *) handle; int rc; uint32_t pad_len = ALIGN(sreq->size, ADM_CE_BLOCK_SIZE) - sreq->size; struct ahash_request *areq = (struct ahash_request *)sreq->areq; _chain_buffer_in_init(pce_dev); pce_dev->src_nents = count_sg(sreq->src, sreq->size); qce_dma_map_sg(pce_dev->pdev, sreq->src, pce_dev->src_nents, DMA_TO_DEVICE); if (_chain_sg_buffer_in(pce_dev, sreq->src, sreq->size) < 0) { rc = -ENOMEM; goto bad; } if (pad_len) { if (_chain_pm_buffer_in(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } } _ce_in_final(pce_dev, 2, sreq->size + pad_len); #ifdef QCE_DEBUG _ce_in_dump(pce_dev); #endif rc = _sha_ce_setup(pce_dev, sreq); if (rc < 0) goto bad; pce_dev->areq = areq; pce_dev->qce_cb = sreq->qce_cb; pce_dev->chan_ce_in_cmd->complete_func = _sha_ce_in_call_back; rc = _qce_start_dma(pce_dev, true, false); if (rc == 0) return 0; bad: if (pce_dev->src_nents) { qce_dma_unmap_sg(pce_dev->pdev, sreq->src, pce_dev->src_nents, DMA_TO_DEVICE); } return rc; } EXPORT_SYMBOL(qce_process_sha_req); int qce_enable_clk(void *handle) { return 0; } EXPORT_SYMBOL(qce_enable_clk); int qce_disable_clk(void *handle) { return 0; } EXPORT_SYMBOL(qce_disable_clk); /* * crypto engine open function. */ void *qce_open(struct platform_device *pdev, int *rc) { struct qce_device *pce_dev; struct resource *resource; struct clk *ce_clk; pce_dev = kzalloc(sizeof(struct qce_device), GFP_KERNEL); if (!pce_dev) { *rc = -ENOMEM; dev_err(&pdev->dev, "Can not allocate memory\n"); return NULL; } pce_dev->pdev = &pdev->dev; ce_clk = clk_get(pce_dev->pdev, "core_clk"); if (IS_ERR(ce_clk)) { kfree(pce_dev); *rc = PTR_ERR(ce_clk); return NULL; } pce_dev->ce_clk = ce_clk; *rc = clk_enable(pce_dev->ce_clk); if (*rc) { kfree(pce_dev); return NULL; } resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!resource) { *rc = -ENXIO; dev_err(pce_dev->pdev, "Missing MEM resource\n"); goto err; }; pce_dev->phy_iobase = resource->start; pce_dev->iobase = ioremap_nocache(resource->start, resource->end - resource->start + 1); if (!pce_dev->iobase) { *rc = -ENOMEM; dev_err(pce_dev->pdev, "Can not map io memory\n"); goto err; } pce_dev->chan_ce_in_cmd = kzalloc(sizeof(struct msm_dmov_cmd), GFP_KERNEL); pce_dev->chan_ce_out_cmd = kzalloc(sizeof(struct msm_dmov_cmd), GFP_KERNEL); if (pce_dev->chan_ce_in_cmd == NULL || pce_dev->chan_ce_out_cmd == NULL) { dev_err(pce_dev->pdev, "Can not allocate memory\n"); *rc = -ENOMEM; goto err; } resource = platform_get_resource_byname(pdev, IORESOURCE_DMA, "crypto_channels"); if (!resource) { *rc = -ENXIO; dev_err(pce_dev->pdev, "Missing DMA channel resource\n"); goto err; }; pce_dev->chan_ce_in = resource->start; pce_dev->chan_ce_out = resource->end; resource = platform_get_resource_byname(pdev, IORESOURCE_DMA, "crypto_crci_in"); if (!resource) { *rc = -ENXIO; dev_err(pce_dev->pdev, "Missing DMA crci in resource\n"); goto err; }; pce_dev->crci_in = resource->start; resource = platform_get_resource_byname(pdev, IORESOURCE_DMA, "crypto_crci_out"); if (!resource) { *rc = -ENXIO; dev_err(pce_dev->pdev, "Missing DMA crci out resource\n"); goto err; }; pce_dev->crci_out = resource->start; resource = platform_get_resource_byname(pdev, IORESOURCE_DMA, "crypto_crci_hash"); if (!resource) { *rc = -ENXIO; dev_err(pce_dev->pdev, "Missing DMA crci hash resource\n"); goto err; }; pce_dev->crci_hash = resource->start; pce_dev->coh_vmem = dma_alloc_coherent(pce_dev->pdev, 2*PAGE_SIZE, &pce_dev->coh_pmem, GFP_KERNEL); if (pce_dev->coh_vmem == NULL) { *rc = -ENOMEM; dev_err(pce_dev->pdev, "Can not allocate coherent memory.\n"); goto err; } _setup_cmd_template(pce_dev); pce_dev->chan_ce_in_state = QCE_CHAN_STATE_IDLE; pce_dev->chan_ce_out_state = QCE_CHAN_STATE_IDLE; if (_init_ce_engine(pce_dev)) { *rc = -ENXIO; clk_disable(pce_dev->ce_clk); goto err; } *rc = 0; clk_disable(pce_dev->ce_clk); pce_dev->err = 0; return pce_dev; err: if (pce_dev) qce_close(pce_dev); return NULL; } EXPORT_SYMBOL(qce_open); /* * crypto engine close function. */ int qce_close(void *handle) { struct qce_device *pce_dev = (struct qce_device *) handle; if (handle == NULL) return -ENODEV; if (pce_dev->iobase) iounmap(pce_dev->iobase); if (pce_dev->coh_vmem) dma_free_coherent(pce_dev->pdev, 2*PAGE_SIZE, pce_dev->coh_vmem, pce_dev->coh_pmem); kfree(pce_dev->chan_ce_in_cmd); kfree(pce_dev->chan_ce_out_cmd); clk_put(pce_dev->ce_clk); kfree(handle); return 0; } EXPORT_SYMBOL(qce_close); int qce_hw_support(void *handle, struct ce_hw_support *ce_support) { struct qce_device *pce_dev = (struct qce_device *) handle; if (ce_support == NULL) return -EINVAL; if (pce_dev->hmac == 1) ce_support->sha1_hmac_20 = true; else ce_support->sha1_hmac_20 = false; ce_support->sha1_hmac = false; ce_support->sha256_hmac = false; ce_support->sha_hmac = false; ce_support->cmac = false; ce_support->aes_key_192 = true; ce_support->aes_xts = false; ce_support->aes_ccm = false; ce_support->ota = pce_dev->ota; ce_support->aligned_only = false; ce_support->is_shared = false; ce_support->bam = false; ce_support->max_request = 1; return 0; } EXPORT_SYMBOL(qce_hw_support); int qce_f8_req(void *handle, struct qce_f8_req *req, void *cookie, qce_comp_func_ptr_t qce_cb) { struct qce_device *pce_dev = (struct qce_device *) handle; bool key_stream_mode; dma_addr_t dst; int rc; uint32_t pad_len = ALIGN(req->data_len, ADM_CE_BLOCK_SIZE) - req->data_len; _chain_buffer_in_init(pce_dev); _chain_buffer_out_init(pce_dev); key_stream_mode = (req->data_in == NULL); /* F8 cipher input */ if (key_stream_mode) pce_dev->phy_ota_src = 0; else { pce_dev->phy_ota_src = dma_map_single(pce_dev->pdev, req->data_in, req->data_len, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (_chain_pm_buffer_in(pce_dev, pce_dev->phy_ota_src, req->data_len) < 0) { pce_dev->phy_ota_dst = 0; rc = -ENOMEM; goto bad; } } /* F8 cipher output */ if (req->data_in != req->data_out) { dst = dma_map_single(pce_dev->pdev, req->data_out, req->data_len, DMA_FROM_DEVICE); pce_dev->phy_ota_dst = dst; } else { dst = pce_dev->phy_ota_src; pce_dev->phy_ota_dst = 0; } if (_chain_pm_buffer_out(pce_dev, dst, req->data_len) < 0) { rc = -ENOMEM; goto bad; } pce_dev->ota_size = req->data_len; /* pad data */ if (pad_len) { if (!key_stream_mode && _chain_pm_buffer_in(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } if (_chain_pm_buffer_out(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } } /* finalize the ce_in and ce_out channels command lists */ if (!key_stream_mode) _ce_in_final(pce_dev, 1, req->data_len + pad_len); _ce_out_final(pce_dev, 1, req->data_len + pad_len); /* set up crypto device */ rc = _ce_f8_setup(pce_dev, req, key_stream_mode, 1, 0, req->data_len); if (rc < 0) goto bad; /* setup for callback, and issue command to adm */ pce_dev->areq = cookie; pce_dev->qce_cb = qce_cb; if (!key_stream_mode) pce_dev->chan_ce_in_cmd->complete_func = _f8_ce_in_call_back; pce_dev->chan_ce_out_cmd->complete_func = _f8_ce_out_call_back; rc = _qce_start_dma(pce_dev, !(key_stream_mode), true); if (rc == 0) return 0; bad: if (pce_dev->phy_ota_dst != 0) dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_dst, req->data_len, DMA_FROM_DEVICE); if (pce_dev->phy_ota_src != 0) dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_src, req->data_len, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); return rc; } EXPORT_SYMBOL(qce_f8_req); int qce_f8_multi_pkt_req(void *handle, struct qce_f8_multi_pkt_req *mreq, void *cookie, qce_comp_func_ptr_t qce_cb) { struct qce_device *pce_dev = (struct qce_device *) handle; uint16_t num_pkt = mreq->num_pkt; uint16_t cipher_start = mreq->cipher_start; uint16_t cipher_size = mreq->cipher_size; struct qce_f8_req *req = &mreq->qce_f8_req; uint32_t total; uint32_t pad_len; dma_addr_t dst = 0; int rc = 0; total = num_pkt * req->data_len; pad_len = ALIGN(total, ADM_CE_BLOCK_SIZE) - total; _chain_buffer_in_init(pce_dev); _chain_buffer_out_init(pce_dev); /* F8 cipher input */ pce_dev->phy_ota_src = dma_map_single(pce_dev->pdev, req->data_in, total, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (_chain_pm_buffer_in(pce_dev, pce_dev->phy_ota_src, total) < 0) { pce_dev->phy_ota_dst = 0; rc = -ENOMEM; goto bad; } /* F8 cipher output */ if (req->data_in != req->data_out) { dst = dma_map_single(pce_dev->pdev, req->data_out, total, DMA_FROM_DEVICE); pce_dev->phy_ota_dst = dst; } else { dst = pce_dev->phy_ota_src; pce_dev->phy_ota_dst = 0; } if (_chain_pm_buffer_out(pce_dev, dst, total) < 0) { rc = -ENOMEM; goto bad; } pce_dev->ota_size = total; /* pad data */ if (pad_len) { if (_chain_pm_buffer_in(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } if (_chain_pm_buffer_out(pce_dev, pce_dev->phy_ce_pad, pad_len) < 0) { rc = -ENOMEM; goto bad; } } /* finalize the ce_in and ce_out channels command lists */ _ce_in_final(pce_dev, 1, total + pad_len); _ce_out_final(pce_dev, 1, total + pad_len); /* set up crypto device */ rc = _ce_f8_setup(pce_dev, req, false, num_pkt, cipher_start, cipher_size); if (rc) goto bad; /* setup for callback, and issue command to adm */ pce_dev->areq = cookie; pce_dev->qce_cb = qce_cb; pce_dev->chan_ce_in_cmd->complete_func = _f8_ce_in_call_back; pce_dev->chan_ce_out_cmd->complete_func = _f8_ce_out_call_back; rc = _qce_start_dma(pce_dev, true, true); if (rc == 0) return 0; bad: if (pce_dev->phy_ota_dst) dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_dst, total, DMA_FROM_DEVICE); dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_src, total, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); return rc; } EXPORT_SYMBOL(qce_f8_multi_pkt_req); int qce_f9_req(void *handle, struct qce_f9_req *req, void *cookie, qce_comp_func_ptr_t qce_cb) { struct qce_device *pce_dev = (struct qce_device *) handle; int rc; uint32_t pad_len = ALIGN(req->msize, ADM_CE_BLOCK_SIZE) - req->msize; pce_dev->phy_ota_src = dma_map_single(pce_dev->pdev, req->message, req->msize, DMA_TO_DEVICE); _chain_buffer_in_init(pce_dev); rc = _chain_pm_buffer_in(pce_dev, pce_dev->phy_ota_src, req->msize); if (rc < 0) { rc = -ENOMEM; goto bad; } pce_dev->ota_size = req->msize; if (pad_len) { rc = _chain_pm_buffer_in(pce_dev, pce_dev->phy_ce_pad, pad_len); if (rc < 0) { rc = -ENOMEM; goto bad; } } _ce_in_final(pce_dev, 2, req->msize + pad_len); rc = _ce_f9_setup(pce_dev, req); if (rc < 0) goto bad; /* setup for callback, and issue command to adm */ pce_dev->areq = cookie; pce_dev->qce_cb = qce_cb; pce_dev->chan_ce_in_cmd->complete_func = _f9_ce_in_call_back; rc = _qce_start_dma(pce_dev, true, false); if (rc == 0) return 0; bad: dma_unmap_single(pce_dev->pdev, pce_dev->phy_ota_src, req->msize, DMA_TO_DEVICE); return rc; } EXPORT_SYMBOL(qce_f9_req); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Crypto Engine driver");