I just updated aes-gcm module and diff, seems that I did not remember correct date or we are in different time zone. I mean my original code was not fetched on 2023/01/02. Now it works. I will test the performance to confirm the hw crypto really works. May compare with sw crypto.
#include <asm/octeon/octeon.h>
#include "octeon-crypto.h"
-MODULE_DESCRIPTION("GHASH & AES-GCM using Octeon HW Crypto");
-MODULE_AUTHOR("Sebastian Gottschall <s.gottschall@dd-wrt.com>");
-MODULE_LICENSE("GPL v2");
#define GHASH_BLOCK_SIZE 16
#define GHASH_DIGEST_SIZE 16
@@ -53,7 +50,7 @@
return 0;
}
-static void ghash_do_update(int blocks, u64 dg[], const char *src, struct ghash_key *key, const char *head)
+static __always_inline void ghash_do_update(int blocks, u64 dg[], const char *src, struct ghash_key *key, const char *head)
{
write_octeon_64bit_gfm_poly((uint64_t) 0xe100);
write_octeon_64bit_gfm_resinp(dg[0], 0);
@@ -169,7 +166,7 @@
.descsize = sizeof(struct ghash_desc_ctx),
};
-static void __octeon_aes_encrypt(u32 *rk, u8 *out, u8 *in, u32 keylen)
+static __always_inline void __octeon_aes_encrypt(u32 *rk, u8 *out, u8 *in, u32 keylen)
{
__be64 *dataout = (__be64 *)out;
__be64 *data = (__be64 *)in;
@@ -178,7 +175,7 @@
write_octeon_64bit_aes_key(key[1], 1);
write_octeon_64bit_aes_key(key[2], 2);
write_octeon_64bit_aes_key(key[3], 3);
- write_octeon_64bit_aes_keylength(keylen / 8 - 1);
+ write_octeon_64bit_aes_keylength(keylen);
write_octeon_64bit_aes_enc0(*data++);
write_octeon_64bit_aes_enc1(*data);
*dataout++ = read_octeon_64bit_aes_result(0);
@@ -191,19 +188,21 @@
unsigned long flags;
struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
u8 key[GHASH_BLOCK_SIZE];
- int ret;
-
+ int ret,i;
ret = crypto_aes_expand_key(&ctx->aes_key, inkey, keylen);
if (ret) {
tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
-
+ for (i=0;i<ctx->aes_key.key_length / 4;i++)
+ ctx->aes_key.key_enc[i] = cpu_to_le32(ctx->aes_key.key_enc[i]);
+ ctx->aes_key.key_length = ctx->aes_key.key_length / 8 - 1;
flags = octeon_crypto_enable(&state);
__octeon_aes_encrypt(ctx->aes_key.key_enc, key, (u8[AES_BLOCK_SIZE]) { },
ctx->aes_key.key_length);
octeon_crypto_disable(&state, flags);
+
return __ghash_setkey(&ctx->ghash_key, key, sizeof(be128));
}
}
@@ -287,7 +286,6 @@
{
u8 mac[AES_BLOCK_SIZE];
u128 lengths;
-
lengths.a = cpu_to_be64(req->assoclen * 8);
lengths.b = cpu_to_be64(cryptlen * 8);
@@ -314,7 +312,7 @@
if (req->assoclen)
gcm_calculate_auth_mac(req, dg);
- memcpy(iv, req->iv, GCM_IV_SIZE);
+ memcpy(iv, req->iv, GCM_IV_SIZE);
put_unaligned_be32(1, iv + GCM_IV_SIZE);
err = skcipher_walk_aead_encrypt(&walk, req, false);
@@ -327,7 +325,6 @@
u8 *dst = walk.dst.virt.addr;
u8 *src = walk.src.virt.addr;
int remaining = blocks;
-
do {
__octeon_aes_encrypt(ctx->aes_key.key_enc, ks, iv, ctx->aes_key.key_length);
crypto_xor_cpy(dst, src, ks, AES_BLOCK_SIZE);
@@ -341,20 +338,17 @@
err = skcipher_walk_done(&walk, walk.nbytes % (2 * AES_BLOCK_SIZE));
}
- if (walk.nbytes) {
- __octeon_aes_encrypt(ctx->aes_key.key_enc, ks, iv, ctx->aes_key.key_length);
- if (walk.nbytes > AES_BLOCK_SIZE) {
- crypto_inc(iv, AES_BLOCK_SIZE);
- __octeon_aes_encrypt(ctx->aes_key.key_enc, ks + AES_BLOCK_SIZE, iv, ctx->aes_key.key_length);
- }
- }
- /* handle the tail */
if (walk.nbytes) {
u8 buf[GHASH_BLOCK_SIZE];
unsigned int nbytes = walk.nbytes;
u8 *dst = walk.dst.virt.addr;
u8 *head = NULL;
+ __octeon_aes_encrypt(ctx->aes_key.key_enc, ks, iv, ctx->aes_key.key_length);
+ if (walk.nbytes > AES_BLOCK_SIZE) {
+ crypto_inc(iv, AES_BLOCK_SIZE);
+ __octeon_aes_encrypt(ctx->aes_key.key_enc, ks + AES_BLOCK_SIZE, iv, ctx->aes_key.key_length);
+ }
crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, ks, walk.nbytes);
@@ -428,6 +422,9 @@
err = skcipher_walk_done(&walk, walk.nbytes % (2 * AES_BLOCK_SIZE));
}
if (walk.nbytes) {
+ const u8 *src = walk.src.virt.addr;
+ const u8 *head = NULL;
+ unsigned int nbytes = walk.nbytes;
if (walk.nbytes > AES_BLOCK_SIZE) {
u8 *iv2 = iv + AES_BLOCK_SIZE;
@@ -437,13 +434,6 @@
__octeon_aes_encrypt(ctx->aes_key.key_enc, iv2, iv2, ctx->aes_key.key_length);
}
__octeon_aes_encrypt(ctx->aes_key.key_enc, iv, iv, ctx->aes_key.key_length);
- }
-
- /* handle the tail */
- if (walk.nbytes) {
- const u8 *src = walk.src.virt.addr;
- const u8 *head = NULL;
- unsigned int nbytes = walk.nbytes;
if (walk.nbytes > GHASH_BLOCK_SIZE) {
head = src;
@@ -516,3 +506,7 @@
module_init(ghash_ce_mod_init);
module_exit(ghash_ce_mod_exit);
+
+MODULE_DESCRIPTION("GHASH & AES-GCM using Octeon HW Crypto");
+MODULE_AUTHOR("Sebastian Gottschall <s.gottschall@dd-wrt.com>");
+MODULE_LICENSE("GPL v2");
