I should have realized it would not work with partial blocks 
; with the patch, it accepts operations with any length. As for the ipsec failure, maybe the bad IV handling is causing it. I have come up with a patch to compute and update the IV, software-only. I'd recommend leaving ipsec disabled during the boot, in case this causes a kernel crash upon use. You may bring it up later after it boots. If I got this right, then you should be able to reproduce my run exactly:
diff --git a/target/linux/ipq40xx/patches-4.19/183-crypto-qce-update-ctr-mode-iv.patch b/target/linux/ipq40xx/patches-4.19/183-crypto-qce-update-ctr-mode-iv.patch
new file mode 100644
index 0000000000..198e1ac268
--- /dev/null
+++ b/target/linux/ipq40xx/patches-4.19/183-crypto-qce-update-ctr-mode-iv.patch
@@ -0,0 +1,36 @@
+diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c
+index 7a98bf5cc967..decbfaf3feeb 100644
+--- a/drivers/crypto/qce/ablkcipher.c
++++ b/drivers/crypto/qce/ablkcipher.c
+@@ -14,6 +14,20 @@
+
+ static LIST_HEAD(ablkcipher_algs);
+
++static void qce_update_ctr_iv(u8 *iv, unsigned int ivsize, int blocksize,
++ unsigned int cryptlen)
++{
++ unsigned int nblocks;
++
++ nblocks = DIV_ROUND_UP(cryptlen, blocksize);
++ do {
++ ivsize--;
++ nblocks += iv[ivsize];
++ iv[ivsize] = (u8) nblocks;
++ nblocks >>= 8;
++ } while (ivsize);
++}
++
+ static void qce_ablkcipher_done(void *data)
+ {
+ struct crypto_async_request *async_req = data;
+@@ -45,6 +59,10 @@ static void qce_ablkcipher_done(void *data)
+ if (error < 0)
+ dev_dbg(qce->dev, "ablkcipher operation error (%x)\n", status);
+
++ if (IS_CTR(rctx->flags) && IS_AES(rctx->flags))
++ qce_update_ctr_iv(rctx->iv, rctx->ivsize, AES_BLOCK_SIZE,
++ rctx->cryptlen);
++
+ qce->async_req_done(tmpl->qce, error);
+ }
+
Please try the previous test-program again after applying the above patch. It should have the same output as I posted above.
After that, I have another the program to test CTR along with CBC-mode to see if CBC needs IV handling as well--I have a thin hope that it may be at least partly responsible for the problem reported by @cwbsw:
#include <stdint.h>
#include <sys/uio.h>
#include <sys/syscall.h>
#include <sys/socket.h>
#include <linux/if_alg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
struct afalg_ctx_st {
struct sockaddr_alg sa;
int sfd, bfd;
};
typedef struct afalg_ctx_st afalg_ctx;
#define IV_LEN 16
#define KEY_LEN 16
static char IV[] = { 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08, 0x07,
0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 };
static char KEY[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
static int CipherInit(afalg_ctx *ctx, char *alg_name, char *key,
size_t keylen, char *iv, size_t ivlen, int enc)
{
struct sockaddr_alg sa;
struct msghdr msg = { 0 };
struct cmsghdr *cmsg;
struct af_alg_iv *aiv;
struct iovec iov;
int op = enc ? ALG_OP_ENCRYPT : ALG_OP_DECRYPT;
size_t set_op_len = sizeof op;
size_t set_iv_len = offsetof(struct af_alg_iv, iv) + ivlen;
char buf[CMSG_SPACE(set_op_len) + CMSG_SPACE(set_iv_len)];
memset(&sa, 0, sizeof ctx->sa);
sa.salg_family = AF_ALG;
strcpy(sa.salg_type, "skcipher");
strncpy(sa.salg_name, alg_name, sizeof sa.salg_name);
if (( ctx->bfd = socket(AF_ALG, SOCK_SEQPACKET, 0)) < 0) {
perror("Failed to open socket");
goto err;
}
if (bind(ctx->bfd, (struct sockaddr *)&sa, sizeof sa) < 0) {
perror("Failed to bind socket");
goto err;
}
if (setsockopt(ctx->bfd, SOL_ALG, ALG_SET_KEY, KEY, KEY_LEN) < 0) {
perror("Failed to set key");
}
if ((ctx->sfd = accept(ctx->bfd, NULL, 0)) < 0) {
perror("Socket accept failed");
goto err;
}
memset(&buf, 0, sizeof buf);
msg.msg_control = buf;
/* set op */
msg.msg_controllen = CMSG_SPACE(set_op_len);
cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = SOL_ALG;
cmsg->cmsg_type = ALG_SET_OP;
cmsg->cmsg_len = CMSG_LEN(set_op_len);
memcpy(CMSG_DATA(cmsg), &op, sizeof op);
/* set IV */
msg.msg_controllen += CMSG_SPACE(set_iv_len);
cmsg = CMSG_NXTHDR(&msg, cmsg);
cmsg->cmsg_level = SOL_ALG;
cmsg->cmsg_type = ALG_SET_IV;
cmsg->cmsg_len = CMSG_LEN(set_iv_len);
aiv = (void *)CMSG_DATA(cmsg);
aiv->ivlen = IV_LEN;
memcpy(aiv->iv, iv, IV_LEN);
iov.iov_base = NULL;
iov.iov_len = 0;
if (sendmsg(ctx->sfd, &msg, 0) < 0) {
perror("sendmsg: Failed to set op, iv");
goto err;
}
return 1;
err:
if (ctx->bfd >= 0)
close(ctx->bfd);
if (ctx->sfd >= 0)
close(ctx->sfd);
ctx->bfd = ctx->sfd = -1;
return 0;
}
static int CipherUpdate(afalg_ctx *ctx, char *out, size_t *outl,
const char* in, size_t inl)
{
struct msghdr msg = { 0 };
struct cmsghdr *cmsg;
struct iovec iov;
ssize_t nbytes;
int ret = 1;
iov.iov_base = (void *)in;
iov.iov_len = inl;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
if ((nbytes = send(ctx->sfd, in, inl, MSG_MORE)) != (ssize_t) inl) {
fprintf(stderr, "CipherUpdate: sent %zd bytes != inl %zd\n", nbytes, inl);
if (nbytes <= 0)
return 0;
ret = 0;
}
if ((nbytes = read(ctx->sfd, out, (size_t) nbytes)) != (ssize_t) inl) {
fprintf(stderr, "CipherUpdate: read %zd bytes != inl %zd\n", nbytes, inl);
if (nbytes < 0)
return 0;
ret = 0;
}
if (outl != NULL)
*outl = (size_t) nbytes;
return ret;
}
static int CipherFinal(afalg_ctx *ctx)
{
close(ctx->sfd);
close(ctx->bfd);
ctx->bfd = ctx->sfd = -1;
return 1;
}
static char *CipherHex(unsigned char *text, unsigned int text_len)
{
char *res = malloc(text_len * 3 + 1);
char *res_ptr = res;
unsigned char *text_ptr = text;
if (res == NULL)
return NULL;
for(int i=0; i < text_len; i++) {
snprintf(res_ptr, 4, "%02hhx ", *(text_ptr++));
res_ptr += 3;
}
return res;
}
static int do_enc(char *cipher, char *key, size_t keylen, char* iv,
size_t ivlen, const char *text, size_t len, int enc, int n)
{
afalg_ctx ctx;
char *cipher_hex;
char cipher_out[1024];
int roundlen = len / n;
size_t outl;
if (!CipherInit(&ctx, cipher, key, keylen, iv, ivlen, enc)) {
fprintf(stderr, "Error in CipherInit\n");
return -1;
}
for (int i = 0; i < len; i += roundlen) {
if (i + roundlen > len)
roundlen = len - i;
if(!CipherUpdate(&ctx, cipher_out, &outl, text + i, roundlen)) {
fprintf(stderr, "Error in CipherUpdate\n");
if (outl < 1)
return -1;
}
cipher_hex = CipherHex(cipher_out, outl);
printf("%s: %.*s - %s\n", cipher, roundlen, text + i, cipher_hex);
free(cipher_hex);
}
if(!CipherFinal(&ctx)) {
fprintf(stderr, "Error in CipherFinal_ex\n");
return -1;
}
return 0;
}
int main(int argc, char **argv)
{
afalg_ctx ctx;
char *cipher_hex;
char cipher_out[1024];
char text[] = "0123456789abcdefghijklmnopqrstuv"
"0123456789abcdefghijklmnopqrstuv";
do_enc("cbc(aes)", KEY, KEY_LEN, IV, IV_LEN, text, sizeof text - 1, 1, 2);
printf("\n");
do_enc("ctr(aes)", KEY, KEY_LEN, IV, IV_LEN, text, sizeof text - 1, 1, 2);
printf("\n");
do_enc("cbc(aes)", KEY, KEY_LEN, IV, IV_LEN, text, sizeof text - 1, 1, 4);
printf("\n");
do_enc("ctr(aes)", KEY, KEY_LEN, IV, IV_LEN, text, sizeof text - 1, 1, 4);
printf("\n");
return 0;
}
Here's my output:
cbc(aes): 0123456789abcdefghijklmnopqrstuv - ff 14 db e4 05 cc 0e e2 4d 0d e4 12 89 f0 fc 98 89 c5 61 23 4e d8 19 f1 32 57 3e fe 59 de 7a 21
cbc(aes): 0123456789abcdefghijklmnopqrstuv - e1 81 26 cd 21 30 f9 a8 5f e0 7b bc 0f bc e8 01 ca 04 03 03 14 e1 03 02 a0 32 af a5 48 e7 37 5d
ctr(aes): 0123456789abcdefghijklmnopqrstuv - 10 98 cb a1 80 79 6d df 3c 26 9d be 0f ca fc 0c 20 ce cd cf 1e 32 0d 2a 01 c2 e0 9e 3a 4d 74 29
ctr(aes): 0123456789abcdefghijklmnopqrstuv - 8b ee ca 8f da f6 5f 8d e9 fa d4 3d 13 e8 11 f3 21 a6 f6 3f 97 ba a5 c8 3b 28 5d 3f fd 13 7a 3d
cbc(aes): 0123456789abcdef - ff 14 db e4 05 cc 0e e2 4d 0d e4 12 89 f0 fc 98
cbc(aes): ghijklmnopqrstuv - 89 c5 61 23 4e d8 19 f1 32 57 3e fe 59 de 7a 21
cbc(aes): 0123456789abcdef - e1 81 26 cd 21 30 f9 a8 5f e0 7b bc 0f bc e8 01
cbc(aes): ghijklmnopqrstuv - ca 04 03 03 14 e1 03 02 a0 32 af a5 48 e7 37 5d
ctr(aes): 0123456789abcdef - 10 98 cb a1 80 79 6d df 3c 26 9d be 0f ca fc 0c
ctr(aes): ghijklmnopqrstuv - 20 ce cd cf 1e 32 0d 2a 01 c2 e0 9e 3a 4d 74 29
ctr(aes): 0123456789abcdef - 8b ee ca 8f da f6 5f 8d e9 fa d4 3d 13 e8 11 f3
ctr(aes): ghijklmnopqrstuv - 21 a6 f6 3f 97 ba a5 c8 3b 28 5d 3f fd 13 7a 3d
PS: The patch is to be applied along with the previous one. As before, expect some fuzziness when applying it. Adjust line numbers as appropriate. I have compile-tested it here.
Edited again to use tabs instead of spaces, so that hopefully, copy & paste will work; also, notice that unlike last time, this patch applies to openwrt, not directly to kernel.
.
but this is not happening this week for sure... See if you can document encryption/decryption of partial blocks (len < 16 bytes).
Just feed it to