Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/2183217/?format=api
{ "id": 2183217, "url": "http://patchwork.ozlabs.org/api/patches/2183217/?format=api", "web_url": "http://patchwork.ozlabs.org/project/linuxppc-dev/patch/20260112192035.10427-3-ebiggers@kernel.org/", "project": { "id": 2, "url": "http://patchwork.ozlabs.org/api/projects/2/?format=api", "name": "Linux PPC development", "link_name": "linuxppc-dev", "list_id": "linuxppc-dev.lists.ozlabs.org", "list_email": "linuxppc-dev@lists.ozlabs.org", "web_url": "https://github.com/linuxppc/wiki/wiki", "scm_url": "https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git", "webscm_url": "https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git/", "list_archive_url": "https://lore.kernel.org/linuxppc-dev/", "list_archive_url_format": "https://lore.kernel.org/linuxppc-dev/{}/", "commit_url_format": "https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git/commit/?id={}" }, "msgid": "<20260112192035.10427-3-ebiggers@kernel.org>", "list_archive_url": "https://lore.kernel.org/linuxppc-dev/20260112192035.10427-3-ebiggers@kernel.org/", "date": "2026-01-12T19:20:00", "name": "[v2,02/35] lib/crypto: aes: Introduce improved AES library", "commit_ref": null, "pull_url": null, "state": "handled-elsewhere", "archived": false, "hash": "53ff6ba9e629f6a02bcc5fdf599351f74610e607", "submitter": { "id": 74690, "url": "http://patchwork.ozlabs.org/api/people/74690/?format=api", "name": "Eric Biggers", "email": "ebiggers@kernel.org" }, "delegate": null, "mbox": "http://patchwork.ozlabs.org/project/linuxppc-dev/patch/20260112192035.10427-3-ebiggers@kernel.org/mbox/", "series": [ { "id": 488089, "url": "http://patchwork.ozlabs.org/api/series/488089/?format=api", "web_url": "http://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=488089", "date": "2026-01-12T19:19:58", "name": "AES library improvements", "version": 2, "mbox": "http://patchwork.ozlabs.org/series/488089/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/2183217/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/2183217/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "\n <linuxppc-dev+bounces-15577-incoming=patchwork.ozlabs.org@lists.ozlabs.org>", "X-Original-To": [ "incoming@patchwork.ozlabs.org", "linuxppc-dev@lists.ozlabs.org" ], "Delivered-To": "patchwork-incoming@legolas.ozlabs.org", "Authentication-Results": [ "legolas.ozlabs.org;\n\tdkim=pass (2048-bit key;\n unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256\n header.s=k20201202 header.b=LpvdVmF4;\n\tdkim-atps=neutral", "legolas.ozlabs.org;\n spf=pass (sender SPF authorized) smtp.mailfrom=lists.ozlabs.org\n (client-ip=112.213.38.117; helo=lists.ozlabs.org;\n envelope-from=linuxppc-dev+bounces-15577-incoming=patchwork.ozlabs.org@lists.ozlabs.org;\n receiver=patchwork.ozlabs.org)", "lists.ozlabs.org;\n arc=none smtp.remote-ip=172.105.4.254", "lists.ozlabs.org;\n dmarc=pass (p=quarantine dis=none) header.from=kernel.org", "lists.ozlabs.org;\n\tdkim=pass (2048-bit key;\n unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256\n header.s=k20201202 header.b=LpvdVmF4;\n\tdkim-atps=neutral", "lists.ozlabs.org;\n spf=pass (sender SPF authorized) smtp.mailfrom=kernel.org\n (client-ip=172.105.4.254; helo=tor.source.kernel.org;\n envelope-from=ebiggers@kernel.org; receiver=lists.ozlabs.org)" ], "Received": [ "from lists.ozlabs.org (lists.ozlabs.org [112.213.38.117])\n\t(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)\n\t key-exchange x25519)\n\t(No client certificate requested)\n\tby legolas.ozlabs.org (Postfix) with ESMTPS id 4dqj6Y2YHRz1xpY\n\tfor <incoming@patchwork.ozlabs.org>; Tue, 13 Jan 2026 06:26:29 +1100 (AEDT)", "from boromir.ozlabs.org (localhost [127.0.0.1])\n\tby lists.ozlabs.org (Postfix) with ESMTP id 4dqj3Y0XfWz3bp0;\n\tTue, 13 Jan 2026 06:23:53 +1100 (AEDT)", "from tor.source.kernel.org (tor.source.kernel.org [172.105.4.254])\n\t(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)\n\t key-exchange x25519)\n\t(No client certificate requested)\n\tby lists.ozlabs.org (Postfix) with ESMTPS id 4dqj3W2vLKz2xS2\n\tfor <linuxppc-dev@lists.ozlabs.org>; Tue, 13 Jan 2026 06:23:51 +1100 (AEDT)", "from smtp.kernel.org (transwarp.subspace.kernel.org [100.75.92.58])\n\tby tor.source.kernel.org (Postfix) with ESMTP id 3102060051;\n\tMon, 12 Jan 2026 19:23:19 +0000 (UTC)", "by smtp.kernel.org (Postfix) with ESMTPSA id 73CE5C2BC86;\n\tMon, 12 Jan 2026 19:23:18 +0000 (UTC)" ], "ARC-Seal": "i=1; a=rsa-sha256; d=lists.ozlabs.org; s=201707; t=1768245832;\n\tcv=none;\n b=A/5srSlTdSYO6rX6JPKRuNT9ZIekc4KnbQLscqcr8cS8+XhSa2Ss6weeildKIJG9G1OkuyT/EBxfhloockEM3ZW3KHJtnrL+WL1Se+QjMc1uerKwhe8bd3to1VAz2bbwB6Y3dKP8oJVgZZcrnQJipXaLnTq0LGdtG+pB2I75zgtR8pQ1U2glISrR/8njtuPjuTMeJQkXcSwg+Ix3WLPbsiRXBlL4o0CjhmqQ+9hQhLPAjyGX119CI+zEVHUyblm9X+e3Zfg8LfJwXgL9iEwB3IptLBMkGhZM2BdHUESyLW3CKzN7LSKlnlMIF5nJT/XAXuh64otvMI4gJ9clbZSAoQ==", "ARC-Message-Signature": "i=1; a=rsa-sha256; d=lists.ozlabs.org; s=201707;\n\tt=1768245832; c=relaxed/relaxed;\n\tbh=AA1NCVt2V0rpl6BaqB7EoiD0n8ZmsE2SaLK3XFGW9XM=;\n\th=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References:\n\t MIME-Version;\n b=KSscXdNJZ59Rr+Iy/s4CyK8jy/nh0NtHdRBBANodGCs/fbbBnArVwleiRq2M6kuSgwgXUqwhkT60HBg7Mk2Dwqx3LGfQhE/u0TlgJ+mHIBZTxrP2YEB7zljJBVtRLyoYQDN5htTQr14SN48q+Hl8ak0im66NpSROy0vYsrKpTi95zW6Kp6gzf9vEUbX7gofPdcCvMDGR/7RSpxAxhI3MFd+4Fwj4gUgCRuehG2vd6wORGEdfx7yqg5qdJ3TuPPfbpK1MXYeGv2+akp0hg2gj6ClobTJO3Ryaea8O6MwU3/yj98dWKEhzfokfFXP9HHJc2Rvnt8xtry7I0Qun2YG6ZA==", "ARC-Authentication-Results": "i=1; lists.ozlabs.org;\n dmarc=pass (p=quarantine dis=none) header.from=kernel.org;\n dkim=pass (2048-bit key;\n unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256\n header.s=k20201202 header.b=LpvdVmF4; dkim-atps=neutral;\n spf=pass (client-ip=172.105.4.254; helo=tor.source.kernel.org;\n envelope-from=ebiggers@kernel.org;\n receiver=lists.ozlabs.org) smtp.mailfrom=kernel.org", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org;\n\ts=k20201202; t=1768245798;\n\tbh=p4ecLxqGxoblVFXBlv9wFmcXn8PSYdygfhvo28J5HNk=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=LpvdVmF4/0I/ZapvM7HXcdj05jDaK/5jl0+SfjKEqSmqfQD2+Xoj8e9xriX6QrYkq\n\t G8+JZbTAP+PBjAP1AMV6vScHocdx29CZEasqSMiZW53Om5nU6UnWAPyEtKkMpTR4lm\n\t /LQKQFFLld1V2mtM2Rt8C6aAuB5+tE4vd8DUoWL7UvhCtXcyHODyRL/bVQJtnTzGMa\n\t OlBIRzm8UJGC4ZLUolUrYdD0ua35nxoPkhVhF8MeCjDCNf9sHMx3nyQUqC+WUumj8F\n\t t1v3sdNpOaA4wVqOo76Xwizw5aHp8PJNqBPCNvMtbcd5zBcYfw01TAcu4LmD4Ld0HF\n\t Ms9I5yGZebxxw==", "From": "Eric Biggers <ebiggers@kernel.org>", "To": "linux-crypto@vger.kernel.org", "Cc": "linux-kernel@vger.kernel.org,\n\tArd Biesheuvel <ardb@kernel.org>,\n\t\"Jason A . Donenfeld\" <Jason@zx2c4.com>,\n\tHerbert Xu <herbert@gondor.apana.org.au>,\n\tlinux-arm-kernel@lists.infradead.org,\n\tlinuxppc-dev@lists.ozlabs.org,\n\tlinux-riscv@lists.infradead.org,\n\tlinux-s390@vger.kernel.org,\n\tsparclinux@vger.kernel.org,\n\tx86@kernel.org,\n\tHolger Dengler <dengler@linux.ibm.com>,\n\tHarald Freudenberger <freude@linux.ibm.com>,\n\tEric Biggers <ebiggers@kernel.org>", "Subject": "[PATCH v2 02/35] lib/crypto: aes: Introduce improved AES library", "Date": "Mon, 12 Jan 2026 11:20:00 -0800", "Message-ID": "<20260112192035.10427-3-ebiggers@kernel.org>", "X-Mailer": "git-send-email 2.52.0", "In-Reply-To": "<20260112192035.10427-1-ebiggers@kernel.org>", "References": "<20260112192035.10427-1-ebiggers@kernel.org>", "X-Mailing-List": "linuxppc-dev@lists.ozlabs.org", "List-Id": "<linuxppc-dev.lists.ozlabs.org>", "List-Help": "<mailto:linuxppc-dev+help@lists.ozlabs.org>", "List-Owner": "<mailto:linuxppc-dev+owner@lists.ozlabs.org>", "List-Post": "<mailto:linuxppc-dev@lists.ozlabs.org>", "List-Archive": "<https://lore.kernel.org/linuxppc-dev/>,\n <https://lists.ozlabs.org/pipermail/linuxppc-dev/>", "List-Subscribe": "<mailto:linuxppc-dev+subscribe@lists.ozlabs.org>,\n <mailto:linuxppc-dev+subscribe-digest@lists.ozlabs.org>,\n <mailto:linuxppc-dev+subscribe-nomail@lists.ozlabs.org>", "List-Unsubscribe": "<mailto:linuxppc-dev+unsubscribe@lists.ozlabs.org>", "Precedence": "list", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-Spam-Status": "No, score=-0.2 required=3.0 tests=DKIMWL_WL_HIGH,DKIM_SIGNED,\n\tDKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,SPF_HELO_NONE,SPF_PASS\n\tautolearn=disabled version=4.0.1 OzLabs 8", "X-Spam-Checker-Version": "SpamAssassin 4.0.1 (2024-03-25) on lists.ozlabs.org" }, "content": "The kernel's AES library currently has the following issues:\n\n- It doesn't take advantage of the architecture-optimized AES code,\n including the implementations using AES instructions.\n\n- It's much slower than even the other software AES implementations: 2-4\n times slower than \"aes-generic\", \"aes-arm\", and \"aes-arm64\".\n\n- It requires that both the encryption and decryption round keys be\n computed and cached. This is wasteful for users that need only the\n forward (encryption) direction of the cipher: the key struct is 484\n bytes when only 244 are actually needed. This missed optimization is\n very common, as many AES modes (e.g. GCM, CFB, CTR, CMAC, and even the\n tweak key in XTS) use the cipher only in the forward (encryption)\n direction even when doing decryption.\n\n- It doesn't provide the flexibility to customize the prepared key\n format. The API is defined to do key expansion, and several callers\n in drivers/crypto/ use it specifically to expand the key. This is an\n issue when integrating the existing powerpc, s390, and sparc code,\n which is necessary to provide full parity with the traditional API.\n\nTo resolve these issues, I'm proposing the following changes:\n\n1. New structs 'aes_key' and 'aes_enckey' are introduced, with\n corresponding functions aes_preparekey() and aes_prepareenckey().\n\n Generally these structs will include the encryption+decryption round\n keys and the encryption round keys, respectively. However, the exact\n format will be under control of the architecture-specific AES code.\n\n (The verb \"prepare\" is chosen over \"expand\" since key expansion isn't\n necessarily done. It's also consistent with hmac*_preparekey().)\n\n2. aes_encrypt() and aes_decrypt() will be changed to operate on the new\n structs instead of struct crypto_aes_ctx.\n\n3. aes_encrypt() and aes_decrypt() will use architecture-optimized code\n when available, or else fall back to a new generic AES implementation\n that unifies the existing two fragmented generic AES implementations.\n\n The new generic AES implementation uses tables for both SubBytes and\n MixColumns, making it almost as fast as \"aes-generic\". However,\n instead of aes-generic's huge 8192-byte tables per direction, it uses\n only 1024 bytes for encryption and 1280 bytes for decryption (similar\n to \"aes-arm\"). The cost is just some extra rotations.\n\n The new generic AES implementation also includes table prefetching,\n making it have some \"constant-time hardening\". That's an improvement\n from aes-generic which has no constant-time hardening.\n\n It does slightly regress in constant-time hardening vs. the old\n lib/crypto/aes.c which had smaller tables, and from aes-fixed-time\n which disabled IRQs on top of that. But I think this is tolerable.\n The real solutions for constant-time AES are AES instructions or\n bit-slicing. The table-based code remains a best-effort fallback for\n the increasingly-rare case where a real solution is unavailable.\n\n4. crypto_aes_ctx and aes_expandkey() will remain for now, but only for\n callers that are using them specifically for the AES key expansion\n (as opposed to en/decrypting data with the AES library).\n\nThis commit begins the migration process by introducing the new structs\nand functions, backed by the new generic AES implementation.\n\nTo allow callers to be incrementally converted, aes_encrypt() and\naes_decrypt() are temporarily changed into macros that use a _Generic\nexpression to call either the old functions (which take crypto_aes_ctx)\nor the new functions (which take the new types). Once all callers have\nbeen updated, these macros will go away, the old functions will be\nremoved, and the \"_new\" suffix will be dropped from the new functions.\n\nAcked-by: Ard Biesheuvel <ardb@kernel.org>\nSigned-off-by: Eric Biggers <ebiggers@kernel.org>\n---\n include/crypto/aes.h | 157 +++++++++++++++--\n lib/crypto/Kconfig | 4 +\n lib/crypto/Makefile | 11 +-\n lib/crypto/aes.c | 399 ++++++++++++++++++++++++++++++++++++-------\n 4 files changed, 501 insertions(+), 70 deletions(-)", "diff": "diff --git a/include/crypto/aes.h b/include/crypto/aes.h\nindex 9339da7c20a8..e8b83d4849fe 100644\n--- a/include/crypto/aes.h\n+++ b/include/crypto/aes.h\n@@ -16,10 +16,64 @@\n #define AES_KEYSIZE_256\t\t32\n #define AES_BLOCK_SIZE\t\t16\n #define AES_MAX_KEYLENGTH\t(15 * 16)\n #define AES_MAX_KEYLENGTH_U32\t(AES_MAX_KEYLENGTH / sizeof(u32))\n \n+union aes_enckey_arch {\n+\tu32 rndkeys[AES_MAX_KEYLENGTH_U32];\n+};\n+\n+union aes_invkey_arch {\n+\tu32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];\n+};\n+\n+/**\n+ * struct aes_enckey - An AES key prepared for encryption\n+ * @len: Key length in bytes: 16 for AES-128, 24 for AES-192, 32 for AES-256.\n+ * @nrounds: Number of rounds: 10 for AES-128, 12 for AES-192, 14 for AES-256.\n+ *\t This is '6 + @len / 4' and is cached so that AES implementations\n+ *\t that need it don't have to recompute it for each en/decryption.\n+ * @padding: Padding to make offsetof(@k) be a multiple of 16, so that aligning\n+ *\t this struct to a 16-byte boundary results in @k also being 16-byte\n+ *\t aligned. Users aren't required to align this struct to 16 bytes,\n+ *\t but it may slightly improve performance.\n+ * @k: This typically contains the AES round keys as an array of '@nrounds + 1'\n+ * groups of four u32 words. However, architecture-specific implementations\n+ * of AES may store something else here, e.g. just the raw key if it's all\n+ * they need.\n+ *\n+ * Note that this struct is about half the size of struct aes_key. This is\n+ * separate from struct aes_key so that modes that need only AES encryption\n+ * (e.g. AES-GCM, AES-CTR, AES-CMAC, tweak key in AES-XTS) don't incur the time\n+ * and space overhead of computing and caching the decryption round keys.\n+ *\n+ * Note that there's no decryption-only equivalent (i.e. \"struct aes_deckey\"),\n+ * since (a) it's rare that modes need decryption-only, and (b) some AES\n+ * implementations use the same @k for both encryption and decryption, either\n+ * always or conditionally; in the latter case both @k and @inv_k are needed.\n+ */\n+struct aes_enckey {\n+\tu32 len;\n+\tu32 nrounds;\n+\tu32 padding[2];\n+\tunion aes_enckey_arch k;\n+};\n+\n+/**\n+ * struct aes_key - An AES key prepared for encryption and decryption\n+ * @aes_enckey: Common fields and the key prepared for encryption\n+ * @inv_k: This generally contains the round keys for the AES Equivalent\n+ *\t Inverse Cipher, as an array of '@nrounds + 1' groups of four u32\n+ *\t words. However, architecture-specific implementations of AES may\n+ *\t store something else here. For example, they may leave this field\n+ *\t uninitialized if they use @k for both encryption and decryption.\n+ */\n+struct aes_key {\n+\tstruct aes_enckey; /* Include all fields of aes_enckey. */\n+\tunion aes_invkey_arch inv_k;\n+};\n+\n /*\n * Please ensure that the first two fields are 16-byte aligned\n * relative to the start of the structure, i.e., don't move them!\n */\n struct crypto_aes_ctx {\n@@ -32,11 +86,11 @@ extern const u32 crypto_ft_tab[4][256] ____cacheline_aligned;\n extern const u32 crypto_it_tab[4][256] ____cacheline_aligned;\n \n /*\n * validate key length for AES algorithms\n */\n-static inline int aes_check_keylen(unsigned int keylen)\n+static inline int aes_check_keylen(size_t keylen)\n {\n \tswitch (keylen) {\n \tcase AES_KEYSIZE_128:\n \tcase AES_KEYSIZE_192:\n \tcase AES_KEYSIZE_256:\n@@ -67,27 +121,108 @@ int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,\n */\n int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,\n \t\t unsigned int key_len);\n \n /**\n- * aes_encrypt - Encrypt a single AES block\n- * @ctx:\tContext struct containing the key schedule\n- * @out:\tBuffer to store the ciphertext\n- * @in:\t\tBuffer containing the plaintext\n+ * aes_preparekey() - Prepare an AES key for encryption and decryption\n+ * @key: (output) The key structure to initialize\n+ * @in_key: The raw AES key\n+ * @key_len: Length of the raw key in bytes. Should be either AES_KEYSIZE_128,\n+ *\t AES_KEYSIZE_192, or AES_KEYSIZE_256.\n+ *\n+ * This prepares an AES key for both the encryption and decryption directions of\n+ * the block cipher. Typically this involves expanding the raw key into both\n+ * the standard round keys and the Equivalent Inverse Cipher round keys, but\n+ * some architecture-specific implementations don't do the full expansion here.\n+ *\n+ * The caller is responsible for zeroizing both the struct aes_key and the raw\n+ * key once they are no longer needed.\n+ *\n+ * If you don't need decryption support, use aes_prepareenckey() instead.\n+ *\n+ * Return: 0 on success or -EINVAL if the given key length is invalid. No other\n+ *\t errors are possible, so callers that always pass a valid key length\n+ *\t don't need to check for errors.\n+ *\n+ * Context: Any context.\n+ */\n+int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len);\n+\n+/**\n+ * aes_prepareenckey() - Prepare an AES key for encryption-only\n+ * @key: (output) The key structure to initialize\n+ * @in_key: The raw AES key\n+ * @key_len: Length of the raw key in bytes. Should be either AES_KEYSIZE_128,\n+ *\t AES_KEYSIZE_192, or AES_KEYSIZE_256.\n+ *\n+ * This prepares an AES key for only the encryption direction of the block\n+ * cipher. Typically this involves expanding the raw key into only the standard\n+ * round keys, resulting in a struct about half the size of struct aes_key.\n+ *\n+ * The caller is responsible for zeroizing both the struct aes_enckey and the\n+ * raw key once they are no longer needed.\n+ *\n+ * Note that while the resulting prepared key supports only AES encryption, it\n+ * can still be used for decrypting in a mode of operation that uses AES in only\n+ * the encryption (forward) direction, for example counter mode.\n+ *\n+ * Return: 0 on success or -EINVAL if the given key length is invalid. No other\n+ *\t errors are possible, so callers that always pass a valid key length\n+ *\t don't need to check for errors.\n+ *\n+ * Context: Any context.\n+ */\n+int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len);\n+\n+typedef union {\n+\tconst struct aes_enckey *enc_key;\n+\tconst struct aes_key *full_key;\n+} aes_encrypt_arg __attribute__ ((__transparent_union__));\n+\n+/**\n+ * aes_encrypt() - Encrypt a single AES block\n+ * @key: The AES key, as a pointer to either an encryption-only key\n+ *\t (struct aes_enckey) or a full, bidirectional key (struct aes_key).\n+ * @out: Buffer to store the ciphertext block\n+ * @in: Buffer containing the plaintext block\n+ *\n+ * Context: Any context.\n */\n-void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);\n+#define aes_encrypt(key, out, in) \\\n+\t_Generic((key), \\\n+\t\t struct crypto_aes_ctx *: aes_encrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \\\n+\t\t const struct crypto_aes_ctx *: aes_encrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \\\n+\t\t struct aes_enckey *: aes_encrypt_new((const struct aes_enckey *)(key), (out), (in)), \\\n+\t\t const struct aes_enckey *: aes_encrypt_new((const struct aes_enckey *)(key), (out), (in)), \\\n+\t\t struct aes_key *: aes_encrypt_new((const struct aes_key *)(key), (out), (in)), \\\n+\t\t const struct aes_key *: aes_encrypt_new((const struct aes_key *)(key), (out), (in)))\n+void aes_encrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);\n+void aes_encrypt_new(aes_encrypt_arg key, u8 out[at_least AES_BLOCK_SIZE],\n+\t\t const u8 in[at_least AES_BLOCK_SIZE]);\n \n /**\n- * aes_decrypt - Decrypt a single AES block\n- * @ctx:\tContext struct containing the key schedule\n- * @out:\tBuffer to store the plaintext\n- * @in:\t\tBuffer containing the ciphertext\n+ * aes_decrypt() - Decrypt a single AES block\n+ * @key: The AES key, previously initialized by aes_preparekey()\n+ * @out: Buffer to store the plaintext block\n+ * @in: Buffer containing the ciphertext block\n+ *\n+ * Context: Any context.\n */\n-void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);\n+#define aes_decrypt(key, out, in) \\\n+\t_Generic((key), \\\n+\t\t struct crypto_aes_ctx *: aes_decrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \\\n+\t\t const struct crypto_aes_ctx *: aes_decrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \\\n+\t\t struct aes_key *: aes_decrypt_new((const struct aes_key *)(key), (out), (in)), \\\n+\t\t const struct aes_key *: aes_decrypt_new((const struct aes_key *)(key), (out), (in)))\n+void aes_decrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);\n+void aes_decrypt_new(const struct aes_key *key, u8 out[at_least AES_BLOCK_SIZE],\n+\t\t const u8 in[at_least AES_BLOCK_SIZE]);\n \n extern const u8 crypto_aes_sbox[];\n extern const u8 crypto_aes_inv_sbox[];\n+extern const u32 aes_enc_tab[256];\n+extern const u32 aes_dec_tab[256];\n \n void aescfb_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src,\n \t\t int len, const u8 iv[AES_BLOCK_SIZE]);\n void aescfb_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src,\n \t\t int len, const u8 iv[AES_BLOCK_SIZE]);\ndiff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig\nindex 781a42c5c572..4efad77daa24 100644\n--- a/lib/crypto/Kconfig\n+++ b/lib/crypto/Kconfig\n@@ -9,10 +9,14 @@ config CRYPTO_LIB_UTILS\n \ttristate\n \n config CRYPTO_LIB_AES\n \ttristate\n \n+config CRYPTO_LIB_AES_ARCH\n+\tbool\n+\tdepends on CRYPTO_LIB_AES && !UML && !KMSAN\n+\n config CRYPTO_LIB_AESCFB\n \ttristate\n \tselect CRYPTO_LIB_AES\n \tselect CRYPTO_LIB_UTILS\n \ndiff --git a/lib/crypto/Makefile b/lib/crypto/Makefile\nindex 45128eccedef..01193b3f47ba 100644\n--- a/lib/crypto/Makefile\n+++ b/lib/crypto/Makefile\n@@ -13,12 +13,19 @@ obj-$(CONFIG_KUNIT)\t\t\t\t+= tests/\n obj-$(CONFIG_CRYPTO_HASH_INFO)\t\t\t+= hash_info.o\n \n obj-$(CONFIG_CRYPTO_LIB_UTILS)\t\t\t+= libcryptoutils.o\n libcryptoutils-y\t\t\t\t:= memneq.o utils.o\n \n-obj-$(CONFIG_CRYPTO_LIB_AES)\t\t\t+= libaes.o\n-libaes-y\t\t\t\t\t:= aes.o\n+################################################################################\n+\n+obj-$(CONFIG_CRYPTO_LIB_AES) += libaes.o\n+libaes-y := aes.o\n+ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y)\n+CFLAGS_aes.o += -I$(src)/$(SRCARCH)\n+endif # CONFIG_CRYPTO_LIB_AES_ARCH\n+\n+################################################################################\n \n obj-$(CONFIG_CRYPTO_LIB_AESCFB)\t\t\t+= libaescfb.o\n libaescfb-y\t\t\t\t\t:= aescfb.o\n \n obj-$(CONFIG_CRYPTO_LIB_AESGCM)\t\t\t+= libaesgcm.o\ndiff --git a/lib/crypto/aes.c b/lib/crypto/aes.c\nindex 102aaa76bc8d..88da68dcf5a8 100644\n--- a/lib/crypto/aes.c\n+++ b/lib/crypto/aes.c\n@@ -1,11 +1,13 @@\n // SPDX-License-Identifier: GPL-2.0\n /*\n * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org>\n+ * Copyright 2026 Google LLC\n */\n \n #include <crypto/aes.h>\n+#include <linux/cache.h>\n #include <linux/crypto.h>\n #include <linux/export.h>\n #include <linux/module.h>\n #include <linux/unaligned.h>\n \n@@ -87,10 +89,114 @@ extern const u8 crypto_aes_sbox[256] __alias(aes_sbox);\n extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox);\n \n EXPORT_SYMBOL(crypto_aes_sbox);\n EXPORT_SYMBOL(crypto_aes_inv_sbox);\n \n+/* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */\n+const u32 ____cacheline_aligned aes_enc_tab[256] = {\n+\t0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6,\n+\t0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56,\n+\t0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f,\n+\t0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb,\n+\t0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453,\n+\t0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c,\n+\t0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551,\n+\t0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a,\n+\t0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637,\n+\t0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df,\n+\t0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d,\n+\t0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b,\n+\t0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd,\n+\t0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1,\n+\t0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d,\n+\t0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85,\n+\t0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a,\n+\t0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe,\n+\t0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d,\n+\t0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1,\n+\t0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5,\n+\t0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3,\n+\t0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755,\n+\t0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6,\n+\t0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54,\n+\t0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428,\n+\t0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264,\n+\t0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8,\n+\t0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531,\n+\t0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda,\n+\t0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac,\n+\t0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810,\n+\t0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657,\n+\t0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e,\n+\t0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c,\n+\t0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c,\n+\t0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199,\n+\t0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122,\n+\t0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c,\n+\t0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5,\n+\t0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7,\n+\t0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e,\n+\t0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c,\n+};\n+EXPORT_SYMBOL(aes_enc_tab);\n+\n+/* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */\n+const u32 ____cacheline_aligned aes_dec_tab[256] = {\n+\t0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f,\n+\t0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5,\n+\t0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25,\n+\t0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b,\n+\t0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458,\n+\t0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927,\n+\t0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5,\n+\t0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9,\n+\t0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72,\n+\t0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3,\n+\t0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7,\n+\t0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4,\n+\t0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040,\n+\t0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d,\n+\t0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6,\n+\t0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879,\n+\t0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32,\n+\t0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36,\n+\t0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793,\n+\t0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c,\n+\t0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2,\n+\t0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3,\n+\t0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb,\n+\t0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684,\n+\t0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc,\n+\t0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947,\n+\t0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9,\n+\t0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f,\n+\t0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890,\n+\t0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf,\n+\t0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e,\n+\t0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef,\n+\t0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a,\n+\t0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733,\n+\t0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43,\n+\t0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546,\n+\t0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92,\n+\t0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb,\n+\t0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255,\n+\t0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478,\n+\t0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc,\n+\t0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664,\n+\t0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0,\n+};\n+EXPORT_SYMBOL(aes_dec_tab);\n+\n+/* Prefetch data into L1 cache. @mem should be cacheline-aligned. */\n+static __always_inline void aes_prefetch(const void *mem, size_t len)\n+{\n+\tfor (size_t i = 0; i < len; i += L1_CACHE_BYTES)\n+\t\t*(volatile const u8 *)(mem + i);\n+\tbarrier();\n+}\n+\n static u32 mul_by_x(u32 w)\n {\n \tu32 x = w & 0x7f7f7f7f;\n \tu32 y = w & 0x80808080;\n \n@@ -167,42 +273,21 @@ static u32 subw(u32 in)\n \t (aes_sbox[(in >> 8) & 0xff] << 8) ^\n \t (aes_sbox[(in >> 16) & 0xff] << 16) ^\n \t (aes_sbox[(in >> 24) & 0xff] << 24);\n }\n \n-/**\n- * aes_expandkey - Expands the AES key as described in FIPS-197\n- * @ctx:\tThe location where the computed key will be stored.\n- * @in_key:\tThe supplied key.\n- * @key_len:\tThe length of the supplied key.\n- *\n- * Returns 0 on success. The function fails only if an invalid key size (or\n- * pointer) is supplied.\n- * The expanded key size is 240 bytes (max of 14 rounds with a unique 16 bytes\n- * key schedule plus a 16 bytes key which is used before the first round).\n- * The decryption key is prepared for the \"Equivalent Inverse Cipher\" as\n- * described in FIPS-197. The first slot (16 bytes) of each key (enc or dec) is\n- * for the initial combination, the second slot for the first round and so on.\n- */\n-int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,\n-\t\t unsigned int key_len)\n+static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys,\n+\t\t\t\t const u8 *in_key, int key_len)\n {\n \tu32 kwords = key_len / sizeof(u32);\n \tu32 rc, i, j;\n-\tint err;\n-\n-\terr = aes_check_keylen(key_len);\n-\tif (err)\n-\t\treturn err;\n-\n-\tctx->key_length = key_len;\n \n \tfor (i = 0; i < kwords; i++)\n-\t\tctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));\n+\t\trndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]);\n \n \tfor (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) {\n-\t\tu32 *rki = ctx->key_enc + (i * kwords);\n+\t\tu32 *rki = &rndkeys[i * kwords];\n \t\tu32 *rko = rki + kwords;\n \n \t\trko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0];\n \t\trko[1] = rko[0] ^ rki[1];\n \t\trko[2] = rko[1] ^ rki[2];\n@@ -227,38 +312,42 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,\n \t * Generate the decryption keys for the Equivalent Inverse Cipher.\n \t * This involves reversing the order of the round keys, and applying\n \t * the Inverse Mix Columns transformation to all but the first and\n \t * the last one.\n \t */\n-\tctx->key_dec[0] = ctx->key_enc[key_len + 24];\n-\tctx->key_dec[1] = ctx->key_enc[key_len + 25];\n-\tctx->key_dec[2] = ctx->key_enc[key_len + 26];\n-\tctx->key_dec[3] = ctx->key_enc[key_len + 27];\n-\n-\tfor (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {\n-\t\tctx->key_dec[i] = inv_mix_columns(ctx->key_enc[j]);\n-\t\tctx->key_dec[i + 1] = inv_mix_columns(ctx->key_enc[j + 1]);\n-\t\tctx->key_dec[i + 2] = inv_mix_columns(ctx->key_enc[j + 2]);\n-\t\tctx->key_dec[i + 3] = inv_mix_columns(ctx->key_enc[j + 3]);\n-\t}\n+\tif (inv_rndkeys) {\n+\t\tinv_rndkeys[0] = rndkeys[key_len + 24];\n+\t\tinv_rndkeys[1] = rndkeys[key_len + 25];\n+\t\tinv_rndkeys[2] = rndkeys[key_len + 26];\n+\t\tinv_rndkeys[3] = rndkeys[key_len + 27];\n+\n+\t\tfor (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {\n+\t\t\tinv_rndkeys[i] = inv_mix_columns(rndkeys[j]);\n+\t\t\tinv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]);\n+\t\t\tinv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]);\n+\t\t\tinv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]);\n+\t\t}\n \n-\tctx->key_dec[i] = ctx->key_enc[0];\n-\tctx->key_dec[i + 1] = ctx->key_enc[1];\n-\tctx->key_dec[i + 2] = ctx->key_enc[2];\n-\tctx->key_dec[i + 3] = ctx->key_enc[3];\n+\t\tinv_rndkeys[i] = rndkeys[0];\n+\t\tinv_rndkeys[i + 1] = rndkeys[1];\n+\t\tinv_rndkeys[i + 2] = rndkeys[2];\n+\t\tinv_rndkeys[i + 3] = rndkeys[3];\n+\t}\n+}\n \n+int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,\n+\t\t unsigned int key_len)\n+{\n+\tif (aes_check_keylen(key_len) != 0)\n+\t\treturn -EINVAL;\n+\tctx->key_length = key_len;\n+\taes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len);\n \treturn 0;\n }\n EXPORT_SYMBOL(aes_expandkey);\n \n-/**\n- * aes_encrypt - Encrypt a single AES block\n- * @ctx:\tContext struct containing the key schedule\n- * @out:\tBuffer to store the ciphertext\n- * @in:\t\tBuffer containing the plaintext\n- */\n-void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)\n+void aes_encrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)\n {\n \tconst u32 *rkp = ctx->key_enc + 4;\n \tint rounds = 6 + ctx->key_length / 4;\n \tu32 st0[4], st1[4];\n \tint round;\n@@ -297,19 +386,121 @@ void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)\n \tput_unaligned_le32(subshift(st1, 0) ^ rkp[4], out);\n \tput_unaligned_le32(subshift(st1, 1) ^ rkp[5], out + 4);\n \tput_unaligned_le32(subshift(st1, 2) ^ rkp[6], out + 8);\n \tput_unaligned_le32(subshift(st1, 3) ^ rkp[7], out + 12);\n }\n-EXPORT_SYMBOL(aes_encrypt);\n+EXPORT_SYMBOL(aes_encrypt_old);\n \n-/**\n- * aes_decrypt - Decrypt a single AES block\n- * @ctx:\tContext struct containing the key schedule\n- * @out:\tBuffer to store the plaintext\n- * @in:\t\tBuffer containing the ciphertext\n- */\n-void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)\n+static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk)\n+{\n+\treturn rk ^ aes_enc_tab[(u8)w[i]] ^\n+\t rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^\n+\t rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^\n+\t rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24);\n+}\n+\n+static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk)\n+{\n+\treturn rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^\n+\t (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^\n+\t ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^\n+\t ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16);\n+}\n+\n+static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds,\n+\t\t\t\t\t u8 out[AES_BLOCK_SIZE],\n+\t\t\t\t\t const u8 in[AES_BLOCK_SIZE])\n+{\n+\tconst u32 *rkp = rndkeys;\n+\tint n = nrounds - 1;\n+\tu32 w[4];\n+\n+\tw[0] = get_unaligned_le32(&in[0]) ^ *rkp++;\n+\tw[1] = get_unaligned_le32(&in[4]) ^ *rkp++;\n+\tw[2] = get_unaligned_le32(&in[8]) ^ *rkp++;\n+\tw[3] = get_unaligned_le32(&in[12]) ^ *rkp++;\n+\n+\t/*\n+\t * Prefetch the table before doing data and key-dependent loads from it.\n+\t *\n+\t * This is intended only as a basic constant-time hardening measure that\n+\t * avoids interfering with performance too much. Its effectiveness is\n+\t * not guaranteed. For proper constant-time AES, a CPU that supports\n+\t * AES instructions should be used instead.\n+\t */\n+\taes_prefetch(aes_enc_tab, sizeof(aes_enc_tab));\n+\n+\tdo {\n+\t\tu32 w0 = enc_quarterround(w, 0, *rkp++);\n+\t\tu32 w1 = enc_quarterround(w, 1, *rkp++);\n+\t\tu32 w2 = enc_quarterround(w, 2, *rkp++);\n+\t\tu32 w3 = enc_quarterround(w, 3, *rkp++);\n+\n+\t\tw[0] = w0;\n+\t\tw[1] = w1;\n+\t\tw[2] = w2;\n+\t\tw[3] = w3;\n+\t} while (--n);\n+\n+\tput_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]);\n+\tput_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]);\n+\tput_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]);\n+\tput_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]);\n+}\n+\n+static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk)\n+{\n+\treturn rk ^ aes_dec_tab[(u8)w[i]] ^\n+\t rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^\n+\t rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^\n+\t rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24);\n+}\n+\n+static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk)\n+{\n+\treturn rk ^ aes_inv_sbox[(u8)w[i]] ^\n+\t ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^\n+\t ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^\n+\t ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24);\n+}\n+\n+static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[],\n+\t\t\t\t\t int nrounds,\n+\t\t\t\t\t u8 out[AES_BLOCK_SIZE],\n+\t\t\t\t\t const u8 in[AES_BLOCK_SIZE])\n+{\n+\tconst u32 *rkp = inv_rndkeys;\n+\tint n = nrounds - 1;\n+\tu32 w[4];\n+\n+\tw[0] = get_unaligned_le32(&in[0]) ^ *rkp++;\n+\tw[1] = get_unaligned_le32(&in[4]) ^ *rkp++;\n+\tw[2] = get_unaligned_le32(&in[8]) ^ *rkp++;\n+\tw[3] = get_unaligned_le32(&in[12]) ^ *rkp++;\n+\n+\taes_prefetch(aes_dec_tab, sizeof(aes_dec_tab));\n+\n+\tdo {\n+\t\tu32 w0 = dec_quarterround(w, 0, *rkp++);\n+\t\tu32 w1 = dec_quarterround(w, 1, *rkp++);\n+\t\tu32 w2 = dec_quarterround(w, 2, *rkp++);\n+\t\tu32 w3 = dec_quarterround(w, 3, *rkp++);\n+\n+\t\tw[0] = w0;\n+\t\tw[1] = w1;\n+\t\tw[2] = w2;\n+\t\tw[3] = w3;\n+\t} while (--n);\n+\n+\taes_prefetch((const void *)aes_inv_sbox, sizeof(aes_inv_sbox));\n+\tput_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]);\n+\tput_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]);\n+\tput_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]);\n+\tput_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]);\n+}\n+\n+void aes_decrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)\n {\n \tconst u32 *rkp = ctx->key_dec + 4;\n \tint rounds = 6 + ctx->key_length / 4;\n \tu32 st0[4], st1[4];\n \tint round;\n@@ -348,10 +539,104 @@ void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)\n \tput_unaligned_le32(inv_subshift(st1, 0) ^ rkp[4], out);\n \tput_unaligned_le32(inv_subshift(st1, 1) ^ rkp[5], out + 4);\n \tput_unaligned_le32(inv_subshift(st1, 2) ^ rkp[6], out + 8);\n \tput_unaligned_le32(inv_subshift(st1, 3) ^ rkp[7], out + 12);\n }\n-EXPORT_SYMBOL(aes_decrypt);\n+EXPORT_SYMBOL(aes_decrypt_old);\n+\n+/*\n+ * Note: the aes_prepare*key_* names reflect the fact that the implementation\n+ * might not actually expand the key. (The s390 code for example doesn't.)\n+ * Where the key is expanded we use the more specific names aes_expandkey_*.\n+ *\n+ * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to\n+ * the area to store the prepared decryption key. It will be NULL if the user\n+ * is requesting encryption-only. aes_preparekey_arch() is also passed a valid\n+ * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256.\n+ */\n+#ifdef CONFIG_CRYPTO_LIB_AES_ARCH\n+/* An arch-specific implementation of AES is available. Include it. */\n+#include \"aes.h\" /* $(SRCARCH)/aes.h */\n+#else\n+/* No arch-specific implementation of AES is available. Use generic code. */\n+\n+static void aes_preparekey_arch(union aes_enckey_arch *k,\n+\t\t\t\tunion aes_invkey_arch *inv_k,\n+\t\t\t\tconst u8 *in_key, int key_len, int nrounds)\n+{\n+\taes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,\n+\t\t\t in_key, key_len);\n+}\n+\n+static void aes_encrypt_arch(const struct aes_enckey *key,\n+\t\t\t u8 out[AES_BLOCK_SIZE],\n+\t\t\t const u8 in[AES_BLOCK_SIZE])\n+{\n+\taes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);\n+}\n+\n+static void aes_decrypt_arch(const struct aes_key *key,\n+\t\t\t u8 out[AES_BLOCK_SIZE],\n+\t\t\t const u8 in[AES_BLOCK_SIZE])\n+{\n+\taes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in);\n+}\n+#endif\n+\n+static int __aes_preparekey(struct aes_enckey *enc_key,\n+\t\t\t union aes_invkey_arch *inv_k,\n+\t\t\t const u8 *in_key, size_t key_len)\n+{\n+\tif (aes_check_keylen(key_len) != 0)\n+\t\treturn -EINVAL;\n+\tenc_key->len = key_len;\n+\tenc_key->nrounds = 6 + key_len / 4;\n+\taes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len,\n+\t\t\t enc_key->nrounds);\n+\treturn 0;\n+}\n+\n+int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len)\n+{\n+\treturn __aes_preparekey((struct aes_enckey *)key, &key->inv_k,\n+\t\t\t\tin_key, key_len);\n+}\n+EXPORT_SYMBOL(aes_preparekey);\n+\n+int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len)\n+{\n+\treturn __aes_preparekey(key, NULL, in_key, key_len);\n+}\n+EXPORT_SYMBOL(aes_prepareenckey);\n+\n+void aes_encrypt_new(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE],\n+\t\t const u8 in[AES_BLOCK_SIZE])\n+{\n+\taes_encrypt_arch(key.enc_key, out, in);\n+}\n+EXPORT_SYMBOL(aes_encrypt_new);\n+\n+void aes_decrypt_new(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],\n+\t\t const u8 in[AES_BLOCK_SIZE])\n+{\n+\taes_decrypt_arch(key, out, in);\n+}\n+EXPORT_SYMBOL(aes_decrypt_new);\n+\n+#ifdef aes_mod_init_arch\n+static int __init aes_mod_init(void)\n+{\n+\taes_mod_init_arch();\n+\treturn 0;\n+}\n+subsys_initcall(aes_mod_init);\n+\n+static void __exit aes_mod_exit(void)\n+{\n+}\n+module_exit(aes_mod_exit);\n+#endif\n \n-MODULE_DESCRIPTION(\"Generic AES library\");\n+MODULE_DESCRIPTION(\"AES block cipher\");\n MODULE_AUTHOR(\"Ard Biesheuvel <ard.biesheuvel@linaro.org>\");\n+MODULE_AUTHOR(\"Eric Biggers <ebiggers@kernel.org>\");\n MODULE_LICENSE(\"GPL v2\");\n", "prefixes": [ "v2", "02/35" ] }