Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/808360/?format=api
{ "id": 808360, "url": "http://patchwork.ozlabs.org/api/patches/808360/?format=api", "web_url": "http://patchwork.ozlabs.org/project/glibc/patch/1504198860-12951-17-git-send-email-Dave.Martin@arm.com/", "project": { "id": 41, "url": "http://patchwork.ozlabs.org/api/projects/41/?format=api", "name": "GNU C Library", "link_name": "glibc", "list_id": "libc-alpha.sourceware.org", "list_email": "libc-alpha@sourceware.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<1504198860-12951-17-git-send-email-Dave.Martin@arm.com>", "list_archive_url": null, "date": "2017-08-31T17:00:48", "name": "[v2,16/28] arm64/sve: Probe SVE capabilities and usable vector lengths", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "3af150600a86abddf829603e91534c04dc8f7194", "submitter": { "id": 26612, "url": "http://patchwork.ozlabs.org/api/people/26612/?format=api", "name": "Dave Martin", "email": "Dave.Martin@arm.com" }, "delegate": null, "mbox": "http://patchwork.ozlabs.org/project/glibc/patch/1504198860-12951-17-git-send-email-Dave.Martin@arm.com/mbox/", "series": [ { "id": 882, "url": "http://patchwork.ozlabs.org/api/series/882/?format=api", "web_url": "http://patchwork.ozlabs.org/project/glibc/list/?series=882", "date": "2017-08-31T17:00:32", "name": "ARM Scalable Vector Extension (SVE)", "version": 2, "mbox": "http://patchwork.ozlabs.org/series/882/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/808360/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/808360/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<libc-alpha-return-83985-incoming=patchwork.ozlabs.org@sourceware.org>", "X-Original-To": "incoming@patchwork.ozlabs.org", "Delivered-To": [ "patchwork-incoming@bilbo.ozlabs.org", "mailing list libc-alpha@sourceware.org" ], "Authentication-Results": [ "ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=sourceware.org\n\t(client-ip=209.132.180.131; helo=sourceware.org;\n\tenvelope-from=libc-alpha-return-83985-incoming=patchwork.ozlabs.org@sourceware.org;\n\treceiver=<UNKNOWN>)", "ozlabs.org; dkim=pass (1024-bit key;\n\tsecure) header.d=sourceware.org header.i=@sourceware.org\n\theader.b=\"Koa6dY6I\"; dkim-atps=neutral", "sourceware.org; auth=none" ], "Received": [ "from sourceware.org (server1.sourceware.org [209.132.180.131])\n\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256\n\tbits)) (No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 3xjpld1y6rz9s81\n\tfor <incoming@patchwork.ozlabs.org>;\n\tFri, 1 Sep 2017 03:10:05 +1000 (AEST)", "(qmail 122424 invoked by alias); 31 Aug 2017 17:09:18 -0000", "(qmail 122210 invoked by uid 89); 31 Aug 2017 17:09:17 -0000" ], "DomainKey-Signature": "a=rsa-sha1; c=nofws; d=sourceware.org; h=list-id\n\t:list-unsubscribe:list-subscribe:list-archive:list-post\n\t:list-help:sender:from:to:cc:subject:date:message-id:in-reply-to\n\t:references:mime-version:content-type:content-transfer-encoding;\n\tq=dns; s=default; b=YCzt2DKo+qfxwUAGLSjntCo7YSRp9yGQjsLjE5FXmh1\n\t9bJon+CJW01g5gptsZy7YCB7x5MLI4yknLsANhkOQbJvFNDK1iBG2f6WmBOAZOJE\n\tm2h01lKiVGl0dojZX+D5s0EP51Fa1xR/QOowczaC9OJzp620EILyE7Y3FjGrgF9M\n\t=", "DKIM-Signature": "v=1; a=rsa-sha1; c=relaxed; d=sourceware.org; h=list-id\n\t:list-unsubscribe:list-subscribe:list-archive:list-post\n\t:list-help:sender:from:to:cc:subject:date:message-id:in-reply-to\n\t:references:mime-version:content-type:content-transfer-encoding;\n\ts=default; bh=/h3U0LzqIX6uVQwqjit74kgqmdo=; b=Koa6dY6I3DhOofLai\n\tFcaZW1Z1SPZ+uYSWVE60ALnDcZogPhFZKbrry7n0s5P7AJ6LnOzdZF3F9rVl0scO\n\tfwe+6ylX1eT/760FaUAMhX/xi9ZIXNG7WM0I2Cw5dY2uUkGOYCfgdKjDRd0w4t/X\n\tytbSdWUAzKRKhMmamkMUQi6cCU=", "Mailing-List": "contact libc-alpha-help@sourceware.org; run by ezmlm", "Precedence": "bulk", "List-Id": "<libc-alpha.sourceware.org>", "List-Unsubscribe": "<mailto:libc-alpha-unsubscribe-incoming=patchwork.ozlabs.org@sourceware.org>", "List-Subscribe": "<mailto:libc-alpha-subscribe@sourceware.org>", "List-Archive": "<http://sourceware.org/ml/libc-alpha/>", "List-Post": "<mailto:libc-alpha@sourceware.org>", "List-Help": "<mailto:libc-alpha-help@sourceware.org>,\n\t<http://sourceware.org/ml/#faqs>", "Sender": "libc-alpha-owner@sourceware.org", "X-Virus-Found": "No", "X-Spam-SWARE-Status": "No, score=-26.9 required=5.0 tests=BAYES_00, GIT_PATCH_0,\n\tGIT_PATCH_1, GIT_PATCH_2, GIT_PATCH_3, RP_MATCHES_RCVD,\n\tSPF_PASS autolearn=ham version=3.3.2 spammy=", "X-HELO": "foss.arm.com", "From": "Dave Martin <Dave.Martin@arm.com>", "To": "linux-arm-kernel@lists.infradead.org", "Cc": "Catalin Marinas <catalin.marinas@arm.com>, Will Deacon\n\t<will.deacon@arm.com>, \tArd Biesheuvel <ard.biesheuvel@linaro.org>,\n\t=?utf-8?q?Alex_Benn=C3=A9?= =?utf-8?q?e?= <alex.bennee@linaro.org>,\n\tSzabolcs Nagy <szabolcs.nagy@arm.com>, Richard Sandiford\n\t<richard.sandiford@arm.com>, \tkvmarm@lists.cs.columbia.edu,\n\tlibc-alpha@sourceware.org, \tlinux-arch@vger.kernel.org,\n\tSuzuki K Poulose <Suzuki.Poulose@arm.com>", "Subject": "[PATCH v2 16/28] arm64/sve: Probe SVE capabilities and usable\n\tvector lengths", "Date": "Thu, 31 Aug 2017 18:00:48 +0100", "Message-Id": "<1504198860-12951-17-git-send-email-Dave.Martin@arm.com>", "In-Reply-To": "<1504198860-12951-1-git-send-email-Dave.Martin@arm.com>", "References": "<1504198860-12951-1-git-send-email-Dave.Martin@arm.com>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit" }, "content": "This patch uses the cpufeatures framework to determine common SVE\ncapabilities and vector lengths, and configures the runtime SVE\nsupport code appropriately.\n\nZCR_ELx is not really a feature register, but it is convenient to\nuse it as a template for recording the maximum vector length\nsupported by a CPU, using the LEN field. This field is similar to\na feature field in that it is a contiguous bitfield for which we\nwant to determine the minimum system-wide value. This patch adds\nZCR as a pseudo-register in cpuinfo/cpufeatures, with appropriate\ncustom code to populate it. Finding the minimum supported value of\nthe LEN field is left to the cpufeatures framework in the usual\nway.\n\nThe meaning of ID_AA64ZFR0_EL1 is not architecturally defined yet,\nso for now we just require it to be zero.\n\nNote that much of this code is dormant and SVE still won't be used\nyet, since system_supports_sve() remains hardwired to false.\n\nSigned-off-by: Dave Martin <Dave.Martin@arm.com>\nCc: Alex Bennée <alex.bennee@linaro.org>\nCc: Suzuki K Poulose <Suzuki.Poulose@arm.com>\n\n---\n\nChanges since v1\n----------------\n\nRequested by Alex Bennée:\n\n* Thin out BUG_ON()s:\nRedundant BUG_ON()s and ones that just check invariants are removed.\nImportant sanity-checks are migrated to WARN_ON()s, with some\nminimal best-effort patch-up code.\n\nOther changes related to Alex Bennée's comments:\n\n* Migrate away from magic numbers for converting VL to VQ.\n\nRequested by Suzuki Poulose:\n\n* Make sve_vq_map __ro_after_init.\n\nOther changes related to Suzuki Poulose's comments:\n\n* Rely on cpufeatures for not attempting to update the vq map after boot.\n---\n arch/arm64/include/asm/cpu.h | 4 ++\n arch/arm64/include/asm/cpufeature.h | 29 ++++++++++\n arch/arm64/include/asm/fpsimd.h | 10 ++++\n arch/arm64/kernel/cpufeature.c | 50 +++++++++++++++++\n arch/arm64/kernel/cpuinfo.c | 6 ++\n arch/arm64/kernel/fpsimd.c | 106 +++++++++++++++++++++++++++++++++++-\n 6 files changed, 202 insertions(+), 3 deletions(-)", "diff": "diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h\nindex 889226b..8839227 100644\n--- a/arch/arm64/include/asm/cpu.h\n+++ b/arch/arm64/include/asm/cpu.h\n@@ -41,6 +41,7 @@ struct cpuinfo_arm64 {\n \tu64\t\treg_id_aa64mmfr2;\n \tu64\t\treg_id_aa64pfr0;\n \tu64\t\treg_id_aa64pfr1;\n+\tu64\t\treg_id_aa64zfr0;\n \n \tu32\t\treg_id_dfr0;\n \tu32\t\treg_id_isar0;\n@@ -59,6 +60,9 @@ struct cpuinfo_arm64 {\n \tu32\t\treg_mvfr0;\n \tu32\t\treg_mvfr1;\n \tu32\t\treg_mvfr2;\n+\n+\t/* pseudo-ZCR for recording maximum ZCR_EL1 LEN value: */\n+\tu64\t\treg_zcr;\n };\n \n DECLARE_PER_CPU(struct cpuinfo_arm64, cpu_data);\ndiff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h\nindex 4ea3441..d98e7ba 100644\n--- a/arch/arm64/include/asm/cpufeature.h\n+++ b/arch/arm64/include/asm/cpufeature.h\n@@ -10,7 +10,9 @@\n #define __ASM_CPUFEATURE_H\n \n #include <asm/cpucaps.h>\n+#include <asm/fpsimd.h>\n #include <asm/hwcap.h>\n+#include <asm/sigcontext.h>\n #include <asm/sysreg.h>\n \n /*\n@@ -223,6 +225,13 @@ static inline bool id_aa64pfr0_32bit_el0(u64 pfr0)\n \treturn val == ID_AA64PFR0_EL0_32BIT_64BIT;\n }\n \n+static inline bool id_aa64pfr0_sve(u64 pfr0)\n+{\n+\tu32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_SVE_SHIFT);\n+\n+\treturn val > 0;\n+}\n+\n void __init setup_cpu_features(void);\n \n void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,\n@@ -267,6 +276,26 @@ static inline bool system_supports_sve(void)\n \treturn false;\n }\n \n+/*\n+ * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE\n+ * vector length.\n+ * Use only if SVE is present. This function clobbers the SVE vector length.\n+ */\n+static u64 __maybe_unused read_zcr_features(void)\n+{\n+\tu64 zcr;\n+\tunsigned int vq_max;\n+\n+\twrite_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);\n+\n+\tzcr = read_sysreg_s(SYS_ZCR_EL1);\n+\tzcr &= ~(u64)ZCR_ELx_LEN_MASK;\n+\tvq_max = sve_vq_from_vl(sve_get_vl());\n+\tzcr |= vq_max - 1;\n+\n+\treturn zcr;\n+}\n+\n #endif /* __ASSEMBLY__ */\n \n #endif\ndiff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h\nindex 32c8e19..6c22624 100644\n--- a/arch/arm64/include/asm/fpsimd.h\n+++ b/arch/arm64/include/asm/fpsimd.h\n@@ -92,12 +92,22 @@ extern void fpsimd_dup_sve(struct task_struct *dst,\n extern int sve_set_vector_length(struct task_struct *task,\n \t\t\t\t unsigned long vl, unsigned long flags);\n \n+extern void __init sve_init_vq_map(void);\n+extern void sve_update_vq_map(void);\n+extern int sve_verify_vq_map(void);\n+extern void __init sve_setup(void);\n+\n #else /* ! CONFIG_ARM64_SVE */\n \n static void __maybe_unused sve_alloc(struct task_struct *task) { }\n static void __maybe_unused fpsimd_release_thread(struct task_struct *task) { }\n static void __maybe_unused fpsimd_dup_sve(struct task_struct *dst,\n \t\t\t\t\t struct task_struct const *src) { }\n+static void __maybe_unused sve_init_vq_map(void) { }\n+static void __maybe_unused sve_update_vq_map(void) { }\n+static int __maybe_unused sve_verify_vq_map(void) { return 0; }\n+static void __maybe_unused sve_setup(void) { }\n+\n #endif /* ! CONFIG_ARM64_SVE */\n \n /* For use by EFI runtime services calls only */\ndiff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c\nindex 43ba8df..c30bb6b 100644\n--- a/arch/arm64/kernel/cpufeature.c\n+++ b/arch/arm64/kernel/cpufeature.c\n@@ -27,6 +27,7 @@\n #include <asm/cpu.h>\n #include <asm/cpufeature.h>\n #include <asm/cpu_ops.h>\n+#include <asm/fpsimd.h>\n #include <asm/mmu_context.h>\n #include <asm/processor.h>\n #include <asm/sysreg.h>\n@@ -283,6 +284,12 @@ static const struct arm64_ftr_bits ftr_id_dfr0[] = {\n \tARM64_FTR_END,\n };\n \n+static const struct arm64_ftr_bits ftr_zcr[] = {\n+\tARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,\n+\t\tZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0),\t/* LEN */\n+\tARM64_FTR_END,\n+};\n+\n /*\n * Common ftr bits for a 32bit register with all hidden, strict\n * attributes, with 4bit feature fields and a default safe value of\n@@ -349,6 +356,7 @@ static const struct __ftr_reg_entry {\n \t/* Op1 = 0, CRn = 0, CRm = 4 */\n \tARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),\n \tARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_raz),\n+\tARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),\n \n \t/* Op1 = 0, CRn = 0, CRm = 5 */\n \tARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),\n@@ -363,6 +371,9 @@ static const struct __ftr_reg_entry {\n \tARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),\n \tARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),\n \n+\t/* Op1 = 0, CRn = 1, CRm = 2 */\n+\tARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),\n+\n \t/* Op1 = 3, CRn = 0, CRm = 0 */\n \t{ SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },\n \tARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),\n@@ -500,6 +511,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)\n \tinit_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);\n \tinit_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);\n \tinit_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);\n+\tinit_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);\n \n \tif (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {\n \t\tinit_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);\n@@ -520,6 +532,10 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)\n \t\tinit_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);\n \t}\n \n+\tif (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {\n+\t\tinit_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);\n+\t\tsve_init_vq_map();\n+\t}\n }\n \n static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)\n@@ -623,6 +639,9 @@ void update_cpu_features(int cpu,\n \ttaint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,\n \t\t\t\t info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);\n \n+\ttaint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,\n+\t\t\t\t info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);\n+\n \t/*\n \t * If we have AArch32, we care about 32-bit features for compat.\n \t * If the system doesn't support AArch32, don't update them.\n@@ -670,6 +689,14 @@ void update_cpu_features(int cpu,\n \t\t\t\t\tinfo->reg_mvfr2, boot->reg_mvfr2);\n \t}\n \n+\tif (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {\n+\t\ttaint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,\n+\t\t\t\t\tinfo->reg_zcr, boot->reg_zcr);\n+\n+\t\tif (!sys_caps_initialised)\n+\t\t\tsve_update_vq_map();\n+\t}\n+\n \t/*\n \t * Mismatched CPU features are a recipe for disaster. Don't even\n \t * pretend to support them.\n@@ -1097,6 +1124,23 @@ verify_local_cpu_features(const struct arm64_cpu_capabilities *caps)\n \t}\n }\n \n+static void verify_sve_features(void)\n+{\n+\tu64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);\n+\tu64 zcr = read_zcr_features();\n+\n+\tunsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;\n+\tunsigned int len = zcr & ZCR_ELx_LEN_MASK;\n+\n+\tif (len < safe_len || sve_verify_vq_map()) {\n+\t\tpr_crit(\"CPU%d: SVE: required vector length(s) missing\\n\",\n+\t\t\tsmp_processor_id());\n+\t\tcpu_die_early();\n+\t}\n+\n+\t/* Add checks on other ZCR bits here if necessary */\n+}\n+\n /*\n * Run through the enabled system capabilities and enable() it on this CPU.\n * The capabilities were decided based on the available CPUs at the boot time.\n@@ -1110,8 +1154,12 @@ static void verify_local_cpu_capabilities(void)\n \tverify_local_cpu_errata_workarounds();\n \tverify_local_cpu_features(arm64_features);\n \tverify_local_elf_hwcaps(arm64_elf_hwcaps);\n+\n \tif (system_supports_32bit_el0())\n \t\tverify_local_elf_hwcaps(compat_elf_hwcaps);\n+\n+\tif (system_supports_sve())\n+\t\tverify_sve_features();\n }\n \n void check_local_cpu_capabilities(void)\n@@ -1189,6 +1237,8 @@ void __init setup_cpu_features(void)\n \tif (system_supports_32bit_el0())\n \t\tsetup_elf_hwcaps(compat_elf_hwcaps);\n \n+\tsve_setup();\n+\n \t/* Advertise that we have computed the system capabilities */\n \tset_sys_caps_initialised();\n \ndiff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c\nindex 3118859..be260e8 100644\n--- a/arch/arm64/kernel/cpuinfo.c\n+++ b/arch/arm64/kernel/cpuinfo.c\n@@ -19,6 +19,7 @@\n #include <asm/cpu.h>\n #include <asm/cputype.h>\n #include <asm/cpufeature.h>\n+#include <asm/fpsimd.h>\n \n #include <linux/bitops.h>\n #include <linux/bug.h>\n@@ -326,6 +327,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)\n \tinfo->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);\n \tinfo->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1);\n \tinfo->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1);\n+\tinfo->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1);\n \n \t/* Update the 32bit ID registers only if AArch32 is implemented */\n \tif (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {\n@@ -348,6 +350,10 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)\n \t\tinfo->reg_mvfr2 = read_cpuid(MVFR2_EL1);\n \t}\n \n+\tif (IS_ENABLED(CONFIG_ARM64_SVE) &&\n+\t id_aa64pfr0_sve(info->reg_id_aa64pfr0))\n+\t\tinfo->reg_zcr = read_zcr_features();\n+\n \tcpuinfo_detect_icache_policy(info);\n }\n \ndiff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c\nindex 713476e..cea05a7 100644\n--- a/arch/arm64/kernel/fpsimd.c\n+++ b/arch/arm64/kernel/fpsimd.c\n@@ -110,19 +110,19 @@\n static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state);\n \n /* Default VL for tasks that don't set it explicitly: */\n-static int sve_default_vl = SVE_VL_MIN;\n+static int sve_default_vl = -1;\n \n #ifdef CONFIG_ARM64_SVE\n \n /* Maximum supported vector length across all CPUs (initially poisoned) */\n int __ro_after_init sve_max_vl = -1;\n /* Set of available vector lengths, as vq_to_bit(vq): */\n-static DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);\n+static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);\n \n #else /* ! CONFIG_ARM64_SVE */\n \n /* Dummy declaration for code that will be optimised out: */\n-extern DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);\n+extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);\n \n #endif /* ! CONFIG_ARM64_SVE */\n \n@@ -387,6 +387,103 @@ int sve_set_vector_length(struct task_struct *task,\n \treturn 0;\n }\n \n+static unsigned long *sve_alloc_vq_map(void)\n+{\n+\treturn kzalloc(BITS_TO_LONGS(SVE_VQ_MAX) * sizeof(unsigned long),\n+\t\t GFP_KERNEL);\n+}\n+\n+static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))\n+{\n+\tunsigned int vq, vl;\n+\tunsigned long zcr;\n+\n+\tzcr = ZCR_ELx_LEN_MASK;\n+\tzcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;\n+\n+\tfor (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {\n+\t\twrite_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */\n+\t\tvl = sve_get_vl();\n+\t\tvq = sve_vq_from_vl(vl); /* skip intervening lengths */\n+\t\tset_bit(vq_to_bit(vq), map);\n+\t}\n+}\n+\n+void __init sve_init_vq_map(void)\n+{\n+\tsve_probe_vqs(sve_vq_map);\n+}\n+\n+/*\n+ * If we haven't committed to the set of supported VQs yet, filter out\n+ * those not supported by the current CPU.\n+ */\n+void sve_update_vq_map(void)\n+{\n+\tunsigned long *map;\n+\n+\tmap = sve_alloc_vq_map();\n+\tsve_probe_vqs(map);\n+\tbitmap_and(sve_vq_map, sve_vq_map, map, SVE_VQ_MAX);\n+\tkfree(map);\n+}\n+\n+/* Check whether the current CPU supports all VQs in the committed set */\n+int sve_verify_vq_map(void)\n+{\n+\tint ret = 0;\n+\tunsigned long *map = sve_alloc_vq_map();\n+\n+\tsve_probe_vqs(map);\n+\tbitmap_andnot(map, sve_vq_map, map, SVE_VQ_MAX);\n+\tif (!bitmap_empty(map, SVE_VQ_MAX)) {\n+\t\tpr_warn(\"SVE: cpu%d: Required vector length(s) missing\\n\",\n+\t\t\tsmp_processor_id());\n+\t\tret = -EINVAL;\n+\t}\n+\n+\tkfree(map);\n+\n+\treturn ret;\n+}\n+\n+void __init sve_setup(void)\n+{\n+\tu64 zcr;\n+\n+\tif (!system_supports_sve())\n+\t\treturn;\n+\n+\t/*\n+\t * The SVE architecture mandates support for 128-bit vectors,\n+\t * so sve_vq_map must have at least SVE_VQ_MIN set.\n+\t * If something went wrong, at least try to patch it up:\n+\t */\n+\tif (WARN_ON(!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)))\n+\t\tset_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);\n+\n+\tzcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);\n+\tsve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);\n+\n+\t/*\n+\t * Sanity-check that the max VL we determined through CPU features\n+\t * corresponds properly to sve_vq_map. If not, do our best:\n+\t */\n+\tif (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))\n+\t\tsve_max_vl = find_supported_vector_length(sve_max_vl);\n+\n+\t/*\n+\t * For the default VL, pick the maximum supported value <= 64.\n+\t * VL == 64 is guaranteed not to grow the signal frame.\n+\t */\n+\tsve_default_vl = find_supported_vector_length(64);\n+\n+\tpr_info(\"SVE: maximum available vector length %u bytes per vector\\n\",\n+\t\tsve_max_vl);\n+\tpr_info(\"SVE: default vector length %u bytes per vector\\n\",\n+\t\tsve_default_vl);\n+}\n+\n void fpsimd_release_thread(struct task_struct *dead_task)\n {\n \tsve_free(dead_task);\n@@ -502,6 +599,9 @@ void fpsimd_flush_thread(void)\n \t\t * This is where we ensure that all user tasks have a valid\n \t\t * vector length configured: no kernel task can become a user\n \t\t * task without an exec and hence a call to this function.\n+\t\t * By the time the first call to this function is made, all\n+\t\t * early hardware probing is complete, so sve_default_vl\n+\t\t * should be valid.\n \t\t * If a bug causes this to go wrong, we make some noise and\n \t\t * try to fudge thread.sve_vl to a safe value here.\n \t\t */\n", "prefixes": [ "v2", "16/28" ] }