lib: sbi: Allow platforms to override local TLB operations

Some T-Head based processors, for example the Sophgo SG2044, have a
JTLB errata that requires special handling of certain TLB maintenance
instructions (sfence.vma).

Introduce a sbi_tlb_local_operations structure so platforms can provide
custom local TLB flush implementations. Use it to implement the JTLB
workaround on affected SoCs (currently Sophgo SG2044).

Signed-off-by: Xiang W <wangxiang@iscas.ac.cn>
Signed-off-by: Han Gao <gaohan@iscas.ac.cn>
Reviewed-by: Anup Patel <anup@brainfault.org>
Link: https://lore.kernel.org/r/20260617112925.1144190-2-wangxiang@iscas.ac.cn
Signed-off-by: Anup Patel <anup@brainfault.org>

include/sbi/sbi_tlb.h

@@ -54,6 +54,37 @@ do { \
 
 #define SBI_TLB_INFO_SIZE		sizeof(struct sbi_tlb_info)
 
+/** Collection of Local TLB operations */
+struct sbi_tlb_local_operations {
+	/** Local fence.i */
+	void (*local_fence_i)(struct sbi_tlb_info *tinfo);
+
+	/** Local supervisor-mode TLB flush */
+	void (*local_sfence_vma)(struct sbi_tlb_info *tinfo);
+
+	/** Local supervisor-mode TLB flush with ASID */
+	void (*local_sfence_vma_asid)(struct sbi_tlb_info *tinfo);
+
+	/** Local G-stage TLB flush for a specific VMID */
+	void (*local_hfence_gvma_vmid)(struct sbi_tlb_info *tinfo);
+
+	/** Local G-stage TLB flush */
+	void (*local_hfence_gvma)(struct sbi_tlb_info *tinfo);
+
+	/** Local VS-stage TLB flush with ASID */
+	void (*local_hfence_vvma_asid)(struct sbi_tlb_info *tinfo);
+
+	/** Local VS-stage TLB flush */
+	void (*local_hfence_vvma)(struct sbi_tlb_info *tinfo);
+};
+
+/** Get the local TLB operations */
+const struct sbi_tlb_local_operations *sbi_tlb_get_local_operations(void);
+
+/** Set the local TLB operations */
+void sbi_tlb_set_local_operations(
+		const struct sbi_tlb_local_operations *ops);
+
 void __sbi_sfence_vma_all();
 
 int sbi_tlb_request(ulong hmask, ulong hbase, struct sbi_tlb_info *tinfo);

lib/sbi/sbi_tlb.c

@@ -29,6 +29,38 @@ static unsigned long tlb_fifo_off;
 static unsigned long tlb_fifo_mem_off;
 static unsigned long tlb_range_flush_limit;
 
+static void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo);
+static void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo);
+static void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo);
+static void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo);
+static void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo);
+static void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo);
+static void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo);
+
+static struct sbi_tlb_local_operations default_tlb_local_ops = {
+	.local_fence_i = sbi_tlb_local_fence_i,
+	.local_sfence_vma = sbi_tlb_local_sfence_vma,
+	.local_sfence_vma_asid = sbi_tlb_local_sfence_vma_asid,
+	.local_hfence_gvma_vmid = sbi_tlb_local_hfence_gvma_vmid,
+	.local_hfence_gvma = sbi_tlb_local_hfence_gvma,
+	.local_hfence_vvma_asid = sbi_tlb_local_hfence_vvma_asid,
+	.local_hfence_vvma = sbi_tlb_local_hfence_vvma
+};
+
+static const struct sbi_tlb_local_operations *tlb_local_ops =
+	&default_tlb_local_ops;
+
+const struct sbi_tlb_local_operations *sbi_tlb_get_local_operations(void)
+{
+	return tlb_local_ops;
+}
+
+void sbi_tlb_set_local_operations(
+		const struct sbi_tlb_local_operations *ops)
+{
+	tlb_local_ops = ops;
+}
+
 void __sbi_sfence_vma_all(void)
 {
 	__asm__ __volatile("sfence.vma");
@@ -183,25 +215,25 @@ static void tlb_entry_local_process(struct sbi_tlb_info *data)
 
 	switch (data->type) {
 	case SBI_TLB_FENCE_I:
-		sbi_tlb_local_fence_i(data);
+		tlb_local_ops->local_fence_i(data);
 		break;
 	case SBI_TLB_SFENCE_VMA:
-		sbi_tlb_local_sfence_vma(data);
+		tlb_local_ops->local_sfence_vma(data);
 		break;
 	case SBI_TLB_SFENCE_VMA_ASID:
-		sbi_tlb_local_sfence_vma_asid(data);
+		tlb_local_ops->local_sfence_vma_asid(data);
 		break;
 	case SBI_TLB_HFENCE_GVMA_VMID:
-		sbi_tlb_local_hfence_gvma_vmid(data);
+		tlb_local_ops->local_hfence_gvma_vmid(data);
 		break;
 	case SBI_TLB_HFENCE_GVMA:
-		sbi_tlb_local_hfence_gvma(data);
+		tlb_local_ops->local_hfence_gvma(data);
 		break;
 	case SBI_TLB_HFENCE_VVMA_ASID:
-		sbi_tlb_local_hfence_vvma_asid(data);
+		tlb_local_ops->local_hfence_vvma_asid(data);
 		break;
 	case SBI_TLB_HFENCE_VVMA:
-		sbi_tlb_local_hfence_vvma(data);
+		tlb_local_ops->local_hfence_vvma(data);
 		break;
 	default:
 		break;

platform/generic/include/thead/c9xx_errata.h

@@ -8,6 +8,7 @@
  */
 #define THEAD_QUIRK_ERRATA_TLB_FLUSH		BIT(0)
 #define THEAD_QUIRK_ERRATA_THEAD_PMU		BIT(1)
+#define THEAD_QUIRK_ERRATA_JTLB			BIT(2)
 
 void thead_register_tlb_flush_trap_handler(void);
 

platform/generic/thead/thead-generic.c

@@ -11,8 +11,10 @@
 #include <thead/c9xx_pmu.h>
 #include <sbi/sbi_const.h>
 #include <sbi/sbi_platform.h>
+#include <sbi/sbi_pmu.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
+#include <sbi/sbi_tlb.h>
 #include <sbi_utils/fdt/fdt_helper.h>
 
 struct thead_generic_quirks {
@@ -39,15 +41,38 @@ static int thead_pmu_extensions_init(bool cold_boot)
 	return 0;
 }
 
+static void thead_jtlb_local_sfence_vma(struct sbi_tlb_info *tinfo)
+{
+	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SFENCE_VMA_RCVD);
+	__asm__ __volatile__("sfence.vma");
+}
+
+static void thead_jtlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
+{
+	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SFENCE_VMA_ASID_RCVD);
+	/* Flush entire MM context for a given ASID */
+	__asm__ __volatile__("sfence.vma x0, %0"
+			     :
+			     : "r"(tinfo->asid)
+			     : "memory");
+}
+
 static int thead_generic_platform_init(const void *fdt, int nodeoff,
 				       const struct fdt_match *match)
 {
 	const struct thead_generic_quirks *quirks = match->data;
+	static struct sbi_tlb_local_operations tlb_local_ops;
 
 	if (quirks->errata & THEAD_QUIRK_ERRATA_TLB_FLUSH)
 		generic_platform_ops.early_init = thead_tlb_flush_early_init;
 	if (quirks->errata & THEAD_QUIRK_ERRATA_THEAD_PMU)
 		generic_platform_ops.extensions_init = thead_pmu_extensions_init;
+	if (quirks->errata &THEAD_QUIRK_ERRATA_JTLB) {
+		sbi_memcpy(&tlb_local_ops, sbi_tlb_get_local_operations(), sizeof(tlb_local_ops));
+		tlb_local_ops.local_sfence_vma = thead_jtlb_local_sfence_vma;
+		tlb_local_ops.local_sfence_vma_asid = thead_jtlb_local_sfence_vma_asid;
+		sbi_tlb_set_local_operations(&tlb_local_ops);
+	}
 
 	return 0;
 }
@@ -60,13 +85,17 @@ static const struct thead_generic_quirks thead_pmu_quirks = {
 	.errata = THEAD_QUIRK_ERRATA_THEAD_PMU,
 };
 
+static const struct thead_generic_quirks thead_pmu_jtlb_quirks = {
+	.errata = THEAD_QUIRK_ERRATA_THEAD_PMU | THEAD_QUIRK_ERRATA_JTLB,
+};
+
 static const struct fdt_match thead_generic_match[] = {
 	{ .compatible = "canaan,kendryte-k230", .data = &thead_pmu_quirks },
 	{ .compatible = "sophgo,cv1800b", .data = &thead_pmu_quirks },
 	{ .compatible = "sophgo,cv1812h", .data = &thead_pmu_quirks },
 	{ .compatible = "sophgo,sg2000", .data = &thead_pmu_quirks },
 	{ .compatible = "sophgo,sg2002", .data = &thead_pmu_quirks },
-	{ .compatible = "sophgo,sg2044", .data = &thead_pmu_quirks },
+	{ .compatible = "sophgo,sg2044", .data = &thead_pmu_jtlb_quirks },
 	{ .compatible = "thead,th1520", .data = &thead_th1520_quirks },
 	{ },
 };