lib: sbi_domain: Re-work boot-time assignment of HARTs to non-ROOT domains

Assign a non-ROOT domain to a HART on first come first serve basis if the
HART is listed as a possible HART of the non-ROOT domain. If no non-ROOT
domain list a HART as possible HART then the HART is assigned to the ROOT
domain.

This allows us to drop the OpenSBI specific DT property from each CPU DT
node (aka "opensbi-domain" Dt property).

Signed-off-by: Anup Patel <anup.patel@oss.qualcomm.com>
Reviewed-by: Pawandeep Oza <pawandeep.oza@oss.qualcomm.com>
Tested-by: Pawandeep Oza <pawandeep.oza@oss.qualcomm.com>
Reviewed-by: Yu-Chien Peter Lin <peter.lin@sifive.com>
Link: https://lore.kernel.org/r/20260905131748.922920-3-anup.patel@oss.qualcomm.com
Signed-off-by: Anup Patel <anup@brainfault.org>

docs/domain_support.md

@@ -183,23 +183,16 @@ The DT properties of a domain instance DT node are as follows:
   domain instance. If not specified, defaults to the coldboot HART. Note that
   if the coldboot HART is assigned to this domain, it will be forced as
   the boot HART regardless of this property.
-* **next-arg1** (Optional) - The 64 bit next booting stage arg1 for the
+* **next-arg1** (Optional) - The 64 bit next booting stage arg1
-  domain instance. If this DT property is not available and coldboot HART
+  for the domain instance. If this DT property is not available
-  is not assigned to the domain instance then **next booting stage arg1 of coldboot HART**
+  then **next booting stage arg1 of coldboot HART** is used as default value.
-  is used as default value.
+* **next-addr** (Optional) - The 64 bit next booting stage address
-* **next-addr** (Optional) - The 64 bit next booting stage address for the
+  for the domain instance. If this DT property is not available
-  domain instance. If this DT property is not available and coldboot HART
+  then **next booting stage address of coldboot HART** is used as default value.
-  is not assigned to the domain instance then **0x0** is used as default
-  value. If this DT property is not available and coldboot HART is assigned
-  to the domain instance then **next booting stage address of coldboot HART**
-  is used as default value.
 * **next-mode** (Optional) - The 32 bit next booting stage mode for the
   domain instance. The possible values of this DT property are: **0x1**
   (S-mode), and **0x0** (U-mode). If this DT property is not available
-  and coldboot HART is not assigned to the domain instance then **0x1**
+  then **next booting stage mode of coldboot HART** is used as default value.
-  is used as default value. If this DT property is not available and
-  coldboot HART is assigned to the domain instance then **next booting
-  stage mode of coldboot HART** is used as default value.
 * **system-reset-allowed** (Optional) - A boolean flag representing
   whether the domain instance is allowed to do system reset.
 * **system-suspend-allowed** (Optional) - A boolean flag representing
@@ -207,15 +200,14 @@ The DT properties of a domain instance DT node are as follows:
 
 ### Assigning HART To Domain Instance
 
-By default, all HARTs are assigned to **the ROOT domain**. The OpenSBI
+At boot-time, a HART is assigned to a non-ROOT domain on first come
-platform support can provide the HART to domain instance assignment using
+first serve basis if the HART is listed as a possible HART of the
-platform specific callback.
+non-ROOT domain. If no non-ROOT domain list a HART as possible HART
+then the HART is assigned to **the ROOT domain**.
 
-The HART to domain instance assignment can be parsed from the device tree
+At runtime, the assignment of a HART can change from one domain to
-using optional DT property **opensbi-domain** in each CPU DT node. The
+another domain as long as the HART is listed in possible HARTs of
-value of DT property **opensbi-domain** is the DT phandle of the domain
+both domains.
-instance DT node. If **opensbi-domain** DT property is not specified then
-corresponding HART is assigned to **the ROOT domain**.
 
 ### Domain Configuration Only Accessible to OpenSBI
 
@@ -289,7 +281,6 @@ be done:
             device_type = "cpu";
             reg = <0x00>;
             compatible = "riscv";
-            opensbi-domain = <&tdomain>;
             ...
         };
 
@@ -297,7 +288,6 @@ be done:
             device_type = "cpu";
             reg = <0x01>;
             compatible = "riscv";
-            opensbi-domain = <&udomain>;
             ...
         };
 
@@ -305,7 +295,6 @@ be done:
             device_type = "cpu";
             reg = <0x02>;
             compatible = "riscv";
-            opensbi-domain = <&udomain>;
             ...
         };
 
@@ -313,7 +302,6 @@ be done:
             device_type = "cpu";
             reg = <0x03>;
             compatible = "riscv";
-            opensbi-domain = <&udomain>;
             ...
         };
 
@@ -321,7 +309,6 @@ be done:
             device_type = "cpu";
             reg = <0x04>;
             compatible = "riscv";
-            opensbi-domain = <&udomain>;
             ...
         };
     };

include/sbi/sbi_domain.h

@@ -328,12 +328,10 @@ void sbi_domain_dump_all(const char *suffix);
 /**
  * Register a new domain
  * @param dom pointer to domain
- * @param assign_mask pointer to HART mask of HARTs assigned to the domain
  *
  * @return 0 on success and negative error code on failure
  */
-int sbi_domain_register(struct sbi_domain *dom,
+int sbi_domain_register(struct sbi_domain *dom);
-			const struct sbi_hartmask *assign_mask);
 
 /**
  * Add a memory range with its flags to the root domain

include/sbi_utils/fdt/fdt_domain.h

@@ -50,8 +50,7 @@ int fdt_iterate_each_memregion(void *fdt, int domain_offset, void *opaque,
  *
  * This routine:
  * 1. Disables MMIO devices not accessible to the coldboot HART domain
- * 2. Removes "opensbi-domain" DT property from CPU DT nodes
+ * 2. Removes domain configuration DT node under /chosen DT node
- * 3. Removes domain configuration DT node under /chosen DT node
  *
  * It is recommended that platform support call this function in
  * their final_init() platform operation.

lib/sbi/sbi_domain.c

@@ -625,16 +625,14 @@ void sbi_domain_dump_all(const char *suffix)
 	}
 }
 
-int sbi_domain_register(struct sbi_domain *dom,
+int sbi_domain_register(struct sbi_domain *dom)
-			const struct sbi_hartmask *assign_mask)
 {
-	u32 i;
+	u32 i, cold_hartid = current_hartid();
-	int rc;
 	struct sbi_domain *tdom;
-	u32 cold_hartid = current_hartid();
+	int rc;
 
 	/* Sanity checks */
-	if (!dom || !assign_mask || domain_finalized)
+	if (!dom || domain_finalized)
 		return SBI_EINVAL;
 
 	/* Check if domain already discovered */
@@ -663,15 +661,21 @@ int sbi_domain_register(struct sbi_domain *dom,
 	/* Clear assigned HARTs of domain */
 	sbi_hartmask_clear_all(&dom->assigned_harts);
 
-	/* Assign domain to HART if HART is a possible HART */
+	/*
-	sbi_hartmask_for_each_hartindex(i, assign_mask) {
+	 * Assign a non-ROOT domain to a HART on first come first serve
-		if (!sbi_hartmask_test_hartindex(i, dom->possible_harts))
+	 * basis if the HART is listed as a possible HART of the non-ROOT
-			continue;
+	 * domain. If no non-ROOT domain list a HART as possible HART then
-
+	 * the HART is assigned to the ROOT domain.
+	 */
+	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
 		tdom = sbi_hartindex_to_domain(i);
-		if (tdom)
+		if (tdom) {
-			sbi_hartmask_clear_hartindex(i,
+			if (tdom == &root)
-					&tdom->assigned_harts);
+				sbi_hartmask_clear_hartindex(i, &tdom->assigned_harts);
+			else
+				continue;
+		}
+
 		sbi_update_hartindex_to_domain(i, dom);
 		sbi_hartmask_set_hartindex(i, &dom->assigned_harts);
 
@@ -975,7 +979,7 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid)
 		sbi_hartmask_set_hartindex(i, root_hmask);
 
 	/* Finally register the root domain */
-	rc = sbi_domain_register(&root, root_hmask);
+	rc = sbi_domain_register(&root);
 	if (rc)
 		goto fail_free_root_hmask;
 

lib/utils/fdt/fdt_domain.c

@@ -170,26 +170,11 @@ static int __fixup_disable_devices(void *fdt, int doff, int roff,
 
 void fdt_domain_fixup(void *fdt)
 {
-	u32 i, dcount;
+	u32 dcount;
 	int err, poffset, doffset;
 	struct sbi_domain *dom = sbi_domain_thishart_ptr();
 	struct __fixup_find_domain_offset_info fdo;
 
-	/* Remove the domain assignment DT property from CPU DT nodes */
-	poffset = fdt_path_offset(fdt, "/cpus");
-	if (poffset < 0)
-		return;
-	fdt_for_each_subnode(doffset, fdt, poffset) {
-		err = fdt_parse_hart_id(fdt, doffset, &i);
-		if (err)
-			continue;
-
-		if (!fdt_node_is_enabled(fdt, doffset))
-			continue;
-
-		fdt_nop_property(fdt, doffset, "opensbi-domain");
-	}
-
 	/* Skip device disable for root domain */
 	if (!dom->index)
 		goto skip_device_disable;
@@ -315,12 +300,10 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque)
 	const char *inherit;
 	struct sbi_domain *dom;
 	struct sbi_hartmask *mask;
-	struct sbi_hartmask assign_mask;
 	struct parse_region_data preg;
-	int *cold_domain_offset = opaque;
 	struct sbi_domain_memregion *reg;
 	int inheritance_mode = FDT_ROOT_REGION_INHERIT_M_ONLY;
-	int i, err = 0, len, cpus_offset, cpu_offset, doffset;
+	int i, err = 0, len, cpu_offset;
 
 	dom = sbi_zalloc(sizeof(*dom));
 	if (!dom)
@@ -440,7 +423,7 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque)
 		val64 = fdt32_to_cpu(val[0]);
 		val64 = (val64 << 32) | fdt32_to_cpu(val[1]);
 	} else {
-		val64 = sbi_scratch_thishart_ptr()->next_arg1;
+		val64 = root.next_arg1;
 	}
 	dom->next_arg1 = val64;
 
@@ -451,8 +434,7 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque)
 		val64 = fdt32_to_cpu(val[0]);
 		val64 = (val64 << 32) | fdt32_to_cpu(val[1]);
 	} else {
-		if (domain_offset == *cold_domain_offset)
+		val64 = root.next_addr;
-			val64 = sbi_scratch_thishart_ptr()->next_addr;
 	}
 	dom->next_addr = val64;
 
@@ -464,8 +446,7 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque)
 		if (val32 != 0x0 && val32 != 0x1)
 			val32 = 0x1;
 	} else {
-		if (domain_offset == *cold_domain_offset)
+		val32 = root.next_mode;
-			val32 = sbi_scratch_thishart_ptr()->next_mode;
 	}
 	dom->next_mode = val32;
 
@@ -483,44 +464,8 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque)
 	else
 		dom->system_suspend_allowed = false;
 
-	/* Find /cpus DT node */
-	cpus_offset = fdt_path_offset(fdt, "/cpus");
-	if (cpus_offset < 0) {
-		err = cpus_offset;
-		goto fail_free_all;
-	}
-
-	/* HART to domain assignment mask based on CPU DT nodes */
-	sbi_hartmask_clear_all(&assign_mask);
-	fdt_for_each_subnode(cpu_offset, fdt, cpus_offset) {
-		err = fdt_parse_hart_id(fdt, cpu_offset, &val32);
-		if (err)
-			continue;
-
-		if (SBI_HARTMASK_MAX_BITS <= sbi_hartid_to_hartindex(val32))
-			continue;
-
-		if (!fdt_node_is_enabled(fdt, cpu_offset))
-			continue;
-
-		/* This is an optional property */
-		val = fdt_getprop(fdt, cpu_offset, "opensbi-domain", &len);
-		if (!val || len < 4)
-			continue;
-
-		/* However, it should be valid if specified */
-		doffset = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(*val));
-		if (doffset < 0) {
-			err = doffset;
-			goto fail_free_all;
-		}
-
-		if (doffset == domain_offset)
-			sbi_hartmask_set_hartid(val32, &assign_mask);
-	}
-
 	/* Register the domain */
-	err = sbi_domain_register(dom, &assign_mask);
+	err = sbi_domain_register(dom);
 	if (err)
 		goto fail_free_all;
 
@@ -537,43 +482,10 @@ fail_free_domain:
 
 int fdt_domains_populate(const void *fdt)
 {
-	const u32 *val;
-	int cold_domain_offset;
-	u32 hartid, cold_hartid;
-	int err, len, cpus_offset, cpu_offset;
-
 	/* Sanity checks */
 	if (!fdt)
 		return SBI_EINVAL;
 
-	/* Find /cpus DT node */
-	cpus_offset = fdt_path_offset(fdt, "/cpus");
-	if (cpus_offset < 0)
-		return cpus_offset;
-
-	/* Find coldboot HART domain DT node offset */
-	cold_domain_offset = -1;
-	cold_hartid = current_hartid();
-	fdt_for_each_subnode(cpu_offset, fdt, cpus_offset) {
-		err = fdt_parse_hart_id(fdt, cpu_offset, &hartid);
-		if (err)
-			continue;
-
-		if (hartid != cold_hartid)
-			continue;
-
-		if (!fdt_node_is_enabled(fdt, cpu_offset))
-			continue;
-
-		val = fdt_getprop(fdt, cpu_offset, "opensbi-domain", &len);
-		if (val && len >= 4)
-			cold_domain_offset = fdt_node_offset_by_phandle(fdt,
-							   fdt32_to_cpu(*val));
-
-		break;
-	}
-
 	/* Iterate over each domain in FDT and populate details */
-	return fdt_iterate_each_domain_ro(fdt, &cold_domain_offset,
+	return fdt_iterate_each_domain_ro(fdt, NULL, __fdt_parse_domain);
-					  __fdt_parse_domain);
 }