lib: sbi_domain: Introduce domain intialization order

Currently, the domain initialization order is implied by the order
in which domains are populated by sbi_platform_domains_init() from
sbi_domain_finalize(). This is not documented anywhere and forces
unecessary ordering between domain DT nodes.

To address the above, introduce per-domain 32-bit integer to represent
intialization order (aka "init_order") where domain with a lower
initialization order will be booted first and two domains must not
have same initialization order. For DT based domain creation, new
"init-order" DT property can be used in domain DT node to specify
the initialization order. The ROOT domain is assumed to have lowest
initialization order (aka 0xffffffff).

Signed-off-by: Anup Patel <anup.patel@oss.qualcomm.com>
Reviewed-by: Pawandeep Oza <pawandeep.oza@oss.qualcomm.com>
Tested-by: Pawandeep Oza <pawandeep.oza@oss.qualcomm.com>
Reviewed-by: Yu-Chien Peter Lin <peter.lin@sifive.com>
Link: https://lore.kernel.org/r/20260905131748.922920-4-anup.patel@oss.qualcomm.com
Signed-off-by: Anup Patel <anup@brainfault.org>

docs/domain_support.md

@@ -39,6 +39,9 @@ has following details:
 
 * **index** - Logical index of this domain
 * **name** - Name of this domain
+* **init_order** - Initialization order of this domain. Domain with a
+  lower initialization order will be booted first and two domains must
+  not have same initialization order.
 * **assigned_harts** - HARTs assigned to this domain
 * **possible_harts** - HARTs possible in this domain
 * **regions** - Array of memory regions terminated by a memory region
@@ -75,6 +78,8 @@ following manner:
 
 * **index** - Logical index of the ROOT domain is always zero
 * **name** - Name of the ROOT domain is "root"
+* **init_order** - Initialization order of the ROOT domain is always
+  0xffffffff (aka maximum possible 32-bit value)
 * **assigned_harts** - At boot-time all valid HARTs of a RISC-V platform
   are assigned the ROOT domain which changes later based on OpenSBI
   platform support
@@ -156,6 +161,9 @@ The DT properties of a domain instance DT node are as follows:
 
 * **compatible** (Mandatory) - The compatible string of the domain instance.
   This DT property should have value *"opensbi,domain,instance"*
+* **init-order** (Optional) - The 32-bit initialization order for the domain
+  instance. If this DT property is not available then domain instance DT node
+  offset is used as default value.
 * **possible-harts** (Optional) - The list of CPU DT node phandles for the
   the domain instance. This list represents the possible HARTs of the
   domain instance.

include/sbi/sbi_domain.h

@@ -199,6 +199,8 @@ struct sbi_domain {
 	spinlock_t assigned_harts_lock;
 	/** Name of this domain */
 	char name[64];
+	/** Initialization order of this domain */
+	u32 init_order;
 	/** Possible HARTs in this domain */
 	const struct sbi_hartmask *possible_harts;
 	/** Array of memory regions terminated by a region with order zero */

lib/sbi/sbi_domain.c

@@ -28,6 +28,7 @@ static bool domain_finalized = false;
 
 struct sbi_domain root = {
 	.name = "root",
+	.init_order = -1U,
 	.possible_harts = NULL,
 	.regions = NULL,
 	.system_reset_allowed = true,
@@ -537,6 +538,9 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 	sbi_printf("Domain%d Name        %s: %s\n",
 		   dom->index, suffix, dom->name);
 
+	sbi_printf("Domain%d Init Order  %s: 0x%x\n",
+		   dom->index, suffix, dom->init_order);
+
 	sbi_printf("Domain%d Boot HART   %s: %d\n",
 		   dom->index, suffix, dom->boot_hartid);
 
@@ -635,9 +639,14 @@ int sbi_domain_register(struct sbi_domain *dom)
 	if (!dom || domain_finalized)
 		return SBI_EINVAL;
 
-	/* Check if domain already discovered */
+	/*
+	 * Ensure that:
+	 *  1) Domain not already registered
+	 *  2) Initialization order is unique
+	 */
 	sbi_domain_for_each(tdom) {
-		if (tdom == dom)
+		if (tdom == dom ||
+		    tdom->init_order == dom->init_order)
 			return SBI_EALREADY;
 	}
 
@@ -662,15 +671,13 @@ int sbi_domain_register(struct sbi_domain *dom)
 	sbi_hartmask_clear_all(&dom->assigned_harts);
 
 	/*
-	 * Assign a non-ROOT domain to a HART on first come first serve
+	 * Assign HART to a domain with the least initialization order
-	 * basis if the HART is listed as a possible HART of the non-ROOT
+	 * where the HART is listed as a possible HART of the domain.
-	 * domain. If no non-ROOT domain list a HART as possible HART then
-	 * the HART is assigned to the ROOT domain.
 	 */
 	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
 		tdom = sbi_hartindex_to_domain(i);
 		if (tdom) {
-			if (tdom == &root)
+			if (tdom->init_order > dom->init_order)
 				sbi_hartmask_clear_hartindex(i, &tdom->assigned_harts);
 			else
 				continue;

lib/sbi/sbi_domain_context.c

@@ -291,28 +291,29 @@ int sbi_domain_context_exit(void)
 	}
 
 	dom_ctx = ctx->prev_ctx;
+	ctx->prev_ctx = NULL;
 
 	/* If no previous caller context */
 	if (!dom_ctx) {
-		/* Try to find next uninitialized user-defined domain's context */
+		/* Try to find next uninitialized domain with least initialization order */
+		dom_ctx = NULL;
 		sbi_domain_for_each(dom) {
-			if (dom == &root || dom == sbi_domain_thishart_ptr())
+			if (dom == sbi_domain_thishart_ptr())
 				continue;
 
 			if (!sbi_hartmask_test_hartindex(hartindex, dom->possible_harts))
 				continue;
 
 			tmp = hart_context_get(dom, hartindex);
-			if (tmp && !tmp->initialized) {
+			if (tmp && tmp->initialized)
-				dom_ctx = tmp;
+				continue;
-				break;
-			}
-		}
-	}
 
-	/* Take the root domain context if fail to find */
+			if (!dom_ctx || tmp->dom->init_order < dom_ctx->dom->init_order)
+				dom_ctx = tmp;
+		}
+	}
 	if (!dom_ctx)
-		dom_ctx = hart_context_get(&root, hartindex);
+		return SBI_ENOENT;
 
 	return switch_to_next_domain_context(ctx, dom_ctx);
 }

lib/utils/fdt/fdt_domain.c

@@ -330,6 +330,14 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque)
 		    sizeof(dom->name));
 	dom->name[sizeof(dom->name) - 1] = '\0';
 
+	/* Read initalization order */
+	val = fdt_getprop(fdt, domain_offset, "init-order", &len);
+	len = len / sizeof(u32);
+	if (val && len)
+		dom->init_order = fdt32_to_cpu(val[0]);
+	else
+		dom->init_order = (u32)domain_offset;
+
 	/* Setup possible HARTs mask */
 	SBI_HARTMASK_INIT(mask);
 	dom->possible_harts = mask;