lib: sbi_mpxy: Change MPXY state as per-domain data

OpenSBI supports multiple supervisor domains run on same platform. When
these supervisor domains want to communicate with OpenSBI through MPXY
channels, they will allocate MPXY shared memory from their own memory
regions. Therefore, the MPXY state data structure must be per-domain and
per-hart data structure.

This commit registers per-domain MPXY state data in sbi_mpxy_init(). The
original MPXY state allocated in scratch region is also removed. We also
replace sbi_scratch_thishart_offset_ptr() macro as new
sbi_domain_mpxy_state_thishart_ptr() macro which gets MPXY state from
per-domain data.

Signed-off-by: Alvin Chang <alvinga@andestech.com>
Reviewed-by: Yu-Chien Peter Lin <peter.lin@sifive.com>
Reviewed-by: Samuel Holland <samuel.holland@sifive.com>
Link: https://lore.kernel.org/r/20250325071314.3113941-1-alvinga@andestech.com
Signed-off-by: Anup Patel <anup@brainfault.org>

lib/sbi/sbi_mpxy.c

@@ -11,6 +11,7 @@
 #include <sbi/sbi_domain.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_hart.h>
+#include <sbi/sbi_heap.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_mpxy.h>
 #include <sbi/sbi_scratch.h>
@@ -22,9 +23,6 @@
 /** Shared memory size across all harts */
 static unsigned long mpxy_shmem_size = PAGE_SIZE;
 
-/** Offset of pointer to MPXY state in scratch space */
-static unsigned long mpxy_state_offset;
-
 /** List of MPXY proxy channels */
 static SBI_LIST_HEAD(mpxy_channel_list);
 
@@ -83,6 +81,14 @@ struct mpxy_state {
 	struct mpxy_shmem shmem;
 };
 
+static struct mpxy_state *sbi_domain_get_mpxy_state(struct sbi_domain *dom,
+						    u32 hartindex);
+
+/** Macro to obtain the current hart's MPXY state pointer in current domain */
+#define sbi_domain_mpxy_state_thishart_ptr()			\
+	sbi_domain_get_mpxy_state(sbi_domain_thishart_ptr(),	\
+				  current_hartindex())
+
 /** Disable hart shared memory */
 static inline void sbi_mpxy_shmem_disable(struct mpxy_state *ms)
 {
@@ -165,7 +171,7 @@ bool sbi_mpxy_channel_available(void)
 
 static void mpxy_std_attrs_init(struct sbi_mpxy_channel *channel)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	u32 capability = 0;
 
 	/* Reset values */
@@ -233,23 +239,88 @@ int sbi_mpxy_register_channel(struct sbi_mpxy_channel *channel)
 	return SBI_OK;
 }
 
+/** Setup per domain MPXY state data */
+static int domain_mpxy_state_data_setup(struct sbi_domain *dom,
+					struct sbi_domain_data *data,
+					void *data_ptr)
+{
+	struct mpxy_state **dom_hartindex_to_mpxy_state_table = data_ptr;
+	struct mpxy_state *ms;
+	u32 i;
+
+	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
+		ms = sbi_zalloc(sizeof(*ms));
+		if (!ms)
+			return SBI_ENOMEM;
+
+		/*
+		 * TODO: Proper support for checking msi support from
+		 * platform. Currently disable msi and sse and use
+		 * polling
+		 */
+		ms->msi_avail = false;
+		ms->sse_avail = false;
+
+		sbi_mpxy_shmem_disable(ms);
+
+		dom_hartindex_to_mpxy_state_table[i] = ms;
+	}
+
+	return 0;
+}
+
+/** Cleanup per domain MPXY state data */
+static void domain_mpxy_state_data_cleanup(struct sbi_domain *dom,
+					   struct sbi_domain_data *data,
+					   void *data_ptr)
+{
+	struct mpxy_state **dom_hartindex_to_mpxy_state_table = data_ptr;
+	u32 i;
+
+	sbi_hartmask_for_each_hartindex(i, dom->possible_harts)
+		sbi_free(dom_hartindex_to_mpxy_state_table[i]);
+}
+
+static struct sbi_domain_data dmspriv = {
+	.data_setup = domain_mpxy_state_data_setup,
+	.data_cleanup = domain_mpxy_state_data_cleanup,
+};
+
+/**
+ * Get per-domain MPXY state pointer for a given domain and HART index
+ * @param dom pointer to domain
+ * @param hartindex the HART index
+ *
+ * @return per-domain MPXY state pointer for given HART index
+ */
+static struct mpxy_state *sbi_domain_get_mpxy_state(struct sbi_domain *dom,
+						    u32 hartindex)
+{
+	struct mpxy_state **dom_hartindex_to_mpxy_state_table;
+
+	dom_hartindex_to_mpxy_state_table = sbi_domain_data_ptr(dom, &dmspriv);
+	if (!dom_hartindex_to_mpxy_state_table ||
+	    !sbi_hartindex_valid(hartindex))
+		return NULL;
+
+	return dom_hartindex_to_mpxy_state_table[hartindex];
+}
+
 int sbi_mpxy_init(struct sbi_scratch *scratch)
 {
-	struct mpxy_state *ms;
+	int ret;
-
-	mpxy_state_offset = sbi_scratch_alloc_type_offset(struct mpxy_state);
-	if (!mpxy_state_offset)
-		return SBI_ENOMEM;
 
 	/**
-	 * TODO: Proper support for checking msi support from platform.
+	 * Allocate per-domain and per-hart MPXY state data.
-	 * Currently disable msi and sse and use polling
+	 * The data type is "struct mpxy_state **" whose memory space will be
+	 * dynamically allocated by domain_setup_data_one() and
+	 * domain_mpxy_state_data_setup(). Calculate needed size of memory space
+	 * here.
 	 */
-	ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	dmspriv.data_size = sizeof(struct mpxy_state *) * sbi_hart_count();
-	ms->msi_avail = false;
+	ret = sbi_domain_register_data(&dmspriv);
-	ms->sse_avail = false;
+	if (ret)
-
+		return ret;
-	sbi_mpxy_shmem_disable(ms);
 
 	return sbi_platform_mpxy_init(sbi_platform_ptr(scratch));
 }
@@ -263,7 +334,7 @@ int sbi_mpxy_set_shmem(unsigned long shmem_phys_lo,
 		       unsigned long shmem_phys_hi,
 		       unsigned long flags)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	unsigned long *ret_buf;
 
 	/** Disable shared memory if both hi and lo have all bit 1s */
@@ -319,7 +390,7 @@ int sbi_mpxy_set_shmem(unsigned long shmem_phys_lo,
 
 int sbi_mpxy_get_channel_ids(u32 start_index)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	u32 remaining, returned, max_channelids;
 	u32 node_index = 0, node_ret = 0;
 	struct sbi_mpxy_channel *channel;
@@ -370,7 +441,7 @@ int sbi_mpxy_get_channel_ids(u32 start_index)
 
 int sbi_mpxy_read_attrs(u32 channel_id, u32 base_attr_id, u32 attr_count)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	int ret = SBI_SUCCESS;
 	u32 *attr_ptr, end_id;
 	void *shmem_base;
@@ -484,9 +555,9 @@ static int mpxy_check_write_std_attr(struct sbi_mpxy_channel *channel,
  * Write the attribute value
  */
 static void mpxy_write_std_attr(struct sbi_mpxy_channel *channel, u32 attr_id,
-			        u32 attr_val)
+				u32 attr_val)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	struct sbi_mpxy_channel_attrs *attrs = &channel->attrs;
 
 	switch(attr_id) {
@@ -520,7 +591,7 @@ static void mpxy_write_std_attr(struct sbi_mpxy_channel *channel, u32 attr_id,
 
 int sbi_mpxy_write_attrs(u32 channel_id, u32 base_attr_id, u32 attr_count)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	u32 *mem_ptr, attr_id, end_id, attr_val;
 	struct sbi_mpxy_channel *channel;
 	int ret, mem_idx;
@@ -610,7 +681,7 @@ int sbi_mpxy_send_message(u32 channel_id, u8 msg_id,
 			  unsigned long msg_data_len,
 			  unsigned long *resp_data_len)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	struct sbi_mpxy_channel *channel;
 	void *shmem_base, *resp_buf;
 	u32 resp_bufsize;
@@ -668,7 +739,7 @@ int sbi_mpxy_send_message(u32 channel_id, u8 msg_id,
 
 int sbi_mpxy_get_notification_events(u32 channel_id, unsigned long *events_len)
 {
-	struct mpxy_state *ms = sbi_scratch_thishart_offset_ptr(mpxy_state_offset);
+	struct mpxy_state *ms = sbi_domain_mpxy_state_thishart_ptr();
 	struct sbi_mpxy_channel *channel;
 	void *eventsbuf, *shmem_base;
 	int ret;