include: Bump-up version to 1.6

Update the OpenSBI version to 1.6 as part of release preparation.

Signed-off-by: Anup Patel <apatel@ventanamicro.com>
Signed-off-by: Anup Patel <anup@brainfault.org>

.editorconfig

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# See here for more information about the format and editor support:
+# https://editorconfig.org/
+
+root = true
+
+[{*.{c,dts,h,lds,ldS,mk,s,S},Kconfig,Makefile,Makefile.*}]
+charset = utf-8
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = tab
+indent_size = 8
+
+[*.py]
+charset = utf-8
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+indent_size = 4

.github/workflows/repo-lockdown.yml

@@ -0,0 +1,26 @@
+name: 'Repo Lockdown'
+
+on:
+  pull_request_target:
+    types: opened
+
+permissions:
+  pull-requests: write
+
+jobs:
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/repo-lockdown@v4
+        with:
+          pr-comment: |
+            We have mailing list based patch review so it would be great if you can send these patchs to OpenSBI mailing list.
+
+            You need to join OpenSBI mailing list using following link
+            http://lists.infradead.org/mailman/listinfo/opensbi
+
+            Make sure you use "git send-email" to send the patches.
+
+            Thanks for your contribution to OpenSBI project.
+          lock-pr: true
+          close-pr: true

.gitignore

@@ -4,6 +4,8 @@
 # exceptions we need even begin with dot
 !.clang-format
 !.gitignore
+!.github
+!.editorconfig
 
 # Object files
 *.o
@@ -17,3 +19,4 @@ install/
 # Development friendly files
 tags
 cscope*
+*~

Makefile

@@ -79,6 +79,7 @@ export PYTHONDONTWRITEBYTECODE=1
 export KCONFIG_DIR=$(platform_build_dir)/kconfig
 export KCONFIG_AUTOLIST=$(KCONFIG_DIR)/auto.list
 export KCONFIG_AUTOHEADER=$(KCONFIG_DIR)/autoconf.h
+export KCONFIG_AUTOCONFIG=$(KCONFIG_DIR)/auto.conf
 export KCONFIG_AUTOCMD=$(KCONFIG_DIR)/auto.conf.cmd
 export KCONFIG_CONFIG=$(KCONFIG_DIR)/.config
 # Additional exports for include paths in Kconfig files
@@ -90,7 +91,16 @@ endif
 # Find library version
 OPENSBI_VERSION_MAJOR=`grep "define OPENSBI_VERSION_MAJOR" $(include_dir)/sbi/sbi_version.h | sed 's/.*MAJOR.*\([0-9][0-9]*\)/\1/'`
 OPENSBI_VERSION_MINOR=`grep "define OPENSBI_VERSION_MINOR" $(include_dir)/sbi/sbi_version.h | sed 's/.*MINOR.*\([0-9][0-9]*\)/\1/'`
-OPENSBI_VERSION_GIT=$(shell if [ -d $(src_dir)/.git ]; then git describe 2> /dev/null; fi)
+OPENSBI_VERSION_GIT=
+
+# Detect 'git' presence before issuing 'git' commands
+GIT_AVAIL=$(shell command -v git 2> /dev/null)
+ifneq ($(GIT_AVAIL),)
+GIT_DIR=$(shell git rev-parse --git-dir 2> /dev/null)
+ifneq ($(GIT_DIR),)
+OPENSBI_VERSION_GIT=$(shell if [ -d $(GIT_DIR) ]; then git describe 2> /dev/null; fi)
+endif
+endif
 
 # Setup compilation commands
 ifneq ($(LLVM),)
@@ -167,11 +177,24 @@ endif
 # Check whether the linker supports creating PIEs
 OPENSBI_LD_PIE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) $(USE_LD_FLAG) -fPIE -nostdlib -Wl,-pie -x c /dev/null -o /dev/null >/dev/null 2>&1 && echo y || echo n)
 
+# Check whether the linker supports --exclude-libs
+OPENSBI_LD_EXCLUDE_LIBS := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) $(USE_LD_FLAG) "-Wl,--exclude-libs,ALL" -x c /dev/null -o /dev/null >/dev/null 2>&1 && echo y || echo n)
+
 # Check whether the compiler supports -m(no-)save-restore
 CC_SUPPORT_SAVE_RESTORE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -mno-save-restore -x c /dev/null -o /dev/null 2>&1 | grep -e "-save-restore" >/dev/null && echo n || echo y)
 
+# Check whether the compiler supports -m(no-)strict-align
+CC_SUPPORT_STRICT_ALIGN := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -mstrict-align -x c /dev/null -o /dev/null 2>&1 | grep -e "-mstrict-align" -e "-mno-unaligned-access" >/dev/null && echo n || echo y)
+
 # Check whether the assembler and the compiler support the Zicsr and Zifencei extensions
-CC_SUPPORT_ZICSR_ZIFENCEI := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -march=rv$(OPENSBI_CC_XLEN)imafd_zicsr_zifencei -x c /dev/null -o /dev/null 2>&1 | grep "zicsr\|zifencei" > /dev/null && echo n || echo y)
+CC_SUPPORT_ZICSR_ZIFENCEI := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -march=rv$(OPENSBI_CC_XLEN)imafd_zicsr_zifencei -x c /dev/null -o /dev/null 2>&1 | grep -e "zicsr" -e "zifencei" > /dev/null && echo n || echo y)
+
+# Check whether the assembler and the compiler support the Vector extension
+CC_SUPPORT_VECTOR := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -march=rv$(OPENSBI_CC_XLEN)gv -dM -E -x c /dev/null 2>&1 | grep -q riscv.*vector && echo y || echo n)
+
+ifneq ($(OPENSBI_LD_PIE),y)
+$(error Your linker does not support creating PIEs, opensbi requires this.)
+endif
 
 # Build Info:
 # OPENSBI_BUILD_TIME_STAMP -- the compilation time stamp
@@ -210,24 +233,28 @@ ifdef PLATFORM
 menuconfig: $(platform_src_dir)/Kconfig $(src_dir)/Kconfig
 	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/menuconfig.py $(src_dir)/Kconfig
-	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/genconfig.py --header-path $(KCONFIG_AUTOHEADER) --sync-deps $(KCONFIG_DIR) --file-list $(KCONFIG_AUTOLIST) $(src_dir)/Kconfig
 
 .PHONY: savedefconfig
 savedefconfig: $(platform_src_dir)/Kconfig $(src_dir)/Kconfig
 	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/savedefconfig.py --kconfig $(src_dir)/Kconfig --out $(KCONFIG_DIR)/defconfig
 
-$(KCONFIG_CONFIG): $(platform_src_dir)/configs/$(PLATFORM_DEFCONFIG) $(platform_src_dir)/Kconfig $(src_dir)/Kconfig
+$(KCONFIG_CONFIG): $(platform_src_dir)/configs/$(PLATFORM_DEFCONFIG)
 	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/defconfig.py --kconfig $(src_dir)/Kconfig $(platform_src_dir)/configs/$(PLATFORM_DEFCONFIG)
+
+$(KCONFIG_AUTOCONFIG): $(KCONFIG_CONFIG)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/genconfig.py --header-path $(KCONFIG_AUTOHEADER) --sync-deps $(KCONFIG_DIR) --file-list $(KCONFIG_AUTOLIST) $(src_dir)/Kconfig
 
-$(KCONFIG_AUTOCMD): $(KCONFIG_CONFIG)
-	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
+$(KCONFIG_AUTOHEADER): $(KCONFIG_AUTOCONFIG);
+
+$(KCONFIG_AUTOLIST): $(KCONFIG_AUTOCONFIG);
+
+$(KCONFIG_AUTOCMD): $(KCONFIG_AUTOLIST)
 	$(CMD_PREFIX)printf "%s: " $(KCONFIG_CONFIG) > $(KCONFIG_AUTOCMD)
 	$(CMD_PREFIX)cat $(KCONFIG_AUTOLIST) | tr '\n' ' ' >> $(KCONFIG_AUTOCMD)
 
-include $(KCONFIG_CONFIG)
+include $(KCONFIG_AUTOCONFIG)
 include $(KCONFIG_AUTOCMD)
 endif
 
@@ -270,10 +297,9 @@ ifndef PLATFORM_RISCV_ABI
 endif
 ifndef PLATFORM_RISCV_ISA
   ifneq ($(PLATFORM_RISCV_TOOLCHAIN_DEFAULT), 1)
+    PLATFORM_RISCV_ISA := rv$(PLATFORM_RISCV_XLEN)imafdc
     ifeq ($(CC_SUPPORT_ZICSR_ZIFENCEI), y)
-      PLATFORM_RISCV_ISA = rv$(PLATFORM_RISCV_XLEN)imafdc_zicsr_zifencei
-    else
-      PLATFORM_RISCV_ISA = rv$(PLATFORM_RISCV_XLEN)imafdc
+      PLATFORM_RISCV_ISA := $(PLATFORM_RISCV_ISA)_zicsr_zifencei
     endif
   else
     PLATFORM_RISCV_ISA = $(OPENSBI_CC_ISA)
@@ -331,23 +357,24 @@ GENFLAGS	+=	$(libsbiutils-genflags-y)
 GENFLAGS	+=	$(platform-genflags-y)
 GENFLAGS	+=	$(firmware-genflags-y)
 
-CFLAGS		=	-g -Wall -Werror -ffreestanding -nostdlib -fno-stack-protector -fno-strict-aliasing
-ifneq ($(DEBUG),)
-CFLAGS		+=	-O0
-else
-CFLAGS		+=	-O2
+CFLAGS		=	-g -Wall -Werror -ffreestanding -nostdlib -fno-stack-protector -fno-strict-aliasing -ffunction-sections -fdata-sections
+CFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls
+# Optionally supported flags
+ifeq ($(CC_SUPPORT_VECTOR),y)
+CFLAGS		+=	-DOPENSBI_CC_SUPPORT_VECTOR
 endif
-CFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls -mstrict-align
-# enable -m(no-)save-restore option by CC_SUPPORT_SAVE_RESTORE
 ifeq ($(CC_SUPPORT_SAVE_RESTORE),y)
 CFLAGS		+=	-mno-save-restore
 endif
+ifeq ($(CC_SUPPORT_STRICT_ALIGN),y)
+CFLAGS		+=	-mstrict-align
+endif
 CFLAGS		+=	-mabi=$(PLATFORM_RISCV_ABI) -march=$(PLATFORM_RISCV_ISA)
 CFLAGS		+=	-mcmodel=$(PLATFORM_RISCV_CODE_MODEL)
 CFLAGS		+=	$(RELAX_FLAG)
 CFLAGS		+=	$(GENFLAGS)
 CFLAGS		+=	$(platform-cflags-y)
-CFLAGS		+=	-fno-pie -no-pie
+CFLAGS		+=	-fPIE -pie
 CFLAGS		+=	$(firmware-cflags-y)
 
 CPPFLAGS	+=	$(GENFLAGS)
@@ -355,11 +382,15 @@ CPPFLAGS	+=	$(platform-cppflags-y)
 CPPFLAGS	+=	$(firmware-cppflags-y)
 
 ASFLAGS		=	-g -Wall -nostdlib
-ASFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls -mstrict-align
-# enable -m(no-)save-restore option by CC_SUPPORT_SAVE_RESTORE
+ASFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls
+ASFLAGS		+=	-fPIE
+# Optionally supported flags
 ifeq ($(CC_SUPPORT_SAVE_RESTORE),y)
 ASFLAGS		+=	-mno-save-restore
 endif
+ifeq ($(CC_SUPPORT_STRICT_ALIGN),y)
+ASFLAGS		+=	-mstrict-align
+endif
 ASFLAGS		+=	-mabi=$(PLATFORM_RISCV_ABI) -march=$(PLATFORM_RISCV_ISA)
 ASFLAGS		+=	-mcmodel=$(PLATFORM_RISCV_CODE_MODEL)
 ASFLAGS		+=	$(RELAX_FLAG)
@@ -375,8 +406,12 @@ ASFLAGS		+=	$(firmware-asflags-y)
 ARFLAGS		=	rcs
 
 ELFFLAGS	+=	$(USE_LD_FLAG)
+ELFFLAGS	+=	-Wl,--gc-sections
+ifeq ($(OPENSBI_LD_EXCLUDE_LIBS),y)
 ELFFLAGS	+=	-Wl,--exclude-libs,ALL
+endif
 ELFFLAGS	+=	-Wl,--build-id=none
+ELFFLAGS	+=	-Wl,--no-dynamic-linker -Wl,-pie
 ELFFLAGS	+=	$(platform-ldflags-y)
 ELFFLAGS	+=	$(firmware-ldflags-y)
 
@@ -390,6 +425,13 @@ MERGEFLAGS	+=	-m elf$(PLATFORM_RISCV_XLEN)lriscv
 
 DTSCPPFLAGS	=	$(CPPFLAGS) -nostdinc -nostdlib -fno-builtin -D__DTS__ -x assembler-with-cpp
 
+ifneq ($(DEBUG),)
+CFLAGS		+=	-O0
+ELFFLAGS	+=	-Wl,--print-gc-sections
+else
+CFLAGS		+=	-O2
+endif
+
 # Setup functions for compilation
 define dynamic_flags
 -I$(shell dirname $(2)) -D__OBJNAME__=$(subst -,_,$(shell basename $(1) .o))
@@ -464,8 +506,8 @@ compile_d2c = $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     $(src_dir)/scripts/d2c.sh -i $(6) -a $(D2C_ALIGN_BYTES) -p $(D2C_NAME_PREFIX) -t $(D2C_PADDING_BYTES) > $(1)
 compile_carray = $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     echo " CARRAY    $(subst $(build_dir)/,,$(1))"; \
-	     $(eval CARRAY_VAR_LIST := $(carray-$(subst .c,,$(shell basename $(1)))-y)) \
-	     $(src_dir)/scripts/carray.sh -i $(2) -l "$(CARRAY_VAR_LIST)" > $(1)
+	     $(eval CARRAY_VAR_LIST := $(carray-$(subst .carray.c,,$(shell basename $(1)))-y)) \
+	     $(src_dir)/scripts/carray.sh -i $(2) -l "$(CARRAY_VAR_LIST)" > $(1) || rm $(1)
 compile_gen_dep = $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     echo " GEN-DEP   $(subst $(build_dir)/,,$(1))"; \
 	     echo "$(1:.dep=$(2)): $(3)" >> $(1)
@@ -490,14 +532,14 @@ $(build_dir)/lib/libsbi.a: $(libsbi-objs-path-y)
 $(platform_build_dir)/lib/libplatsbi.a: $(libsbi-objs-path-y) $(libsbiutils-objs-path-y) $(platform-objs-path-y)
 	$(call compile_ar,$@,$^)
 
-$(build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_CONFIG))
+$(build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
-$(build_dir)/%.c: $(src_dir)/%.carray
+$(build_dir)/%.carray.c: $(src_dir)/%.carray $(src_dir)/scripts/carray.sh
 	$(call compile_carray,$@,$<)
 
-$(build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_CONFIG)
+$(build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc_dep,$@,$<)
 
 $(build_dir)/%.o: $(src_dir)/%.c
@@ -511,24 +553,24 @@ $(build_dir)/lib/sbi/sbi_init.o: $(libsbi_dir)/sbi_init.c FORCE
 	$(call compile_cc,$@,$<)
 endif
 
-$(build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_CONFIG)
+$(build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_AUTOHEADER)
 	$(call compile_as_dep,$@,$<)
 
 $(build_dir)/%.o: $(src_dir)/%.S
 	$(call compile_as,$@,$<)
 
 # Rules for platform sources
-$(platform_build_dir)/%.dep: $(platform_src_dir)/%.carray $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_CONFIG))
+$(platform_build_dir)/%.dep: $(platform_src_dir)/%.carray $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
-$(platform_build_dir)/%.c: $(platform_src_dir)/%.carray
+$(platform_build_dir)/%.carray.c: $(platform_src_dir)/%.carray $(src_dir)/scripts/carray.sh
 	$(call compile_carray,$@,$<)
 
-$(platform_build_dir)/%.dep: $(platform_src_dir)/%.c $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(platform_src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc_dep,$@,$<)
 
-$(platform_build_dir)/%.o: $(platform_src_dir)/%.c $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.o: $(platform_src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc,$@,$<)
 
 $(platform_build_dir)/%.dep: $(platform_src_dir)/%.S
@@ -537,8 +579,8 @@ $(platform_build_dir)/%.dep: $(platform_src_dir)/%.S
 $(platform_build_dir)/%.o: $(platform_src_dir)/%.S
 	$(call compile_as,$@,$<)
 
-$(platform_build_dir)/%.dep: $(platform_src_dir)/%.dts $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.dtb,$< $(KCONFIG_CONFIG))
+$(platform_build_dir)/%.dep: $(platform_src_dir)/%.dts $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.dtb,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.c,$(@:.dep=.dtb))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
@@ -555,26 +597,26 @@ $(platform_build_dir)/%.bin: $(platform_build_dir)/%.elf
 $(platform_build_dir)/%.elf: $(platform_build_dir)/%.o $(platform_build_dir)/%.elf.ld $(platform_build_dir)/lib/libplatsbi.a
 	$(call compile_elf,$@,$@.ld,$< $(platform_build_dir)/lib/libplatsbi.a)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.ldS $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(src_dir)/%.ldS $(KCONFIG_AUTOHEADER)
 	$(call compile_cpp_dep,$@,.ld,$<)
 
 $(platform_build_dir)/%.ld: $(src_dir)/%.ldS
 	$(call compile_cpp,$@,$<)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_CONFIG))
+$(platform_build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
-$(platform_build_dir)/%.c: $(src_dir)/%.carray
+$(platform_build_dir)/%.carray.c: $(src_dir)/%.carray $(src_dir)/scripts/carray.sh
 	$(call compile_carray,$@,$<)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc_dep,$@,$<)
 
 $(platform_build_dir)/%.o: $(src_dir)/%.c
 	$(call compile_cc,$@,$<)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_AUTOHEADER)
 	$(call compile_as_dep,$@,$<)
 
 $(platform_build_dir)/%.o: $(src_dir)/%.S
@@ -662,6 +704,8 @@ clean:
 	$(CMD_PREFIX)mkdir -p $(build_dir)
 	$(if $(V), @echo " RM        $(build_dir)/*.o")
 	$(CMD_PREFIX)find $(build_dir) -type f -name "*.o" -exec rm -rf {} +
+	$(if $(V), @echo " RM        $(build_dir)/*.carray.c")
+	$(CMD_PREFIX)find $(build_dir) -type f -name "*.carray.c" -exec rm -rf {} +
 	$(if $(V), @echo " RM        $(build_dir)/*.a")
 	$(CMD_PREFIX)find $(build_dir) -type f -name "*.a" -exec rm -rf {} +
 	$(if $(V), @echo " RM        $(build_dir)/*.elf")

README.md

@@ -276,8 +276,7 @@ document.
 
 NOTE: Using Clang with a `riscv*-linux-gnu` GNU binutils linker has been seen
 to produce broken binaries with missing relocations; it is therefore currently
-recommended that this combination be avoided or *FW_PIC=n* be used to disable
-building OpenSBI as a position-independent binary.
+recommended that this combination be avoided.
 
 Building with timestamp and compiler info
 -----------------------------------------

docs/contributing.md

@@ -1,17 +1,15 @@
 OpenSBI Contribution Guideline
 ==============================
 
-All contributions to OpenSBI can be sent in the following ways:
-1. Email patches to the OpenSBI mailing list at `opensbi@lists.infradead.org`
-2. GitHub Pull Requests (PRs) to the [OpenSBI main repository]
+All contributions to OpenSBI must be sent via email patches to the OpenSBI
+mailing list at `opensbi@lists.infradead.org`
 
 To join the OpenSBI mailing list, please visit the [OpenSBI infradead page].
 
-The OpenSBI maintainers prefer patches via the OpenSBI mailing list
-(option 1 above) so that they are visible to a wider audience. All
-accepted patches on the OpenSBI mailing list will be taken by any of
-the OpenSBI maintainers and merged into the [OpenSBI main repository]
-using GitHub PRs.
+The mailing list based patch approach is preferred over github PRs so that they
+are visible to a wider audience. All accepted patches on the OpenSBI mailing
+list will be taken by one of the OpenSBI maintainers and merged into the
+[OpenSBI main repository].
 
 All contributed work must follow the following rules:
 1. OpenSBI code should be written in accordance to the [Linux coding style].
@@ -33,9 +31,6 @@ riscv/opensbi repository. Instead, prefer using a fork of the riscv/opensbi main
 repository and branches within that fork to create pull requests.
 7. A maintainer cannot merge his own pull requests in the riscv/opensbi main
 repository.
-8. A pull request must get at least one review from a maintainer.
-9. A pull request must spend at least 24 hours in review to allow for other
-developers to review.
 
 -----------------------------------------------------------------------
 

docs/domain_support.md

@@ -126,9 +126,6 @@ The DT properties of a domain configuration DT node are as follows:
 * **compatible** (Mandatory) - The compatible string of the domain
   configuration. This DT property should have value *"opensbi,domain,config"*
 
-* **system-suspend-test** (Optional) - When present, enable a system
-  suspend test implementation which simply waits five seconds and issues a WFI.
-
 ### Domain Memory Region Node
 
 The domain memory region DT node describes details of a memory region and
@@ -181,9 +178,7 @@ The DT properties of a domain instance DT node are as follows:
   boot HART of the domain instance.
 * **next-arg1** (Optional) - The 64 bit next booting stage arg1 for the
   domain instance. If this DT property is not available and coldboot HART
-  is not assigned to the domain instance then **0x0** is used as default
-  value. If this DT property is not available and coldboot HART is assigned
-  to the domain instance then **next booting stage arg1 of coldboot HART**
+  is not assigned to the domain instance then **next booting stage arg1 of coldboot HART**
   is used as default value.
 * **next-addr** (Optional) - The 64 bit next booting stage address for the
   domain instance. If this DT property is not available and coldboot HART
@@ -237,7 +232,6 @@ be done:
     chosen {
         opensbi-domains {
             compatible = "opensbi,domain,config";
-            system-suspend-test;
 
             tmem: tmem {
                 compatible = "opensbi,domain,memregion";

docs/doxygen.cfg

@@ -796,6 +796,8 @@ INPUT                  = @@SRC_DIR@@/README.md \
                          @@SRC_DIR@@/docs/platform_requirements.md \
                          @@SRC_DIR@@/docs/library_usage.md \
                          @@SRC_DIR@@/docs/domain_support.md \
+                         @@SRC_DIR@@/docs/opensbi_config.md \
+                         @@SRC_DIR@@/docs/writing_tests.md \
                          @@SRC_DIR@@/docs/firmware \
                          @@SRC_DIR@@/docs/platform \
                          @@SRC_DIR@@/include \

docs/firmware/fw.md

@@ -61,20 +61,15 @@ Firmware Configuration and Compilation
 All firmware types support the following common compile time configuration
 parameters:
 
-* **FW_TEXT_START** - Defines the execution address of the OpenSBI firmware.
-  This configuration parameter is mandatory.
+* **FW_TEXT_START** - Defines the compile time address of the OpenSBI
+  firmware. This configuration parameter is optional and assumed to be
+  `0` if not specified.
 * **FW_FDT_PATH** - Path to an external flattened device tree binary file to
   be embedded in the *.rodata* section of the final firmware. If this option
   is not provided then the firmware will expect the FDT to be passed as an
   argument by the prior booting stage.
 * **FW_FDT_PADDING** - Optional zero bytes padding to the embedded flattened
   device tree binary file specified by **FW_FDT_PATH** option.
-* **FW_PIC** - "FW_PIC=y" generates position independent executable firmware
-  images. OpenSBI can run at arbitrary address with appropriate alignment.
-  Therefore, the original relocation mechanism ("FW_PIC=n") will be skipped.
-  In other words, OpenSBI will directly run at the load address without any
-  code movement. This option requires a toolchain with PIE support, and it
-  is on by default.
 
 Additionally, each firmware type as a set of type specific configuration
 parameters. Detailed information for each firmware type can be found in the

docs/firmware/fw_jump.md

@@ -31,9 +31,14 @@ follows:
 
 * **FW_JUMP_ADDR** - Address of the entry point of the booting stage to be
   executed following OpenSBI firmware. This address generally corresponds
-  exactly to the address where this next booting stage was loaded. This is a
-  mandatory parameter. Compilation errors will result from not defining this
-  address.
+  exactly to the address where this next booting stage was loaded.
+  At least one of *FW_JUMP_ADDR* and *FW_JUMP_OFFSET* (see below) should be
+  defined. Compilation errors will result from not defining one of them.
+
+* **FW_JUMP_OFFSET** - Address offset from the opensbi load address where the
+  entry point of the next booting stage is located. This offset is used as
+  relocatable address of the next booting stage entry point. If *FW_JUMP_ADDR*
+  is also defined, the firmware will prefer *FW_JUMP_ADDR*.
 
 * **FW_JUMP_FDT_ADDR** - Address where the *flattened device tree (FDT file)*
   passed by the prior booting stage will be placed in memory before executing
@@ -57,6 +62,12 @@ follows:
   echo fdt overlaps kernel, increase FW_JUMP_FDT_ADDR
   ```
 
+* **FW_JUMP_FDT_OFFSET** - Address offset from the opensbi load address where
+  the FDT will be passed to the next booting stage. This offset is used
+  as relocatable address of the FDT passed to the next booting stage. If
+  *FW_JUMP_FDT_ADDR* is also defined, the firmware will prefer
+  *FW_JUMP_FDT_ADDR*.
+
 *FW_JUMP* Example
 -----------------
 

docs/firmware/fw_payload.md

@@ -23,7 +23,7 @@ The *FW_PAYLOAD* firmware can be enabled by any of the following methods:
 2. Specifying `FW_PAYLOAD=y` in the target platform *objects.mk* configuration
    file.
 
-The compiled *FW_PAYLOAD* firmware ELF file is named *fw_jump.elf*. Its
+The compiled *FW_PAYLOAD* firmware ELF file is named *fw_payload.elf*. Its
 expanded image file is *fw_payload.bin*. Both files are created in the
 platform-specific build directory under the
 *build/platform/<platform_subdir>/firmware* directory.
@@ -36,8 +36,8 @@ options. These configuration parameters can be defined using either the top
 level `make` command line or the target platform *objects.mk* configuration
 file. The parameters currently defined are as follows:
 
-* **FW_PAYLOAD_OFFSET** - Offset from *FW_TEXT_START* where the payload binary
-  will be linked in the final *FW_PAYLOAD* firmware binary image. This
+* **FW_PAYLOAD_OFFSET** - Offset from the opensbi load address where the payload
+  binary will be linked in the final *FW_PAYLOAD* firmware binary image. This
   configuration parameter is mandatory if *FW_PAYLOAD_ALIGN* is not defined.
   Compilation errors will result from an incorrect definition of
   *FW_PAYLOAD_OFFSET* or of *FW_PAYLOAD_ALIGN*, or if neither of these
@@ -62,6 +62,11 @@ file. The parameters currently defined are as follows:
   firmware will pass the FDT address passed by the previous booting stage
   to the next booting stage.
 
+* **FW_PAYLOAD_FDT_OFFSET** - Address offset from the opensbi load address where
+  the FDT will be passed to the next booting stage. This offset is used as
+  relocatable address of the FDT passed to the next booting stage. If
+  *FW_PAYLOAD_FDT_ADDR* is also defined, the firmware will prefer *FW_PAYLOAD_FDT_ADDR*.
+
 *FW_PAYLOAD* Example
 --------------------
 

docs/opensbi_config.md

@@ -0,0 +1,91 @@
+OpenSBI Device Tree Configuration Guideline
+==================================
+
+Some configurations of OpenSBI's Generic Platform can be described
+in the **device tree (DT) blob** (or flattened device tree) passed
+to the OpenSBI firmwares by the previous booting stage. OpenSBI will
+parse and use these configurations during the boot phase, but delete
+them from the device tree at the end of cold boot.
+
+### OpenSBI Configuration Node
+
+All nodes related to OpenSBI configuration should be under the OpenSBI
+configuration DT node. The **/chosen** DT node is the preferred parent
+of the OpenSBI configuration DT node.
+
+The DT properties of a domain configuration DT node are as follows:
+
+* **compatible** (Mandatory) - The compatible string of the OpenSBI
+  configuration. This DT property should have value *"opensbi,config"*
+
+* **cold-boot-harts** (Optional) - If a platform lacks an override
+  cold_boot_allowed() mechanism, this DT property specifies that a
+  set of harts is permitted to perform a cold boot. Otherwise, all
+  harts are allowed to cold boot.
+
+* **heap-size** (Optional) - When present, the specified value is used
+  as the size of the heap in bytes.
+
+* **system-suspend-test** (Optional) - When present, enable a system
+  suspend test implementation which simply waits five seconds and issues a WFI.
+
+The OpenSBI Configuration Node will be deleted at the end of cold boot
+(replace the node (subtree) with nop tags).
+
+### Example
+
+```text
+    chosen {
+        opensbi-config {
+            compatible = "opensbi,config";
+            cold-boot-harts = <&cpu1 &cpu2 &cpu3 &cpu4>;
+            heap-size = <0x400000>;
+            system-suspend-test;
+        };
+    };
+
+    cpus {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        timebase-frequency = <10000000>;
+
+        cpu0: cpu@0 {
+            device_type = "cpu";
+            reg = <0x00>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu1: cpu@1 {
+            device_type = "cpu";
+            reg = <0x01>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu2: cpu@2 {
+            device_type = "cpu";
+            reg = <0x02>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu3: cpu@3 {
+            device_type = "cpu";
+            reg = <0x03>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu4: cpu@4 {
+            device_type = "cpu";
+            reg = <0x04>;
+            compatible = "riscv";
+            ...
+        };
+    };
+
+    uart1: serial@10011000 {
+        ...
+    };
+```

docs/platform/generic.md

@@ -9,10 +9,9 @@ boards.
 
 By default, the generic FDT platform makes following assumptions:
 
-1. platform FW_TEXT_START is 0x80000000
-2. platform features are default
-3. platform stack size is default
-4. platform has no quirks or work-arounds
+1. platform features are default
+2. platform stack size is default
+3. platform has no quirks or work-arounds
 
 The above assumptions (except 1) can be overridden by adding special platform
 callbacks which will be called based on FDT root node compatible string.
@@ -33,10 +32,6 @@ Users of the generic FDT platform will have to ensure that:
 To build the platform-specific library and firmware images, provide the
 *PLATFORM=generic* parameter to the top level `make` command.
 
-For custom FW_TEXT_START, we can build the platform-specific library and
-firmware images by passing *PLATFORM=generic FW_TEXT_START=<custom_text_start>*
-parameter to the top level `make` command.
-
 Platform Options
 ----------------
 

docs/platform/qemu_virt.md

@@ -56,7 +56,14 @@ qemu-system-riscv64 -M virt -m 256M -nographic \
 **Linux Kernel Payload**
 
 Note: We assume that the Linux kernel is compiled using
-*arch/riscv/configs/defconfig*.
+*arch/riscv/configs/defconfig*. The kernel must be a flattened image (a file
+called `Image`) rather than an ELF (`vmlinux`).
+
+Example of building a Linux kernel:
+```
+make ARCH=riscv CROSS_COMPILE=riscv64-linux- defconfig
+make ARCH=riscv CROSS_COMPILE=riscv64-linux- Image
+```
 
 Build:
 ```

docs/writing_tests.md

@@ -0,0 +1,127 @@
+Writing tests for OpenSBI
+=========================
+
+SBIUnit
+-------
+SBIUnit is a set of macros and functions which simplify the test development and
+automate the test execution and evaluation. All of the SBIUnit definitions are
+in the `include/sbi/sbi_unit_test.h` header file, and implementations are
+available in `lib/sbi/tests/sbi_unit_test.c`.
+
+Simple SBIUnit test
+-------------------
+
+For instance, we would like to test the following function from
+`lib/sbi/sbi_string.c`:
+
+```c
+size_t sbi_strlen(const char *str)
+{
+	unsigned long ret = 0;
+
+	while (*str != '\0') {
+		ret++;
+		str++;
+	}
+
+	return ret;
+}
+```
+
+which calculates the string length.
+
+Create the file `lib/sbi/tests/sbi_string_test.c` with the following content:
+
+```c
+#include <sbi/sbi_unit_test.h>
+#include <sbi/sbi_string.h>
+
+static void strlen_test(struct sbiunit_test_case *test)
+{
+	SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hello"), 5);
+	SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hell\0o"), 4);
+}
+
+static struct sbiunit_test_case string_test_cases[] = {
+	SBIUNIT_TEST_CASE(strlen_test),
+	SBIUNIT_END_CASE,
+};
+
+SBIUNIT_TEST_SUITE(string_test_suite, string_test_cases);
+```
+
+Then, add the corresponding Makefile entries to `lib/sbi/tests/objects.mk`:
+```lang-makefile
+...
+carray-sbi_unit_tests-$(CONFIG_SBIUNIT) += string_test_suite
+libsbi-objs-$(CONFIG_SBIUNIT) += tests/sbi_string_test.o
+```
+
+Now, run `make clean` in order to regenerate the carray-related files.
+
+Recompile OpenSBI with the CONFIG_SBIUNIT option enabled and run it in QEMU.
+You will see something like this:
+```
+# make PLATFORM=generic run
+...
+# Running SBIUNIT tests #
+...
+## Running test suite: string_test_suite
+[PASSED] strlen_test
+1 PASSED / 0 FAILED / 1 TOTAL
+```
+
+Now let's try to change this test in the way that it will fail:
+
+```c
+- SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hello"), 5);
++ SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hello"), 100);
+```
+
+`make all` and `make run` it again:
+```
+...
+# Running SBIUNIT tests #
+...
+## Running test suite: string_test_suite
+[SBIUnit] [.../opensbi/lib/sbi/tests/sbi_string_test.c:6]: strlen_test: Condition "(sbi_strlen("Hello")) == (100)" expected to be true!
+[FAILED] strlen_test
+0 PASSED / 1 FAILED / 1 TOTAL
+```
+Covering the static functions / using the static definitions
+------------------------------------------------------------
+
+SBIUnit also allows you to test static functions. In order to do so, simply
+include your test source in the file you would like to test. Complementing the
+example above, just add this to the `lib/sbi/sbi_string.c` file:
+
+```c
+#ifdef CONFIG_SBIUNIT
+#include "tests/sbi_string_test.c"
+#endif
+```
+
+In this case you should only add a new carray entry pointing to the test suite
+to `lib/sbi/tests/objects.mk`:
+```lang-makefile
+...
+carray-sbi_unit_tests-$(CONFIG_SBIUNIT) += string_test_suite
+```
+
+You don't have to compile the `sbi_string_test.o` separately, because the
+test code will be included into the `sbi_string` object file.
+
+"Mocking" the structures
+------------------------
+See the example of structure "mocking" in `lib/sbi/tests/sbi_console_test.c`,
+where the sbi_console_device structure was mocked to be used in various
+console-related functions in order to test them.
+
+API Reference
+-------------
+All of the `SBIUNIT_EXPECT_*` macros will cause a test case to fail if the
+corresponding conditions are not met, however, the execution of a particular
+test case will not be stopped.
+
+All of the `SBIUNIT_ASSERT_*` macros will cause a test case to fail and stop
+immediately, triggering a panic.

firmware/fw_base.S

@@ -14,8 +14,8 @@
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_trap.h>
 
-#define BOOT_STATUS_RELOCATE_DONE	1
-#define BOOT_STATUS_BOOT_HART_DONE	2
+#define BOOT_LOTTERY_ACQUIRED		1
+#define BOOT_STATUS_BOOT_HART_DONE	1
 
 .macro	MOV_3R __d0, __s0, __d1, __s1, __d2, __s2
 	add	\__d0, \__s0, zero
@@ -31,15 +31,14 @@
 	add	\__d4, \__s4, zero
 .endm
 
-/*
- * If __start_reg <= __check_reg and __check_reg < __end_reg then
- *   jump to __pass
- */
-.macro BRANGE __start_reg, __end_reg, __check_reg, __jump_lable
-	blt	\__check_reg, \__start_reg, 999f
-	bge	\__check_reg, \__end_reg, 999f
-	j	\__jump_lable
-999:
+.macro CLEAR_MDT tmp
+#if __riscv_xlen == 32
+	li 	\tmp, MSTATUSH_MDT
+	csrc	CSR_MSTATUSH, \tmp
+#else
+	li 	\tmp, MSTATUS_MDT
+	csrc	CSR_MSTATUS, \tmp
+#endif
 .endm
 
 	.section .entry, "ax", %progbits
@@ -55,27 +54,18 @@ _start:
 	li	a7, -1
 	beq	a6, a7, _try_lottery
 	/* Jump to relocation wait loop if we are not boot hart */
-	bne	a0, a6, _wait_relocate_copy_done
+	bne	a0, a6, _wait_for_boot_hart
 _try_lottery:
 	/* Jump to relocation wait loop if we don't get relocation lottery */
-	lla	a6, _relocate_lottery
-	li	a7, 1
-	amoadd.w a6, a7, (a6)
-	bnez	a6, _wait_relocate_copy_done
+	lla	a6, _boot_lottery
+	li	a7, BOOT_LOTTERY_ACQUIRED
+	amoswap.w a6, a7, (a6)
+	bnez	a6, _wait_for_boot_hart
 
-	/* Save load address */
-	lla	t0, _load_start
-	lla	t1, _fw_start
-	REG_S	t1, 0(t0)
-
-#ifdef FW_PIC
 	/* relocate the global table content */
-	lla	t0, _link_start
-	REG_L	t0, 0(t0)
-	/* t1 shall has the address of _fw_start */
-	sub	t2, t1, t0
-	lla	t3, _runtime_offset
-	REG_S	t2, (t3)
+	li	t0, FW_TEXT_START	/* link start */
+	lla	t1, _fw_start		/* load start */
+	sub	t2, t1, t0		/* load offset */
 	lla	t0, __rel_dyn_start
 	lla	t1, __rel_dyn_end
 	beq	t0, t1, _relocate_done
@@ -92,103 +82,10 @@ _try_lottery:
 3:
 	addi	t0, t0, (REGBYTES * 3)
 	blt	t0, t1, 2b
-	j	_relocate_done
-_wait_relocate_copy_done:
-	j	_wait_for_boot_hart
-#else
-	/* Relocate if load address != link address */
-_relocate:
-	lla	t0, _link_start
-	REG_L	t0, 0(t0)
-	lla	t1, _link_end
-	REG_L	t1, 0(t1)
-	lla	t2, _load_start
-	REG_L	t2, 0(t2)
-	beq	t0, t2, _relocate_done
-	sub	t3, t1, t0
-	add	t3, t3, t2
-	lla	t4, _relocate_done
-	sub	t4, t4, t2
-	add	t4, t4, t0
-	blt	t2, t0, _relocate_copy_to_upper
-_relocate_copy_to_lower:
-	ble	t1, t2, _relocate_copy_to_lower_loop
-	lla	t3, _relocate_lottery
-	BRANGE	t2, t1, t3, _start_hang
-	lla	t3, _boot_status
-	BRANGE	t2, t1, t3, _start_hang
-	lla	t3, _relocate
-	lla	t5, _relocate_done
-	BRANGE	t2, t1, t3, _start_hang
-	BRANGE	t2, t1, t5, _start_hang
-	BRANGE  t3, t5, t2, _start_hang
-_relocate_copy_to_lower_loop:
-	REG_L	t3, 0(t2)
-	REG_S	t3, 0(t0)
-	add	t0, t0, __SIZEOF_POINTER__
-	add	t2, t2, __SIZEOF_POINTER__
-	blt	t0, t1, _relocate_copy_to_lower_loop
-	jr	t4
-_relocate_copy_to_upper:
-	ble	t3, t0, _relocate_copy_to_upper_loop
-	lla	t2, _relocate_lottery
-	BRANGE	t0, t3, t2, _start_hang
-	lla	t2, _boot_status
-	BRANGE	t0, t3, t2, _start_hang
-	lla	t2, _relocate
-	lla	t5, _relocate_done
-	BRANGE	t0, t3, t2, _start_hang
-	BRANGE	t0, t3, t5, _start_hang
-	BRANGE	t2, t5, t0, _start_hang
-_relocate_copy_to_upper_loop:
-	add	t3, t3, -__SIZEOF_POINTER__
-	add	t1, t1, -__SIZEOF_POINTER__
-	REG_L	t2, 0(t3)
-	REG_S	t2, 0(t1)
-	blt	t0, t1, _relocate_copy_to_upper_loop
-	jr	t4
-_wait_relocate_copy_done:
-	lla	t0, _fw_start
-	lla	t1, _link_start
-	REG_L	t1, 0(t1)
-	beq	t0, t1, _wait_for_boot_hart
-	lla	t2, _boot_status
-	lla	t3, _wait_for_boot_hart
-	sub	t3, t3, t0
-	add	t3, t3, t1
-1:
-	/* waitting for relocate copy done (_boot_status == 1) */
-	li	t4, BOOT_STATUS_RELOCATE_DONE
-	REG_L	t5, 0(t2)
-	/* Reduce the bus traffic so that boot hart may proceed faster */
-	nop
-	nop
-	nop
-	bgt     t4, t5, 1b
-	jr	t3
-#endif
 _relocate_done:
-
-	/*
-	 * Mark relocate copy done
-	 * Use _boot_status copy relative to the load address
-	 */
-	lla	t0, _boot_status
-#ifndef FW_PIC
-	lla	t1, _link_start
-	REG_L	t1, 0(t1)
-	lla	t2, _load_start
-	REG_L	t2, 0(t2)
-	sub	t0, t0, t1
-	add	t0, t0, t2
-#endif
-	li	t1, BOOT_STATUS_RELOCATE_DONE
-	REG_S	t1, 0(t0)
-	fence	rw, rw
-
 	/* At this point we are running from link address */
 
-	/* Reset all registers for boot HART */
+	/* Reset all registers except ra, a0, a1, a2, a3 and a4 for boot HART */
 	li	ra, 0
 	call	_reset_regs
 
@@ -204,6 +101,14 @@ _bss_zero:
 	lla	s4, _start_hang
 	csrw	CSR_MTVEC, s4
 
+	/*
+	 * While at this point, trap handling is rudimentary, if a trap happens,
+	 * it will end up in _start_hang which is enough to hook up a GDB. Clear
+	 * MDT to avoid generating a double trap and thus entering a
+	 * critical-error state.
+	 */
+	CLEAR_MDT t0
+
 	/* Setup temporary stack */
 	lla	s4, _fw_end
 	li	s5, (SBI_SCRATCH_SIZE * 2)
@@ -319,10 +224,8 @@ _scratch_init:
 	/* Store hartid-to-scratch function address in scratch space */
 	lla	a4, _hartid_to_scratch
 	REG_S	a4, SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET(tp)
-	/* Store trap-exit function address in scratch space */
-	lla	a4, _trap_exit
-	REG_S	a4, SBI_SCRATCH_TRAP_EXIT_OFFSET(tp)
-	/* Clear tmp0 in scratch space */
+	/* Clear trap_context and tmp0 in scratch space */
+	REG_S	zero, SBI_SCRATCH_TRAP_CONTEXT_OFFSET(tp)
 	REG_S	zero, SBI_SCRATCH_TMP0_OFFSET(tp)
 	/* Store firmware options in scratch space */
 	MOV_3R	s0, a0, s1, a1, s2, a2
@@ -333,6 +236,8 @@ _scratch_init:
 #endif
 	REG_S	a0, SBI_SCRATCH_OPTIONS_OFFSET(tp)
 	MOV_3R	a0, s0, a1, s1, a2, s2
+	/* Store hart index in scratch space */
+	REG_S	t1, SBI_SCRATCH_HARTINDEX_OFFSET(tp)
 	/* Move to next scratch space */
 	add	t1, t1, t2
 	blt	t1, s7, _scratch_init
@@ -357,30 +262,14 @@ _scratch_init:
 	beq	t1, a1, _fdt_reloc_done
 	/* t0 = source FDT start address */
 	add	t0, a1, zero
-	/* t2 = source FDT size in big-endian */
-#if __riscv_xlen == 64
-	lwu	t2, 4(t0)
-#else
-	lw	t2, 4(t0)
-#endif
-	/* t3 = bit[15:8] of FDT size */
-	add	t3, t2, zero
-	srli	t3, t3, 16
-	and	t3, t3, a4
+	/* t2 = source FDT size (convert from big-endian) */
+	lbu	t2, 7(t0)
+	lbu	t3, 6(t0)
+	lbu	t4, 5(t0)
+	lbu	t5, 4(t0)
 	slli	t3, t3, 8
-	/* t4 = bit[23:16] of FDT size */
-	add	t4, t2, zero
-	srli	t4, t4, 8
-	and	t4, t4, a4
 	slli	t4, t4, 16
-	/* t5 = bit[31:24] of FDT size */
-	add	t5, t2, zero
-	and	t5, t5, a4
 	slli	t5, t5, 24
-	/* t2 = bit[7:0] of FDT size */
-	srli	t2, t2, 24
-	and	t2, t2, a4
-	/* t2 = FDT size in little-endian */
 	or	t2, t2, t3
 	or	t2, t2, t4
 	or	t2, t2, t5
@@ -415,7 +304,7 @@ _wait_for_boot_hart:
 	bne	t0, t1, _wait_for_boot_hart
 
 _start_warm:
-	/* Reset all registers for non-boot HARTs */
+	/* Reset all registers except ra, a0, a1, a2, a3 and a4 for non-boot HART */
 	li	ra, 0
 	call	_reset_regs
 
@@ -424,7 +313,7 @@ _start_warm:
 
 	/* Find HART count and HART stack size */
 	lla	a4, platform
-#if __riscv_xlen == 64
+#if __riscv_xlen > 32
 	lwu	s7, SBI_PLATFORM_HART_COUNT_OFFSET(a4)
 	lwu	s8, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(a4)
 #else
@@ -440,7 +329,7 @@ _start_warm:
 	beqz	s9, 3f
 	li	a4, 0
 1:
-#if __riscv_xlen == 64
+#if __riscv_xlen > 32
 	lwu	a5, (s9)
 #else
 	lw	a5, (s9)
@@ -469,27 +358,16 @@ _start_warm:
 
 	/* Setup trap handler */
 	lla	a4, _trap_handler
-#if __riscv_xlen == 32
 	csrr	a5, CSR_MISA
 	srli	a5, a5, ('H' - 'A')
 	andi	a5, a5, 0x1
-	beq	a5, zero, _skip_trap_handler_rv32_hyp
-	lla	a4, _trap_handler_rv32_hyp
-_skip_trap_handler_rv32_hyp:
-#endif
+	beq	a5, zero, _skip_trap_handler_hyp
+	lla	a4, _trap_handler_hyp
+_skip_trap_handler_hyp:
 	csrw	CSR_MTVEC, a4
 
-#if __riscv_xlen == 32
-	/* Override trap exit for H-extension */
-	csrr	a5, CSR_MISA
-	srli	a5, a5, ('H' - 'A')
-	andi	a5, a5, 0x1
-	beq	a5, zero, _skip_trap_exit_rv32_hyp
-	lla	a4, _trap_exit_rv32_hyp
-	csrr	a5, CSR_MSCRATCH
-	REG_S	a4, SBI_SCRATCH_TRAP_EXIT_OFFSET(a5)
-_skip_trap_exit_rv32_hyp:
-#endif
+	/* Clear MDT here again for all harts */
+	CLEAR_MDT t0
 
 	/* Initialize SBI runtime */
 	csrr	a0, CSR_MSCRATCH
@@ -500,20 +378,10 @@ _skip_trap_exit_rv32_hyp:
 
 	.data
 	.align 3
-#ifdef FW_PIC
-_runtime_offset:
-	RISCV_PTR	0
-#endif
-_relocate_lottery:
+_boot_lottery:
 	RISCV_PTR	0
 _boot_status:
 	RISCV_PTR	0
-_load_start:
-	RISCV_PTR	_fw_start
-_link_start:
-	RISCV_PTR	FW_TEXT_START
-_link_end:
-	RISCV_PTR	_fw_reloc_end
 
 	.section .entry, "ax", %progbits
 	.align 3
@@ -527,7 +395,7 @@ _hartid_to_scratch:
 	 * t2 -> Temporary
 	 */
 	lla	t2, platform
-#if __riscv_xlen == 64
+#if __riscv_xlen > 32
 	lwu	t0, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(t2)
 	lwu	t2, SBI_PLATFORM_HART_COUNT_OFFSET(t2)
 #else
@@ -611,10 +479,10 @@ memcmp:
 	xor	t0, tp, t0
 
 	/* Save original SP on exception stack */
-	REG_S	sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_REGS_SIZE)(t0)
+	REG_S	sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_CONTEXT_SIZE)(t0)
 
-	/* Set SP to exception stack and make room for trap registers */
-	add	sp, t0, -(SBI_TRAP_REGS_SIZE)
+	/* Set SP to exception stack and make room for trap context */
+	add	sp, t0, -(SBI_TRAP_CONTEXT_SIZE)
 
 	/* Restore T0 from scratch space */
 	REG_L	t0, SBI_SCRATCH_TMP0_OFFSET(tp)
@@ -674,6 +542,35 @@ memcmp:
 	REG_S	t6, SBI_TRAP_REGS_OFFSET(t6)(sp)
 .endm
 
+.macro	TRAP_SAVE_INFO have_mstatush have_h_extension
+	csrr	t0, CSR_MCAUSE
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(cause))(sp)
+	csrr	t0, CSR_MTVAL
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval))(sp)
+.if \have_h_extension
+	csrr	t0, CSR_MTVAL2
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval2))(sp)
+	csrr	t0, CSR_MTINST
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tinst))(sp)
+	.if \have_mstatush
+	csrr	t0, CSR_MSTATUSH
+	srli	t0, t0, MSTATUSH_GVA_SHIFT
+	.else
+	csrr	t0, CSR_MSTATUS
+	srli	t0, t0, MSTATUS_GVA_SHIFT
+	.endif
+	and	t0, t0, 0x1
+.else
+	REG_S	zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval2))(sp)
+	REG_S	zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tinst))(sp)
+	li	t0, 0
+.endif
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(gva))(sp)
+
+	/* We are ready to take another trap, clear MDT */
+	CLEAR_MDT t0
+.endm
+
 .macro	TRAP_CALL_C_ROUTINE
 	/* Call C routine */
 	add	a0, sp, zero
@@ -714,15 +611,18 @@ memcmp:
 .endm
 
 .macro	TRAP_RESTORE_MEPC_MSTATUS have_mstatush
-	/* Restore MEPC and MSTATUS CSRs */
-	REG_L	t0, SBI_TRAP_REGS_OFFSET(mepc)(a0)
-	csrw	CSR_MEPC, t0
-	REG_L	t0, SBI_TRAP_REGS_OFFSET(mstatus)(a0)
-	csrw	CSR_MSTATUS, t0
+	/*
+	 * Restore MSTATUS and MEPC CSRs starting with MSTATUS/H to set MDT
+	 * flags since we can not take a trap now or MEPC would be cloberred
+	 */
 	.if \have_mstatush
 	REG_L	t0, SBI_TRAP_REGS_OFFSET(mstatusH)(a0)
 	csrw	CSR_MSTATUSH, t0
 	.endif
+	REG_L	t0, SBI_TRAP_REGS_OFFSET(mstatus)(a0)
+	csrw	CSR_MSTATUS, t0
+	REG_L	t0, SBI_TRAP_REGS_OFFSET(mepc)(a0)
+	csrw	CSR_MEPC, t0
 .endm
 
 .macro TRAP_RESTORE_A0_T0
@@ -736,7 +636,6 @@ memcmp:
 	.section .entry, "ax", %progbits
 	.align 3
 	.globl _trap_handler
-	.globl _trap_exit
 _trap_handler:
 	TRAP_SAVE_AND_SETUP_SP_T0
 
@@ -744,9 +643,10 @@ _trap_handler:
 
 	TRAP_SAVE_GENERAL_REGS_EXCEPT_SP_T0
 
+	TRAP_SAVE_INFO 0 0
+
 	TRAP_CALL_C_ROUTINE
 
-_trap_exit:
 	TRAP_RESTORE_GENERAL_REGS_EXCEPT_A0_T0
 
 	TRAP_RESTORE_MEPC_MSTATUS 0
@@ -755,29 +655,39 @@ _trap_exit:
 
 	mret
 
-#if __riscv_xlen == 32
 	.section .entry, "ax", %progbits
 	.align 3
-	.globl _trap_handler_rv32_hyp
-	.globl _trap_exit_rv32_hyp
-_trap_handler_rv32_hyp:
+	.globl _trap_handler_hyp
+_trap_handler_hyp:
 	TRAP_SAVE_AND_SETUP_SP_T0
 
+#if __riscv_xlen == 32
 	TRAP_SAVE_MEPC_MSTATUS 1
+#else
+	TRAP_SAVE_MEPC_MSTATUS 0
+#endif
 
 	TRAP_SAVE_GENERAL_REGS_EXCEPT_SP_T0
 
+#if __riscv_xlen == 32
+	TRAP_SAVE_INFO 1 1
+#else
+	TRAP_SAVE_INFO 0 1
+#endif
+
 	TRAP_CALL_C_ROUTINE
 
-_trap_exit_rv32_hyp:
 	TRAP_RESTORE_GENERAL_REGS_EXCEPT_A0_T0
 
+#if __riscv_xlen == 32
 	TRAP_RESTORE_MEPC_MSTATUS 1
+#else
+	TRAP_RESTORE_MEPC_MSTATUS 0
+#endif
 
 	TRAP_RESTORE_A0_T0
 
 	mret
-#endif
 
 	.section .entry, "ax", %progbits
 	.align 3
@@ -786,7 +696,7 @@ _reset_regs:
 
 	/* flush the instruction cache */
 	fence.i
-	/* Reset all registers except ra, a0, a1 and a2 */
+	/* Reset all registers except ra, a0, a1, a2, a3 and a4 */
 	li sp, 0
 	li gp, 0
 	li tp, 0
@@ -795,8 +705,6 @@ _reset_regs:
 	li t2, 0
 	li s0, 0
 	li s1, 0
-	li a3, 0
-	li a4, 0
 	li a5, 0
 	li a6, 0
 	li a7, 0

firmware/fw_base.ldS

@@ -20,6 +20,7 @@
 		PROVIDE(_text_start = .);
 		*(.entry)
 		*(.text)
+		*(.text.*)
 		. = ALIGN(8);
 		PROVIDE(_text_end = .);
 	}
@@ -38,6 +39,11 @@
 		. = ALIGN(8);
 	}
 
+	.dynsym :
+	{
+		*(.dynsym)
+	}
+
 	. = ALIGN(0x1000); /* Ensure next section is page aligned */
 
 	.rela.dyn : {

firmware/fw_dynamic.S

@@ -11,12 +11,6 @@
 
 #include "fw_base.S"
 
-	.section .entry, "ax", %progbits
-	.align 3
-_bad_dynamic_info:
-	wfi
-	j	_bad_dynamic_info
-
 	.section .entry, "ax", %progbits
 	.align 3
 	.global fw_boot_hart
@@ -30,10 +24,10 @@ fw_boot_hart:
 	/* Sanity checks */
 	li	a1, FW_DYNAMIC_INFO_MAGIC_VALUE
 	REG_L	a0, FW_DYNAMIC_INFO_MAGIC_OFFSET(a2)
-	bne	a0, a1, _bad_dynamic_info
+	bne	a0, a1, _start_hang
 	li	a1, FW_DYNAMIC_INFO_VERSION_MAX
 	REG_L	a0, FW_DYNAMIC_INFO_VERSION_OFFSET(a2)
-	bgt	a0, a1, _bad_dynamic_info
+	bgt	a0, a1, _start_hang
 
 	/* Read boot HART id */
 	li	a1, FW_DYNAMIC_INFO_VERSION_2

firmware/fw_jump.S

@@ -46,6 +46,10 @@ fw_save_info:
 fw_next_arg1:
 #ifdef FW_JUMP_FDT_ADDR
 	li	a0, FW_JUMP_FDT_ADDR
+#elif defined(FW_JUMP_FDT_OFFSET)
+	lla	a0, _fw_start
+	li	a1, FW_JUMP_FDT_OFFSET
+	add	a0, a0, a1
 #else
 	add	a0, a1, zero
 #endif
@@ -59,8 +63,16 @@ fw_next_arg1:
 	 * The next address should be returned in 'a0'.
 	 */
 fw_next_addr:
+#ifdef FW_JUMP_ADDR
 	lla	a0, _jump_addr
 	REG_L	a0, (a0)
+#elif defined(FW_JUMP_OFFSET)
+	lla	a0, _fw_start
+	li	a1, FW_JUMP_OFFSET
+	add	a0, a0, a1
+#else
+#error "Must define at least FW_JUMP_ADDR or FW_JUMP_OFFSET"
+#endif
 	ret
 
 	.section .entry, "ax", %progbits
@@ -86,11 +98,9 @@ fw_options:
 	add	a0, zero, zero
 	ret
 
-#ifndef FW_JUMP_ADDR
-#error "Must define FW_JUMP_ADDR"
-#endif
-
+#ifdef FW_JUMP_ADDR
 	.section .rodata
 	.align 3
 _jump_addr:
 	RISCV_PTR FW_JUMP_ADDR
+#endif

firmware/fw_payload.S

@@ -46,6 +46,10 @@ fw_save_info:
 fw_next_arg1:
 #ifdef FW_PAYLOAD_FDT_ADDR
 	li	a0, FW_PAYLOAD_FDT_ADDR
+#elif defined(FW_PAYLOAD_FDT_OFFSET)
+	lla	a0, _fw_start
+	li	a1, FW_PAYLOAD_FDT_OFFSET
+	add	a0, a0, a1
 #else
 	add	a0, a1, zero
 #endif

firmware/objects.mk

@@ -13,19 +13,10 @@ firmware-cflags-y +=
 firmware-asflags-y +=
 firmware-ldflags-y +=
 
-ifndef FW_PIC
-FW_PIC := $(OPENSBI_LD_PIE)
-endif
-
-ifeq ($(FW_PIC),y)
-firmware-genflags-y +=	-DFW_PIC
-firmware-asflags-y  +=	-fpic
-firmware-cflags-y   +=	-fPIE -pie
-firmware-ldflags-y  +=	-Wl,--no-dynamic-linker -Wl,-pie
-endif
-
 ifdef FW_TEXT_START
 firmware-genflags-y += -DFW_TEXT_START=$(FW_TEXT_START)
+else
+firmware-genflags-y += -DFW_TEXT_START=0x0
 endif
 
 ifdef FW_FDT_PATH
@@ -38,9 +29,15 @@ endif
 firmware-bins-$(FW_DYNAMIC) += fw_dynamic.bin
 
 firmware-bins-$(FW_JUMP) += fw_jump.bin
+ifdef FW_JUMP_OFFSET
+firmware-genflags-$(FW_JUMP) += -DFW_JUMP_OFFSET=$(FW_JUMP_OFFSET)
+endif
 ifdef FW_JUMP_ADDR
 firmware-genflags-$(FW_JUMP) += -DFW_JUMP_ADDR=$(FW_JUMP_ADDR)
 endif
+ifdef FW_JUMP_FDT_OFFSET
+firmware-genflags-$(FW_JUMP) += -DFW_JUMP_FDT_OFFSET=$(FW_JUMP_FDT_OFFSET)
+endif
 ifdef FW_JUMP_FDT_ADDR
 firmware-genflags-$(FW_JUMP) += -DFW_JUMP_FDT_ADDR=$(FW_JUMP_FDT_ADDR)
 endif
@@ -59,6 +56,9 @@ ifdef FW_PAYLOAD_ALIGN
 firmware-genflags-$(FW_PAYLOAD) += -DFW_PAYLOAD_ALIGN=$(FW_PAYLOAD_ALIGN)
 endif
 
+ifdef FW_PAYLOAD_FDT_OFFSET
+firmware-genflags-$(FW_PAYLOAD) += -DFW_PAYLOAD_FDT_OFFSET=$(FW_PAYLOAD_FDT_OFFSET)
+endif
 ifdef FW_PAYLOAD_FDT_ADDR
 firmware-genflags-$(FW_PAYLOAD) += -DFW_PAYLOAD_FDT_ADDR=$(FW_PAYLOAD_FDT_ADDR)
 endif

include/sbi/riscv_asm.h

@@ -101,6 +101,14 @@
 		__v;                                            \
 	})
 
+/* Variant of csr_read() that allows the compiler to cache the value. */
+#define csr_read_relaxed(csr)                                     \
+	({                                                        \
+		register unsigned long __v;                       \
+		__asm__ ("csrr %0, " __ASM_STR(csr) : "=r"(__v)); \
+		__v;                                              \
+	})
+
 #define csr_write(csr, val)                                        \
 	({                                                         \
 		unsigned long __v = (unsigned long)(val);          \
@@ -163,7 +171,7 @@ void csr_write_num(int csr_num, unsigned long val);
 	} while (0)
 
 /* Get current HART id */
-#define current_hartid()	((unsigned int)csr_read(CSR_MHARTID))
+#define current_hartid()	((unsigned int)csr_read_relaxed(CSR_MHARTID))
 
 /* determine CPU extension, return non-zero support */
 int misa_extension_imp(char ext);

include/sbi/riscv_barrier.h

@@ -40,7 +40,11 @@
 #define smp_wmb()		RISCV_FENCE(w,w)
 
 /* CPU relax for busy loop */
-#define cpu_relax()		asm volatile ("" : : : "memory")
+#define cpu_relax()						\
+do {								\
+	unsigned long __t;					\
+	__asm__ __volatile__ ("div %0, %0, zero" : "=r" (__t));	\
+} while (0)
 
 /* clang-format on */
 

include/sbi/riscv_dbtr.h

@@ -0,0 +1,249 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro System, Inc.
+ *
+ * Authors:
+ *   Himanshu Chauhan <hchauhan@ventanamicro.com>
+ */
+
+#ifndef __RISCV_DBTR_H__
+#define __RISCV_DBTR_H__
+
+#define RV_MAX_TRIGGERS	32
+
+enum {
+	RISCV_DBTR_TRIG_NONE = 0,
+	RISCV_DBTR_TRIG_LEGACY,
+	RISCV_DBTR_TRIG_MCONTROL,
+	RISCV_DBTR_TRIG_ICOUNT,
+	RISCV_DBTR_TRIG_ITRIGGER,
+	RISCV_DBTR_TRIG_ETRIGGER,
+	RISCV_DBTR_TRIG_MCONTROL6,
+};
+
+#define RV_DBTR_BIT(_prefix, _name)		\
+	RV_DBTR_##_prefix##_##_name##_BIT
+
+#define RV_DBTR_BIT_MASK(_prefix, _name)	\
+	RV_DBTR_##_prefix##_name##_BIT_MASK
+
+#define RV_DBTR_DECLARE_BIT(_prefix, _name, _val)	\
+	RV_DBTR_BIT(_prefix, _name) = _val
+
+#define RV_DBTR_DECLARE_BIT_MASK(_prefix, _name, _width)		\
+	RV_DBTR_BIT_MASK(_prefix, _name) =				\
+		(((1UL << _width) - 1) << RV_DBTR_BIT(_prefix, _name))
+
+#define CLEAR_DBTR_BIT(_target, _prefix, _bit_name)		\
+	__clear_bit(RV_DBTR_BIT(_prefix, _bit_name), &_target)
+
+#define SET_DBTR_BIT(_target, _prefix, _bit_name)		\
+	__set_bit(RV_DBTR_BIT(_prefix, _bit_name), &_target)
+
+/* Trigger Data 1 */
+enum {
+	RV_DBTR_DECLARE_BIT(TDATA1, DATA,   0),
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT(TDATA1, DMODE,  59),
+	RV_DBTR_DECLARE_BIT(TDATA1, TYPE,   60),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT(TDATA1, DMODE,  27),
+	RV_DBTR_DECLARE_BIT(TDATA1, TYPE,   28),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+enum {
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, DATA,  59),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, DATA,  27),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, DMODE, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, TYPE,  4),
+};
+
+/* MC - Match Control Type Register */
+enum {
+	RV_DBTR_DECLARE_BIT(MC, LOAD,    0),
+	RV_DBTR_DECLARE_BIT(MC, STORE,   1),
+	RV_DBTR_DECLARE_BIT(MC, EXEC,    2),
+	RV_DBTR_DECLARE_BIT(MC, U,       3),
+	RV_DBTR_DECLARE_BIT(MC, S,       4),
+	RV_DBTR_DECLARE_BIT(MC, RES2,    5),
+	RV_DBTR_DECLARE_BIT(MC, M,       6),
+	RV_DBTR_DECLARE_BIT(MC, MATCH,   7),
+	RV_DBTR_DECLARE_BIT(MC, CHAIN,   11),
+	RV_DBTR_DECLARE_BIT(MC, ACTION,  12),
+	RV_DBTR_DECLARE_BIT(MC, SIZELO,  16),
+	RV_DBTR_DECLARE_BIT(MC, TIMING,  18),
+	RV_DBTR_DECLARE_BIT(MC, SELECT,  19),
+	RV_DBTR_DECLARE_BIT(MC, HIT,     20),
+#if __riscv_xlen >= 64
+	RV_DBTR_DECLARE_BIT(MC, SIZEHI,  21),
+#endif
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT(MC, MASKMAX, 53),
+	RV_DBTR_DECLARE_BIT(MC, DMODE,   59),
+	RV_DBTR_DECLARE_BIT(MC, TYPE,    60),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT(MC, MASKMAX, 21),
+	RV_DBTR_DECLARE_BIT(MC, DMODE,   27),
+	RV_DBTR_DECLARE_BIT(MC, TYPE,    28),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+enum {
+	RV_DBTR_DECLARE_BIT_MASK(MC, LOAD,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, STORE,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, EXEC,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, U,       1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, S,       1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, RES2,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, M,       1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, MATCH,   4),
+	RV_DBTR_DECLARE_BIT_MASK(MC, CHAIN,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, ACTION,  4),
+	RV_DBTR_DECLARE_BIT_MASK(MC, SIZELO,  2),
+	RV_DBTR_DECLARE_BIT_MASK(MC, TIMING,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, SELECT,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, HIT,     1),
+#if __riscv_xlen >= 64
+	RV_DBTR_DECLARE_BIT_MASK(MC, SIZEHI,  2),
+#endif
+	RV_DBTR_DECLARE_BIT_MASK(MC, MASKMAX, 6),
+	RV_DBTR_DECLARE_BIT_MASK(MC, DMODE,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, TYPE,    4),
+};
+
+/* MC6 - Match Control 6 Type Register */
+enum {
+	RV_DBTR_DECLARE_BIT(MC6, LOAD,   0),
+	RV_DBTR_DECLARE_BIT(MC6, STORE,  1),
+	RV_DBTR_DECLARE_BIT(MC6, EXEC,   2),
+	RV_DBTR_DECLARE_BIT(MC6, U,      3),
+	RV_DBTR_DECLARE_BIT(MC6, S,      4),
+	RV_DBTR_DECLARE_BIT(MC6, RES2,   5),
+	RV_DBTR_DECLARE_BIT(MC6, M,      6),
+	RV_DBTR_DECLARE_BIT(MC6, MATCH,  7),
+	RV_DBTR_DECLARE_BIT(MC6, CHAIN,  11),
+	RV_DBTR_DECLARE_BIT(MC6, ACTION, 12),
+	RV_DBTR_DECLARE_BIT(MC6, SIZE,   16),
+	RV_DBTR_DECLARE_BIT(MC6, TIMING, 20),
+	RV_DBTR_DECLARE_BIT(MC6, SELECT, 21),
+	RV_DBTR_DECLARE_BIT(MC6, HIT,    22),
+	RV_DBTR_DECLARE_BIT(MC6, VU,     23),
+	RV_DBTR_DECLARE_BIT(MC6, VS,     24),
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT(MC6, DMODE,  59),
+	RV_DBTR_DECLARE_BIT(MC6, TYPE,   60),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT(MC6, DMODE,  27),
+	RV_DBTR_DECLARE_BIT(MC6, TYPE,   28),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+enum {
+	RV_DBTR_DECLARE_BIT_MASK(MC6, LOAD,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, STORE,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, EXEC,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, U,      1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, S,      1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, RES2,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, M,      1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, MATCH,  4),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, CHAIN,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, ACTION, 4),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, SIZE,   4),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, TIMING, 1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, SELECT, 1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, HIT,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, VU,     1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, VS,     1),
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT_MASK(MC6, DMODE,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, TYPE,   4),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT_MASK(MC6, DMODE,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, TYPE,   4),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+#define RV_DBTR_SET_TDATA1_TYPE(_t1, _type)				\
+	do {								\
+		_t1 &= ~RV_DBTR_BIT_MASK(TDATA1, TYPE);			\
+		_t1 |= (((unsigned long)_type				\
+			 << RV_DBTR_BIT(TDATA1, TYPE))			\
+			& RV_DBTR_BIT_MASK(TDATA1, TYPE));		\
+	}while (0);
+
+#define RV_DBTR_SET_MC_TYPE(_t1, _type)				\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC, TYPE);		\
+		_t1 |= (((unsigned long)_type			\
+			 << RV_DBTR_BIT(MC, TYPE))		\
+			& RV_DBTR_BIT_MASK(MC, TYPE));		\
+	}while (0);
+
+#define RV_DBTR_SET_MC6_TYPE(_t1, _type)			\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC6, TYPE);		\
+		_t1 |= (((unsigned long)_type			\
+			 << RV_DBTR_BIT(MC6, TYPE))		\
+			& RV_DBTR_BIT_MASK(MC6, TYPE));		\
+	}while (0);
+
+#define RV_DBTR_SET_MC_EXEC(_t1)		\
+	SET_DBTR_BIT(_t1, MC, EXEC)
+
+#define RV_DBTR_SET_MC_LOAD(_t1)		\
+	SET_DBTR_BIT(_t1, MC, LOAD)
+
+#define RV_DBTR_SET_MC_STORE(_t1)		\
+	SET_DBTR_BIT(_t1, MC, STORE)
+
+#define RV_DBTR_SET_MC_SIZELO(_t1, _val)			\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC, SIZELO);		\
+		_t1 |= ((_val << RV_DBTR_BIT(MC, SIZELO))	\
+			& RV_DBTR_BIT_MASK(MC, SIZELO));	\
+	} while(0);
+
+#define RV_DBTR_SET_MC_SIZEHI(_t1, _val)			\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC, SIZEHI);		\
+		_t1 |= ((_val << RV_DBTR_BIT(MC, SIZEHI))	\
+			& RV_DBTR_BIT_MASK(MC, SIZEHI));	\
+	} while(0);
+
+#define RV_DBTR_SET_MC6_EXEC(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, EXEC)
+
+#define RV_DBTR_SET_MC6_LOAD(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, LOAD)
+
+#define RV_DBTR_SET_MC6_STORE(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, STORE)
+
+#define RV_DBTR_SET_MC6_SIZE(_t1, _val)				\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC6, SIZE);		\
+		_t1 |= ((_val << RV_DBTR_BIT(MC6, SIZE))	\
+			& RV_DBTR_BIT_MASK(MC6, SIZE));		\
+	} while(0);
+
+typedef unsigned long riscv_dbtr_tdata1_mcontrol_t;
+typedef unsigned long riscv_dbtr_tdata1_mcontrol6_t;
+typedef unsigned long riscv_dbtr_tdata1_t;
+
+#endif /* __RISCV_DBTR_H__ */

include/sbi/riscv_encoding.h

@@ -32,6 +32,7 @@
 #define MSTATUS_TVM			_UL(0x00100000)
 #define MSTATUS_TW			_UL(0x00200000)
 #define MSTATUS_TSR			_UL(0x00400000)
+#define MSTATUS_SPELP		_UL(0x00800000)
 #define MSTATUS32_SD			_UL(0x80000000)
 #if __riscv_xlen == 64
 #define MSTATUS_UXL			_ULL(0x0000000300000000)
@@ -41,12 +42,16 @@
 #define MSTATUS_GVA			_ULL(0x0000004000000000)
 #define MSTATUS_GVA_SHIFT		38
 #define MSTATUS_MPV			_ULL(0x0000008000000000)
+#define MSTATUS_MPELP			_ULL(0x0000020000000000)
+#define MSTATUS_MDT			_ULL(0x0000040000000000)
 #else
 #define MSTATUSH_SBE			_UL(0x00000010)
 #define MSTATUSH_MBE			_UL(0x00000020)
 #define MSTATUSH_GVA			_UL(0x00000040)
 #define MSTATUSH_GVA_SHIFT		6
 #define MSTATUSH_MPV			_UL(0x00000080)
+#define MSTATUSH_MPELP			_UL(0x00000200)
+#define MSTATUSH_MDT			_UL(0x00000400)
 #endif
 #define MSTATUS32_SD			_UL(0x80000000)
 #define MSTATUS64_SD			_ULL(0x8000000000000000)
@@ -80,6 +85,8 @@
 #define HSTATUS_GVA			_UL(0x00000040)
 #define HSTATUS_VSBE			_UL(0x00000020)
 
+#define MCAUSE_IRQ_MASK			(_UL(1) << (__riscv_xlen - 1))
+
 #define IRQ_S_SOFT			1
 #define IRQ_VS_SOFT			2
 #define IRQ_M_SOFT			3
@@ -205,10 +212,19 @@
 
 #endif
 
-#define MHPMEVENT_SSCOF_MASK		_ULL(0xFFFF000000000000)
+#define MHPMEVENT_SSCOF_MASK		_ULL(0xFF00000000000000)
 
 #define ENVCFG_STCE			(_ULL(1) << 63)
 #define ENVCFG_PBMTE			(_ULL(1) << 62)
+#define ENVCFG_ADUE_SHIFT		61
+#define ENVCFG_ADUE			(_ULL(1) << ENVCFG_ADUE_SHIFT)
+#define ENVCFG_CDE			(_ULL(1) << 60)
+#define ENVCFG_DTE_SHIFT		59
+#define ENVCFG_DTE			(_ULL(1) << ENVCFG_DTE_SHIFT)
+#define ENVCFG_PMM			(_ULL(0x3) << 32)
+#define ENVCFG_PMM_PMLEN_0		(_ULL(0x0) << 32)
+#define ENVCFG_PMM_PMLEN_7		(_ULL(0x2) << 32)
+#define ENVCFG_PMM_PMLEN_16		(_ULL(0x3) << 32)
 #define ENVCFG_CBZE			(_UL(1) << 7)
 #define ENVCFG_CBCFE			(_UL(1) << 6)
 #define ENVCFG_CBIE_SHIFT		4
@@ -216,6 +232,10 @@
 #define ENVCFG_CBIE_ILL			_UL(0x0)
 #define ENVCFG_CBIE_FLUSH		_UL(0x1)
 #define ENVCFG_CBIE_INV			_UL(0x3)
+#define ENVCFG_SSE_SHIFT		3
+#define ENVCFG_SSE			(_UL(1) << ENVCFG_SSE_SHIFT)
+#define ENVCFG_LPE_SHIFT		2
+#define ENVCFG_LPE			(_UL(1) << ENVCFG_LPE_SHIFT)
 #define ENVCFG_FIOM			_UL(0x1)
 
 /* ===== User-level CSRs ===== */
@@ -224,6 +244,7 @@
 #define CSR_USTATUS			0x000
 #define CSR_UIE				0x004
 #define CSR_UTVEC			0x005
+#define CSR_SSP				0x011
 
 /* User Trap Handling (N-extension) */
 #define CSR_USCRATCH			0x040
@@ -314,6 +335,9 @@
 /* Supervisor Configuration */
 #define CSR_SENVCFG			0x10a
 
+/* Supervisor Conter Inhibit */
+#define CSR_SCOUNTINHIBIT		0x120
+
 /* Supervisor Trap Handling */
 #define CSR_SSCRATCH			0x140
 #define CSR_SEPC			0x141
@@ -328,9 +352,14 @@
 /* Supervisor Protection and Translation */
 #define CSR_SATP			0x180
 
-/* Supervisor-Level Window to Indirectly Accessed Registers (AIA) */
+/* Supervisor Indirect Register Alias */
 #define CSR_SISELECT			0x150
 #define CSR_SIREG			0x151
+#define CSR_SIREG2          		0x152
+#define CSR_SIREG3          		0x153
+#define CSR_SIREG4          		0x155
+#define CSR_SIREG5          		0x156
+#define CSR_SIREG6          		0x157
 
 /* Supervisor-Level Interrupts (AIA) */
 #define CSR_STOPEI			0x15c
@@ -391,9 +420,14 @@
 #define CSR_HVIPRIO1			0x646
 #define CSR_HVIPRIO2			0x647
 
-/* VS-Level Window to Indirectly Accessed Registers (H-extension with AIA) */
+/* Virtual Supervisor Indirect Alias */
 #define CSR_VSISELECT			0x250
 #define CSR_VSIREG			0x251
+#define CSR_VSIREG2         		0x252
+#define CSR_VSIREG3         		0x253
+#define CSR_VSIREG4         		0x255
+#define CSR_VSIREG5         		0x256
+#define CSR_VSIREG6         		0x257
 
 /* VS-Level Interrupts (H-extension with AIA) */
 #define CSR_VSTOPEI			0x25c
@@ -686,6 +720,7 @@
 #define CSR_TDATA1			0x7a1
 #define CSR_TDATA2			0x7a2
 #define CSR_TDATA3			0x7a3
+#define CSR_TINFO			0x7a4
 
 /* Debug Mode Registers */
 #define CSR_DCSR			0x7b0
@@ -693,9 +728,14 @@
 #define CSR_DSCRATCH0			0x7b2
 #define CSR_DSCRATCH1			0x7b3
 
-/* Machine-Level Window to Indirectly Accessed Registers (AIA) */
+/* Machine Indirect Register Alias */
 #define CSR_MISELECT			0x350
 #define CSR_MIREG			0x351
+#define CSR_MIREG2          		0x352
+#define CSR_MIREG3          		0x353
+#define CSR_MIREG4          		0x355
+#define CSR_MIREG5          		0x356
+#define CSR_MIREG6          		0x357
 
 /* Machine-Level Interrupts (AIA) */
 #define CSR_MTOPEI			0x35c
@@ -723,6 +763,12 @@
 #define CSR_MVIPH			0x319
 #define CSR_MIPH			0x354
 
+/* Vector extension registers */
+#define CSR_VSTART			0x8
+#define CSR_VL				0xc20
+#define CSR_VTYPE			0xc21
+#define CSR_VLENB			0xc22
+
 /* ===== Trap/Exception Causes ===== */
 
 #define CAUSE_MISALIGNED_FETCH		0x0
@@ -740,6 +786,8 @@
 #define CAUSE_FETCH_PAGE_FAULT		0xc
 #define CAUSE_LOAD_PAGE_FAULT		0xd
 #define CAUSE_STORE_PAGE_FAULT		0xf
+#define CAUSE_DOUBLE_TRAP		0x10
+#define CAUSE_SW_CHECK_EXCP		0x12
 #define CAUSE_FETCH_GUEST_PAGE_FAULT	0x14
 #define CAUSE_LOAD_GUEST_PAGE_FAULT	0x15
 #define CAUSE_VIRTUAL_INST_FAULT	0x16
@@ -836,17 +884,377 @@
 #define INSN_MATCH_C_FSWSP		0xe002
 #define INSN_MASK_C_FSWSP		0xe003
 
+#define INSN_MATCH_C_LHU		0x8400
+#define INSN_MASK_C_LHU		0xfc43
+#define INSN_MATCH_C_LH		0x8440
+#define INSN_MASK_C_LH			0xfc43
+#define INSN_MATCH_C_SH		0x8c00
+#define INSN_MASK_C_SH			0xfc43
+
 #define INSN_MASK_WFI			0xffffff00
 #define INSN_MATCH_WFI			0x10500000
 
 #define INSN_MASK_FENCE_TSO		0xffffffff
 #define INSN_MATCH_FENCE_TSO		0x8330000f
 
+#define INSN_MASK_VECTOR_UNIT_STRIDE		0xfdf0707f
+#define INSN_MASK_VECTOR_FAULT_ONLY_FIRST	0xfdf0707f
+#define INSN_MASK_VECTOR_STRIDE			0xfc00707f
+#define INSN_MASK_VECTOR_WHOLE_REG		0xfff0707f
+#define INSN_MASK_VECTOR_INDEXED		0xfc00707f
+
+#define INSN_MATCH_VLUXSEG(n, bits) ((((n) - 1) << 29) | 0x04000007 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VSUXSEG(n, bits) ((((n) - 1) << 29) | 0x04000027 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VLOXSEG(n, bits) ((((n) - 1) << 29) | 0x0c000007 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VSOXSEG(n, bits) ((((n) - 1) << 29) | 0x0c000027 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VLSSEG(n, bits)  ((((n) - 1) << 29) | 0x08000007 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VSSSEG(n, bits)  ((((n) - 1) << 29) | 0x08000027 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VSSEG(n, bits)   ((((n) - 1) << 29) | 0x00004027 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VLSEG(n, bits)   ((((n) - 1) << 29) | 0x00004007 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+#define INSN_MATCH_VLSEGFF(n, bits) ((((n) - 1) << 29) | 0x1000007 | \
+		((bits) == 16 ? 5 : (bits) == 32 ? 6 : 7) << 12)
+
+#define INSN_MATCH_VLE16V		0x00005007
+#define INSN_MATCH_VLE32V		0x00006007
+#define INSN_MATCH_VLE64V		0x00007007
+#define INSN_MATCH_VSE16V		0x00005027
+#define INSN_MATCH_VSE32V		0x00006027
+#define INSN_MATCH_VSE64V		0x00007027
+#define INSN_MATCH_VLSE16V		0x08005007
+#define INSN_MATCH_VLSE32V		0x08006007
+#define INSN_MATCH_VLSE64V		0x08007007
+#define INSN_MATCH_VSSE16V		0x08005027
+#define INSN_MATCH_VSSE32V		0x08006027
+#define INSN_MATCH_VSSE64V		0x08007027
+#define INSN_MATCH_VLOXEI16V		0x0c005007
+#define INSN_MATCH_VLOXEI32V		0x0c006007
+#define INSN_MATCH_VLOXEI64V		0x0c007007
+#define INSN_MATCH_VSOXEI16V		0x0c005027
+#define INSN_MATCH_VSOXEI32V		0x0c006027
+#define INSN_MATCH_VSOXEI64V		0x0c007027
+#define INSN_MATCH_VLUXEI16V		0x04005007
+#define INSN_MATCH_VLUXEI32V		0x04006007
+#define INSN_MATCH_VLUXEI64V		0x04007007
+#define INSN_MATCH_VSUXEI16V		0x04005027
+#define INSN_MATCH_VSUXEI32V		0x04006027
+#define INSN_MATCH_VSUXEI64V		0x04007027
+#define INSN_MATCH_VLE16FFV		0x01005007
+#define INSN_MATCH_VLE32FFV		0x01006007
+#define INSN_MATCH_VLE64FFV		0x01007007
+#define INSN_MATCH_VL1RE8V		0x02800007
+#define INSN_MATCH_VL1RE16V		0x02805007
+#define INSN_MATCH_VL1RE32V		0x02806007
+#define INSN_MATCH_VL1RE64V		0x02807007
+#define INSN_MATCH_VL2RE8V		0x22800007
+#define INSN_MATCH_VL2RE16V		0x22805007
+#define INSN_MATCH_VL2RE32V		0x22806007
+#define INSN_MATCH_VL2RE64V		0x22807007
+#define INSN_MATCH_VL4RE8V		0x62800007
+#define INSN_MATCH_VL4RE16V		0x62805007
+#define INSN_MATCH_VL4RE32V		0x62806007
+#define INSN_MATCH_VL4RE64V		0x62807007
+#define INSN_MATCH_VL8RE8V		0xe2800007
+#define INSN_MATCH_VL8RE16V		0xe2805007
+#define INSN_MATCH_VL8RE32V		0xe2806007
+#define INSN_MATCH_VL8RE64V		0xe2807007
+#define INSN_MATCH_VS1RV		0x02800027
+#define INSN_MATCH_VS2RV		0x22800027
+#define INSN_MATCH_VS4RV		0x62800027
+#define INSN_MATCH_VS8RV		0xe2800027
+
+#define INSN_MASK_VECTOR_LOAD_STORE	0x7f
+#define INSN_MATCH_VECTOR_LOAD		0x07
+#define INSN_MATCH_VECTOR_STORE		0x27
+
+#define IS_VECTOR_LOAD_STORE(insn) \
+	((((insn) & INSN_MASK_VECTOR_LOAD_STORE) == INSN_MATCH_VECTOR_LOAD) || \
+	(((insn) & INSN_MASK_VECTOR_LOAD_STORE) == INSN_MATCH_VECTOR_STORE))
+
+#define IS_VECTOR_INSN_MATCH(insn, match, mask) \
+	(((insn) & (mask)) == ((match) & (mask)))
+
+#define IS_UNIT_STRIDE_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_UNIT_STRIDE)
+
+#define IS_STRIDE_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_STRIDE)
+
+#define IS_INDEXED_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_INDEXED)
+
+#define IS_FAULT_ONLY_FIRST_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_FAULT_ONLY_FIRST)
+
+#define IS_WHOLE_REG_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_WHOLE_REG)
+
+#define IS_UNIT_STRIDE_LOAD(insn) ( \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLE16V) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLE32V) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLE64V) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(2, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(3, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(4, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(5, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(6, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(7, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(8, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(2, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(3, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(4, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(5, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(6, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(7, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(8, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(2, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(3, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(4, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(5, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(6, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(7, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VLSEG(8, 64)))
+
+#define IS_UNIT_STRIDE_STORE(insn) ( \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSE16V) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSE32V) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSE64V) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(2, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(3, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(4, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(5, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(6, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(7, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(8, 16)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(2, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(3, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(4, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(5, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(6, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(7, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(8, 32)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(2, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(3, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(4, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(5, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(6, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(7, 64)) || \
+	IS_UNIT_STRIDE_MATCH(insn, INSN_MATCH_VSSEG(8, 64)))
+
+#define IS_STRIDE_LOAD(insn) ( \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSE16V) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSE32V) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSE64V) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(2, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(3, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(4, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(5, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(6, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(7, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(8, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(2, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(3, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(4, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(5, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(6, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(7, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(8, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(2, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(3, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(4, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(5, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(6, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(7, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VLSSEG(8, 64)))
+
+#define IS_STRIDE_STORE(insn) ( \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSE16V) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSE32V) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSE64V) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(2, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(3, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(4, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(5, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(6, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(7, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(8, 16)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(2, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(3, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(4, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(5, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(6, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(7, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(8, 32)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(2, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(3, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(4, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(5, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(6, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(7, 64)) || \
+	IS_STRIDE_MATCH(insn, INSN_MATCH_VSSSEG(8, 64)))
+
+#define IS_INDEXED_LOAD(insn) ( \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXEI16V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXEI32V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXEI64V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXEI16V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXEI32V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXEI64V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLUXSEG(8, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VLOXSEG(8, 64)))
+
+#define IS_INDEXED_STORE(insn) ( \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXEI16V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXEI32V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXEI64V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXEI16V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXEI32V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXEI64V) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSUXSEG(8, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(insn, INSN_MATCH_VSOXSEG(8, 64)))
+
+#define IS_FAULT_ONLY_FIRST_LOAD(insn) ( \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLE16FFV) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLE32FFV) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLE64FFV) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(2, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(3, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(4, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(5, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(6, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(7, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(8, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(2, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(3, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(4, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(5, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(6, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(7, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(8, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(2, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(3, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(4, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(5, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(6, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(7, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(insn, INSN_MATCH_VLSEGFF(8, 64)))
+
+	#define IS_WHOLE_REG_LOAD(insn) ( \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL1RE8V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL1RE16V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL1RE32V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL1RE64V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL2RE8V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL2RE16V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL2RE32V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL2RE64V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL4RE8V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL4RE16V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL4RE32V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL4RE64V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL8RE8V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL8RE16V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL8RE32V) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VL8RE64V))
+
+#define IS_WHOLE_REG_STORE(insn) ( \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VS1RV) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VS2RV) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VS4RV) || \
+	IS_WHOLE_REG_MATCH(insn, INSN_MATCH_VS8RV))
+
+
 #if __riscv_xlen == 64
 
 /* 64-bit read for VS-stage address translation (RV64) */
 #define INSN_PSEUDO_VS_LOAD		0x00003000
-
 /* 64-bit write for VS-stage address translation (RV64) */
 #define INSN_PSEUDO_VS_STORE	0x00003020
 
@@ -862,6 +1270,12 @@
 #error "Unexpected __riscv_xlen"
 #endif
 
+#define VM_MASK				0x1
+#define VIEW_MASK				0x3
+#define VSEW_MASK				0x3
+#define VLMUL_MASK				0x7
+#define VD_MASK				0x1f
+#define VS2_MASK				0x1f
 #define INSN_16BIT_MASK			0x3
 #define INSN_32BIT_MASK			0x1c
 
@@ -880,6 +1294,12 @@
 #endif
 #define REGBYTES			(1 << LOG_REGBYTES)
 
+#define SH_VSEW			3
+#define SH_VIEW			12
+#define SH_VD				7
+#define SH_VS2				20
+#define SH_VM				25
+#define SH_MEW				28
 #define SH_RD				7
 #define SH_RS1				15
 #define SH_RS2				20
@@ -917,7 +1337,10 @@
 #define REG_PTR(insn, pos, regs)	\
 	(ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
 
-#define GET_RM(insn)			(((insn) >> 12) & 7)
+#define GET_RM(insn)			((insn & MASK_FUNCT3) >> SHIFT_FUNCT3)
+
+#define GET_RS1_NUM(insn)		((insn & MASK_RS1) >> 15)
+#define GET_CSR_NUM(insn)		((insn & MASK_CSR) >> SHIFT_CSR)
 
 #define GET_RS1(insn, regs)		(*REG_PTR(insn, SH_RS1, regs))
 #define GET_RS2(insn, regs)		(*REG_PTR(insn, SH_RS2, regs))
@@ -929,7 +1352,32 @@
 #define IMM_I(insn)			((s32)(insn) >> 20)
 #define IMM_S(insn)			(((s32)(insn) >> 25 << 5) | \
 					 (s32)(((insn) >> 7) & 0x1f))
+
+#define IS_MASKED(insn)		(((insn >> SH_VM) & VM_MASK) == 0)
+#define GET_VD(insn)			((insn >> SH_VD) & VD_MASK)
+#define GET_VS2(insn)			((insn >> SH_VS2) & VS2_MASK)
+#define GET_VIEW(insn)			(((insn) >> SH_VIEW) & VIEW_MASK)
+#define GET_MEW(insn)			(((insn) >> SH_MEW) & 1)
+#define GET_VSEW(vtype)		(((vtype) >> SH_VSEW) & VSEW_MASK)
+#define GET_VLMUL(vtype)		((vtype) & VLMUL_MASK)
+#define GET_LEN(view)			(1UL << (view))
+#define GET_NF(insn)			(1 + ((insn >> 29) & 7))
+#define GET_VEMUL(vlmul, view, vsew)	((vlmul + view - vsew) & 7)
+#define GET_EMUL(vemul)		(1UL << ((vemul) >= 4 ? 0 : (vemul)))
+
 #define MASK_FUNCT3			0x7000
+#define MASK_RS1			0xf8000
+#define MASK_CSR			0xfff00000
+
+#define SHIFT_FUNCT3			12
+#define SHIFT_CSR			20
+
+#define CSRRW 1
+#define CSRRS 2
+#define CSRRC 3
+#define CSRRWI 5
+#define CSRRSI 6
+#define CSRRCI 7
 
 /* clang-format on */
 

include/sbi/riscv_fp.h

@@ -15,7 +15,6 @@
 #include <sbi/sbi_types.h>
 
 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
-#define GET_RM(insn) (((insn) >> 12) & 7)
 #define PRECISION_S 0
 #define PRECISION_D 1
 

include/sbi/sbi_bitmap.h

@@ -62,6 +62,11 @@ static inline void bitmap_zero(unsigned long *dst, int nbits)
 	}
 }
 
+static inline int bitmap_test(unsigned long *bmap, int bit)
+{
+	return __test_bit(bit, bmap);
+}
+
 static inline void bitmap_zero_except(unsigned long *dst,
 				      int exception, int nbits)
 {

include/sbi/sbi_bitops.h

@@ -14,6 +14,8 @@
 
 #define BITS_PER_LONG		(8 * __SIZEOF_LONG__)
 
+#define BITS_PER_LONG_LONG	64
+
 #define EXTRACT_FIELD(val, which) \
 	(((val) & (which)) / ((which) & ~((which)-1)))
 #define INSERT_FIELD(val, which, fieldval) \
@@ -28,9 +30,13 @@
 #define BIT_WORD_OFFSET(bit)	((bit) & (BITS_PER_LONG - 1))
 #define BIT_ALIGN(bit, align)	(((bit) + ((align) - 1)) & ~((align) - 1))
 
+#define BIT_ULL(nr)		(1ULL << (nr))
+
 #define GENMASK(h, l) \
 	(((~0UL) - (1UL << (l)) + 1) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) - (1ULL << (l)) + 1) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 /**
  * sbi_ffs - find first (less-significant) set bit in a long word.
  * @word: The word to search

include/sbi/sbi_byteorder.h

@@ -7,7 +7,12 @@
 #ifndef __SBI_BYTEORDER_H__
 #define __SBI_BYTEORDER_H__
 
+#ifdef __ASSEMBLER__
+# define _conv_cast(type, val) (val)
+#else
 # include <sbi/sbi_types.h>
+# define _conv_cast(type, val) ((type)(val))
+#endif
 
 #define BSWAP16(x)	((((x) & 0x00ff) << 8) | \
 			 (((x) & 0xff00) >> 8))
@@ -25,37 +30,47 @@
 			 (((x) & 0xff00000000000000ULL) >> 56))
 
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__  /* CPU(little-endian) */
-#define cpu_to_be16(x)		((uint16_t)BSWAP16(x))
-#define cpu_to_be32(x)		((uint32_t)BSWAP32(x))
-#define cpu_to_be64(x)		((uint64_t)BSWAP64(x))
+#define cpu_to_be16(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define cpu_to_be32(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define cpu_to_be64(x)		_conv_cast(uint64_t, BSWAP64(x))
 
-#define be16_to_cpu(x)		((uint16_t)BSWAP16(x))
-#define be32_to_cpu(x)		((uint32_t)BSWAP32(x))
-#define be64_to_cpu(x)		((uint64_t)BSWAP64(x))
+#define be16_to_cpu(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define be32_to_cpu(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define be64_to_cpu(x)		_conv_cast(uint64_t, BSWAP64(x))
 
-#define cpu_to_le16(x)		((uint16_t)(x))
-#define cpu_to_le32(x)		((uint32_t)(x))
-#define cpu_to_le64(x)		((uint64_t)(x))
+#define cpu_to_le16(x)		_conv_cast(uint16_t, (x))
+#define cpu_to_le32(x)		_conv_cast(uint32_t, (x))
+#define cpu_to_le64(x)		_conv_cast(uint64_t, (x))
 
-#define le16_to_cpu(x)		((uint16_t)(x))
-#define le32_to_cpu(x)		((uint32_t)(x))
-#define le64_to_cpu(x)		((uint64_t)(x))
+#define le16_to_cpu(x)		_conv_cast(uint16_t, (x))
+#define le32_to_cpu(x)		_conv_cast(uint32_t, (x))
+#define le64_to_cpu(x)		_conv_cast(uint64_t, (x))
 #else /* CPU(big-endian) */
-#define cpu_to_be16(x)		((uint16_t)(x))
-#define cpu_to_be32(x)		((uint32_t)(x))
-#define cpu_to_be64(x)		((uint64_t)(x))
+#define cpu_to_be16(x)		_conv_cast(uint16_t, (x))
+#define cpu_to_be32(x)		_conv_cast(uint32_t, (x))
+#define cpu_to_be64(x)		_conv_cast(uint64_t, (x))
 
-#define be16_to_cpu(x)		((uint16_t)(x))
-#define be32_to_cpu(x)		((uint32_t)(x))
-#define be64_to_cpu(x)		((uint64_t)(x))
+#define be16_to_cpu(x)		_conv_cast(uint16_t, (x))
+#define be32_to_cpu(x)		_conv_cast(uint32_t, (x))
+#define be64_to_cpu(x)		_conv_cast(uint64_t, (x))
 
-#define cpu_to_le16(x)		((uint16_t)BSWAP16(x))
-#define cpu_to_le32(x)		((uint32_t)BSWAP32(x))
-#define cpu_to_le64(x)		((uint64_t)BSWAP64(x))
+#define cpu_to_le16(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define cpu_to_le32(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define cpu_to_le64(x)		_conv_cast(uint64_t, BSWAP64(x))
 
-#define le16_to_cpu(x)		((uint16_t)BSWAP16(x))
-#define le32_to_cpu(x)		((uint32_t)BSWAP32(x))
-#define le64_to_cpu(x)		((uint64_t)BSWAP64(x))
+#define le16_to_cpu(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define le32_to_cpu(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define le64_to_cpu(x)		_conv_cast(uint64_t, BSWAP64(x))
+#endif
+
+#if __riscv_xlen == 64
+#define cpu_to_lle		cpu_to_le64
+#define lle_to_cpu		le64_to_cpu
+#elif __riscv_xlen == 32
+#define cpu_to_lle		cpu_to_le32
+#define lle_to_cpu		le32_to_cpu
+#else
+#error "Unknown __riscv_xlen"
 #endif
 
 #endif /* __SBI_BYTEORDER_H__ */

include/sbi/sbi_console.h

@@ -58,8 +58,6 @@ void sbi_console_set_device(const struct sbi_console_device *dev);
 
 struct sbi_scratch;
 
-int sbi_console_init(struct sbi_scratch *scratch);
-
 #define SBI_ASSERT(cond, args) do { \
 	if (unlikely(!(cond))) \
 		sbi_panic args; \

include/sbi/sbi_dbtr.h

@@ -0,0 +1,125 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems, Inc.
+ *
+ * Authors:
+ *   Himanshu Chauhan <hchauhan@ventanamicro.com>
+ */
+
+#ifndef __SBI_DBTR_H__
+#define __SBI_DBTR_H__
+
+#include <sbi/riscv_dbtr.h>
+#include <sbi/sbi_types.h>
+
+struct sbi_domain;
+
+enum {
+	RV_DBTR_DECLARE_BIT(TS, MAPPED, 0), /* trigger mapped to hw trigger */
+	RV_DBTR_DECLARE_BIT(TS, U, 1),
+	RV_DBTR_DECLARE_BIT(TS, S, 2),
+	RV_DBTR_DECLARE_BIT(TS, VU, 3),
+	RV_DBTR_DECLARE_BIT(TS, VS, 4),
+	RV_DBTR_DECLARE_BIT(TS, HAVE_TRIG, 5), /* H/w dbtr details available */
+	RV_DBTR_DECLARE_BIT(TS, HW_IDX, 8), /* Hardware index of trigger */
+};
+
+enum {
+	RV_DBTR_DECLARE_BIT_MASK(TS, MAPPED, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, U, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, S, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, VU, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, VS, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, HAVE_TRIG, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, HW_IDX, (__riscv_xlen-9)),
+};
+
+#if __riscv_xlen == 64
+#define SBI_DBTR_SHMEM_INVALID_ADDR	0xFFFFFFFFFFFFFFFFUL
+#elif __riscv_xlen == 32
+#define SBI_DBTR_SHMEM_INVALID_ADDR	0xFFFFFFFFUL
+#else
+#error "Unexpected __riscv_xlen"
+#endif
+
+struct sbi_dbtr_shmem {
+	unsigned long phys_lo;
+	unsigned long phys_hi;
+};
+
+struct sbi_dbtr_trigger {
+	unsigned long index;
+	unsigned long type_mask;
+	unsigned long state;
+	unsigned long tdata1;
+	unsigned long tdata2;
+	unsigned long tdata3;
+};
+
+struct sbi_dbtr_data_msg {
+	unsigned long tstate;
+	unsigned long tdata1;
+	unsigned long tdata2;
+	unsigned long tdata3;
+};
+
+struct sbi_dbtr_id_msg {
+	unsigned long idx;
+};
+
+struct sbi_dbtr_hart_triggers_state {
+	struct sbi_dbtr_trigger triggers[RV_MAX_TRIGGERS];
+	struct sbi_dbtr_shmem shmem;
+	u32 total_trigs;
+	u32 available_trigs;
+	u32 hartid;
+	u32 probed;
+};
+
+#define TDATA1_GET_TYPE(_t1)					\
+	EXTRACT_FIELD(_t1, RV_DBTR_BIT_MASK(TDATA1, TYPE))
+
+/* Set the hardware index of trigger in logical trigger state */
+#define SET_TRIG_HW_INDEX(_state, _idx)				\
+	do {							\
+		_state &= ~RV_DBTR_BIT_MASK(TS, HW_IDX);	\
+		_state |= (((unsigned long)_idx			\
+			    << RV_DBTR_BIT(TS, HW_IDX))		\
+			   & RV_DBTR_BIT_MASK(TS, HW_IDX));	\
+	}while (0);
+
+/** SBI shared mem messages layout */
+struct sbi_dbtr_shmem_entry {
+	struct sbi_dbtr_data_msg data;
+	struct sbi_dbtr_id_msg id;
+};
+
+#define SBI_DBTR_SHMEM_ALIGN_MASK	((__riscv_xlen / 8) - 1)
+
+/** Initialize debug triggers */
+int sbi_dbtr_init(struct sbi_scratch *scratch, bool coldboot);
+
+/** SBI DBTR extension functions */
+int sbi_dbtr_supported(void);
+int sbi_dbtr_setup_shmem(const struct sbi_domain *dom, unsigned long smode,
+			 unsigned long shmem_phys_lo,
+			 unsigned long shmem_phys_hi);
+int sbi_dbtr_num_trig(unsigned long trig_tdata1, unsigned long *out);
+int sbi_dbtr_read_trig(unsigned long smode,
+		       unsigned long trig_idx_base, unsigned long trig_count);
+int sbi_dbtr_install_trig(unsigned long smode,
+			  unsigned long trig_count, unsigned long *out);
+int sbi_dbtr_uninstall_trig(unsigned long trig_idx_base,
+			    unsigned long trig_idx_mask);
+int sbi_dbtr_enable_trig(unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask);
+int sbi_dbtr_update_trig(unsigned long smode,
+			 unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask);
+int sbi_dbtr_disable_trig(unsigned long trig_idx_base,
+			  unsigned long trig_idx_mask);
+
+int sbi_dbtr_get_total_triggers(void);
+
+#endif

include/sbi/sbi_domain.h

@@ -10,8 +10,12 @@
 #ifndef __SBI_DOMAIN_H__
 #define __SBI_DOMAIN_H__
 
+#include <sbi/riscv_locks.h>
+#include <sbi/sbi_list.h>
 #include <sbi/sbi_types.h>
 #include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_domain_context.h>
+#include <sbi/sbi_domain_data.h>
 
 struct sbi_scratch;
 
@@ -156,22 +160,18 @@ struct sbi_domain_memregion {
 	unsigned long flags;
 };
 
-/** Maximum number of domains */
-#define SBI_DOMAIN_MAX_INDEX			32
-
 /** Representation of OpenSBI domain */
 struct sbi_domain {
-	/**
-	 * Logical index of this domain
-	 * Note: This set by sbi_domain_finalize() in the coldboot path
-	 */
+	/** Node in linked list of domains */
+	struct sbi_dlist node;
+	/** Internal state of per-domain data */
+	struct sbi_domain_data_priv data_priv;
+	/** Logical index of this domain */
 	u32 index;
-	/**
-	 * HARTs assigned to this domain
-	 * Note: This set by sbi_domain_init() and sbi_domain_finalize()
-	 * in the coldboot path
-	 */
+	/** HARTs assigned to this domain */
 	struct sbi_hartmask assigned_harts;
+	/** Spinlock for accessing assigned_harts */
+	spinlock_t assigned_harts_lock;
 	/** Name of this domain */
 	char name[64];
 	/** Possible HARTs in this domain */
@@ -200,20 +200,19 @@ extern struct sbi_domain root;
 /** Get pointer to sbi_domain from HART index */
 struct sbi_domain *sbi_hartindex_to_domain(u32 hartindex);
 
+/** Update HART local pointer to point to specified domain */
+void sbi_update_hartindex_to_domain(u32 hartindex, struct sbi_domain *dom);
+
 /** Get pointer to sbi_domain for current HART */
 #define sbi_domain_thishart_ptr() \
-	sbi_hartindex_to_domain(sbi_hartid_to_hartindex(current_hartid()))
+	sbi_hartindex_to_domain(current_hartindex())
 
-/** Index to domain table */
-extern struct sbi_domain *domidx_to_domain_table[];
-
-/** Get pointer to sbi_domain from index */
-#define sbi_index_to_domain(__index) \
-	domidx_to_domain_table[__index]
+/** Head of linked list of domains */
+extern struct sbi_dlist domain_list;
 
 /** Iterate over each domain */
-#define sbi_domain_for_each(__i, __d) \
-	for ((__i) = 0; ((__d) = sbi_index_to_domain(__i)); (__i)++)
+#define sbi_domain_for_each(__d) \
+	sbi_list_for_each_entry(__d, &domain_list, node)
 
 /** Iterate over each memory region of a domain */
 #define sbi_domain_for_each_memregion(__d, __r) \
@@ -222,20 +221,19 @@ extern struct sbi_domain *domidx_to_domain_table[];
 /**
  * Check whether given HART is assigned to specified domain
  * @param dom pointer to domain
- * @param hartid the HART ID
+ * @param hartindex the HART index
  * @return true if HART is assigned to domain otherwise false
  */
-bool sbi_domain_is_assigned_hart(const struct sbi_domain *dom, u32 hartid);
+bool sbi_domain_is_assigned_hart(const struct sbi_domain *dom, u32 hartindex);
 
 /**
- * Get ulong assigned HART mask for given domain and HART base ID
+ * Get the assigned HART mask for given domain
  * @param dom pointer to domain
- * @param hbase the HART base ID
- * @return ulong possible HART mask
- * Note: the return ulong mask will be set to zero on failure.
+ * @param mask the output hartmask to fill
+ * @return 0 on success and SBI_Exxx (< 0) on failure
  */
-ulong sbi_domain_get_assigned_hartmask(const struct sbi_domain *dom,
-				       ulong hbase);
+int sbi_domain_get_assigned_hartmask(const struct sbi_domain *dom,
+				     struct sbi_hartmask *mask);
 
 /**
  * Initialize a domain memory region based on it's physical
@@ -295,16 +293,6 @@ void sbi_domain_dump_all(const char *suffix);
 int sbi_domain_register(struct sbi_domain *dom,
 			const struct sbi_hartmask *assign_mask);
 
-/**
- * Add a memory region to the root domain
- * @param reg pointer to the memory region to be added
- *
- * @return 0 on success
- * @return SBI_EALREADY if memory region conflicts with the existing one
- * @return SBI_EINVAL otherwise
- */
-int sbi_domain_root_add_memregion(const struct sbi_domain_memregion *reg);
-
 /**
  * Add a memory range with its flags to the root domain
  * @param addr start physical address of memory range

include/sbi/sbi_domain_context.h

@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) IPADS@SJTU 2023. All rights reserved.
+ */
+
+#ifndef __SBI_DOMAIN_CONTEXT_H__
+#define __SBI_DOMAIN_CONTEXT_H__
+
+#include <sbi/sbi_types.h>
+
+struct sbi_domain;
+
+/**
+ * Enter a specific domain context synchronously
+ * @param dom pointer to domain
+ *
+ * @return 0 on success and negative error code on failure
+ */
+int sbi_domain_context_enter(struct sbi_domain *dom);
+
+/**
+ * Exit the current domain context, and then return to the caller
+ * of sbi_domain_context_enter or attempt to start the next domain
+ * context to be initialized
+ *
+ * @return 0 on success and negative error code on failure
+ */
+int sbi_domain_context_exit(void);
+
+/**
+ * Initialize domain context support
+ *
+ * @return 0 on success and negative error code on failure
+ */
+int sbi_domain_context_init(void);
+
+/* Deinitialize domain context support */
+void sbi_domain_context_deinit(void);
+
+#endif // __SBI_DOMAIN_CONTEXT_H__

include/sbi/sbi_domain_data.h

@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ */
+
+#ifndef __SBI_DOMAIN_DATA_H__
+#define __SBI_DOMAIN_DATA_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_list.h>
+
+struct sbi_domain;
+
+/** Maximum domain data per-domain */
+#define SBI_DOMAIN_MAX_DATA_PTRS		32
+
+/** Representation of per-domain data */
+struct sbi_domain_data_priv {
+	/** Array of domain data pointers indexed by domain data identifier */
+	void *idx_to_data_ptr[SBI_DOMAIN_MAX_DATA_PTRS];
+};
+
+/** Representation of a domain data */
+struct sbi_domain_data {
+	/**
+	 * Head is used for maintaining data list
+	 *
+	 * Note: initialized by domain framework
+	 */
+	struct sbi_dlist head;
+	/**
+	 * Identifier which used to locate per-domain data
+	 *
+	 * Note: initialized by domain framework
+	 */
+	unsigned long data_idx;
+	/** Size of per-domain data */
+	unsigned long data_size;
+	/** Optional callback to setup domain data */
+	int (*data_setup)(struct sbi_domain *dom,
+			  struct sbi_domain_data *data, void *data_ptr);
+	/** Optional callback to cleanup domain data */
+	void (*data_cleanup)(struct sbi_domain *dom,
+			     struct sbi_domain_data *data, void *data_ptr);
+};
+
+/**
+ * Get per-domain data pointer for a given domain
+ * @param dom pointer to domain
+ * @param data pointer to domain data
+ *
+ * @return per-domain data pointer
+ */
+void *sbi_domain_data_ptr(struct sbi_domain *dom, struct sbi_domain_data *data);
+
+/**
+ * Setup all domain data for a domain
+ * @param dom pointer to domain
+ *
+ * @return 0 on success and negative error code on failure
+ *
+ * Note: This function is used internally within domain framework.
+ */
+int sbi_domain_setup_data(struct sbi_domain *dom);
+
+/**
+ * Cleanup all domain data for a domain
+ * @param dom pointer to domain
+ *
+ * Note: This function is used internally within domain framework.
+ */
+void sbi_domain_cleanup_data(struct sbi_domain *dom);
+
+/**
+ * Register a domain data
+ * @param hndl pointer to domain data
+ *
+ * @return 0 on success and negative error code on failure
+ *
+ * Note: This function must be used only in cold boot path.
+ */
+int sbi_domain_register_data(struct sbi_domain_data *data);
+
+/**
+ * Unregister a domain data
+ * @param hndl pointer to domain data
+ *
+ * Note: This function must be used only in cold boot path.
+ */
+void sbi_domain_unregister_data(struct sbi_domain_data *data);
+
+#endif

include/sbi/sbi_ecall.h

@@ -18,7 +18,7 @@
 #define SBI_OPENSBI_IMPID		1
 
 struct sbi_trap_regs;
-struct sbi_trap_info;
+struct sbi_trap_context;
 
 struct sbi_ecall_return {
 	/* Return flag to skip register update */
@@ -30,6 +30,8 @@ struct sbi_ecall_return {
 struct sbi_ecall_extension {
 	/* head is used by the extension list */
 	struct sbi_dlist head;
+	/* short name of the extension */
+	char name[8];
 	/*
 	 * extid_start and extid_end specify the range for this extension. As
 	 * the initial range may be wider than the valid runtime range, the
@@ -38,6 +40,8 @@ struct sbi_ecall_extension {
 	 */
 	unsigned long extid_start;
 	unsigned long extid_end;
+	/* flag showing whether given extension is experimental or not */
+	bool experimental;
 	/*
 	 * register_extensions
 	 *
@@ -83,11 +87,13 @@ void sbi_ecall_set_impid(unsigned long impid);
 
 struct sbi_ecall_extension *sbi_ecall_find_extension(unsigned long extid);
 
+void sbi_ecall_get_extensions_str(char *exts_str, int exts_str_size, bool experimental);
+
 int sbi_ecall_register_extension(struct sbi_ecall_extension *ext);
 
 void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext);
 
-int sbi_ecall_handler(struct sbi_trap_regs *regs);
+int sbi_ecall_handler(struct sbi_trap_context *tcntx);
 
 int sbi_ecall_init(void);
 

include/sbi/sbi_ecall_interface.h

@@ -12,6 +12,8 @@
 
 /* clang-format off */
 
+#include <sbi/sbi_types.h>
+
 /* SBI Extension IDs */
 #define SBI_EXT_0_1_SET_TIMER			0x0
 #define SBI_EXT_0_1_CONSOLE_PUTCHAR		0x1
@@ -32,6 +34,10 @@
 #define SBI_EXT_DBCN				0x4442434E
 #define SBI_EXT_SUSP				0x53555350
 #define SBI_EXT_CPPC				0x43505043
+#define SBI_EXT_DBTR				0x44425452
+#define SBI_EXT_SSE				0x535345
+#define SBI_EXT_FWFT				0x46574654
+#define SBI_EXT_MPXY				0x4D505859
 
 /* SBI function IDs for BASE extension*/
 #define SBI_EXT_BASE_GET_SPEC_VERSION		0x0
@@ -104,6 +110,44 @@
 #define SBI_EXT_PMU_COUNTER_FW_READ	0x5
 #define SBI_EXT_PMU_COUNTER_FW_READ_HI	0x6
 #define SBI_EXT_PMU_SNAPSHOT_SET_SHMEM	0x7
+#define SBI_EXT_PMU_EVENT_GET_INFO		0x8
+
+/* SBI function IDs for DBTR extension */
+#define SBI_EXT_DBTR_NUM_TRIGGERS	0x0
+#define SBI_EXT_DBTR_SETUP_SHMEM	0x1
+#define SBI_EXT_DBTR_TRIGGER_READ	0x2
+#define SBI_EXT_DBTR_TRIGGER_INSTALL	0x3
+#define SBI_EXT_DBTR_TRIGGER_UPDATE	0x4
+#define SBI_EXT_DBTR_TRIGGER_UNINSTALL	0x5
+#define SBI_EXT_DBTR_TRIGGER_ENABLE	0x6
+#define SBI_EXT_DBTR_TRIGGER_DISABLE	0x7
+
+/* SBI function IDs for FW feature extension */
+#define SBI_EXT_FWFT_SET		0x0
+#define SBI_EXT_FWFT_GET		0x1
+
+enum sbi_fwft_feature_t {
+	SBI_FWFT_MISALIGNED_EXC_DELEG		= 0x0,
+	SBI_FWFT_LANDING_PAD			= 0x1,
+	SBI_FWFT_SHADOW_STACK			= 0x2,
+	SBI_FWFT_DOUBLE_TRAP			= 0x3,
+	SBI_FWFT_PTE_AD_HW_UPDATING		= 0x4,
+	SBI_FWFT_POINTER_MASKING_PMLEN		= 0x5,
+	SBI_FWFT_LOCAL_RESERVED_START		= 0x6,
+	SBI_FWFT_LOCAL_RESERVED_END		= 0x3fffffff,
+	SBI_FWFT_LOCAL_PLATFORM_START		= 0x40000000,
+	SBI_FWFT_LOCAL_PLATFORM_END		= 0x7fffffff,
+
+	SBI_FWFT_GLOBAL_RESERVED_START		= 0x80000000,
+	SBI_FWFT_GLOBAL_RESERVED_END		= 0xbfffffff,
+	SBI_FWFT_GLOBAL_PLATFORM_START		= 0xc0000000,
+	SBI_FWFT_GLOBAL_PLATFORM_END		= 0xffffffff,
+};
+
+#define SBI_FWFT_GLOBAL_FEATURE_BIT		(1 << 31)
+#define SBI_FWFT_PLATFORM_FEATURE_BIT		(1 << 30)
+
+#define SBI_FWFT_SET_FLAG_LOCK			(1 << 0)
 
 /** General pmu event codes specified in SBI PMU extension */
 enum sbi_pmu_hw_generic_events_t {
@@ -205,6 +249,7 @@ enum sbi_pmu_event_type_id {
 	SBI_PMU_EVENT_TYPE_HW				= 0x0,
 	SBI_PMU_EVENT_TYPE_HW_CACHE			= 0x1,
 	SBI_PMU_EVENT_TYPE_HW_RAW			= 0x2,
+	SBI_PMU_EVENT_TYPE_HW_RAW_V2			= 0x3,
 	SBI_PMU_EVENT_TYPE_FW				= 0xf,
 	SBI_PMU_EVENT_TYPE_MAX,
 };
@@ -215,12 +260,19 @@ enum sbi_pmu_ctr_type {
 	SBI_PMU_CTR_TYPE_FW,
 };
 
+struct sbi_pmu_event_info {
+	uint32_t event_idx;
+	uint32_t output;
+	uint64_t event_data;
+};
+
 /* Helper macros to decode event idx */
 #define SBI_PMU_EVENT_IDX_MASK 0xFFFFF
 #define SBI_PMU_EVENT_IDX_TYPE_OFFSET 16
 #define SBI_PMU_EVENT_IDX_TYPE_MASK (0xF << SBI_PMU_EVENT_IDX_TYPE_OFFSET)
 #define SBI_PMU_EVENT_IDX_CODE_MASK 0xFFFF
 #define SBI_PMU_EVENT_RAW_IDX 0x20000
+#define SBI_PMU_EVENT_RAW_V2_IDX 0x30000
 
 #define SBI_PMU_EVENT_IDX_INVALID 0xFFFFFFFF
 
@@ -293,6 +345,92 @@ enum sbi_cppc_reg_id {
 	SBI_CPPC_NON_ACPI_LAST		= SBI_CPPC_TRANSITION_LATENCY,
 };
 
+/* SBI Function IDs for SSE extension */
+#define SBI_EXT_SSE_READ_ATTR		0x00000000
+#define SBI_EXT_SSE_WRITE_ATTR		0x00000001
+#define SBI_EXT_SSE_REGISTER		0x00000002
+#define SBI_EXT_SSE_UNREGISTER		0x00000003
+#define SBI_EXT_SSE_ENABLE		0x00000004
+#define SBI_EXT_SSE_DISABLE		0x00000005
+#define SBI_EXT_SSE_COMPLETE		0x00000006
+#define SBI_EXT_SSE_INJECT		0x00000007
+#define SBI_EXT_SSE_HART_UNMASK		0x00000008
+#define SBI_EXT_SSE_HART_MASK		0x00000009
+
+/* SBI SSE Event Attributes. */
+enum sbi_sse_attr_id {
+	SBI_SSE_ATTR_STATUS		= 0x00000000,
+	SBI_SSE_ATTR_PRIO		= 0x00000001,
+	SBI_SSE_ATTR_CONFIG		= 0x00000002,
+	SBI_SSE_ATTR_PREFERRED_HART	= 0x00000003,
+	SBI_SSE_ATTR_ENTRY_PC		= 0x00000004,
+	SBI_SSE_ATTR_ENTRY_ARG		= 0x00000005,
+	SBI_SSE_ATTR_INTERRUPTED_SEPC	= 0x00000006,
+	SBI_SSE_ATTR_INTERRUPTED_FLAGS	= 0x00000007,
+	SBI_SSE_ATTR_INTERRUPTED_A6	= 0x00000008,
+	SBI_SSE_ATTR_INTERRUPTED_A7	= 0x00000009,
+
+	SBI_SSE_ATTR_MAX		= 0x0000000A
+};
+
+#define SBI_SSE_ATTR_STATUS_STATE_OFFSET	0
+#define SBI_SSE_ATTR_STATUS_STATE_MASK		0x3
+#define SBI_SSE_ATTR_STATUS_PENDING_OFFSET	2
+#define SBI_SSE_ATTR_STATUS_INJECT_OFFSET	3
+
+#define SBI_SSE_ATTR_CONFIG_ONESHOT	(1 << 0)
+
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_STATUS_SPP	BIT(0)
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_STATUS_SPIE	BIT(1)
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPV	BIT(2)
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPVP	BIT(3)
+
+enum sbi_sse_state {
+	SBI_SSE_STATE_UNUSED		= 0,
+	SBI_SSE_STATE_REGISTERED	= 1,
+	SBI_SSE_STATE_ENABLED		= 2,
+	SBI_SSE_STATE_RUNNING		= 3,
+};
+
+/* SBI SSE Event IDs. */
+#define SBI_SSE_EVENT_LOCAL_RAS			0x00000000
+#define SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP		0x00000001
+#define SBI_SSE_EVENT_LOCAL_PLAT_0_START	0x00004000
+#define SBI_SSE_EVENT_LOCAL_PLAT_0_END		0x00007fff
+#define SBI_SSE_EVENT_GLOBAL_RAS		0x00008000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_0_START	0x0000c000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_0_END		0x0000ffff
+
+#define SBI_SSE_EVENT_LOCAL_PMU			0x00010000
+#define SBI_SSE_EVENT_LOCAL_PLAT_1_START	0x00014000
+#define SBI_SSE_EVENT_LOCAL_PLAT_1_END		0x00017fff
+#define SBI_SSE_EVENT_GLOBAL_PLAT_1_START	0x0001c000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_1_END		0x0001ffff
+
+#define SBI_SSE_EVENT_LOCAL_PLAT_2_START	0x00024000
+#define SBI_SSE_EVENT_LOCAL_PLAT_2_END		0x00027fff
+#define SBI_SSE_EVENT_GLOBAL_PLAT_2_START	0x0002c000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_2_END		0x0002ffff
+
+#define SBI_SSE_EVENT_LOCAL_SOFTWARE		0xffff0000
+#define SBI_SSE_EVENT_LOCAL_PLAT_3_START	0xffff4000
+#define SBI_SSE_EVENT_LOCAL_PLAT_3_END		0xffff7fff
+#define SBI_SSE_EVENT_GLOBAL_SOFTWARE		0xffff8000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_3_START	0xffffc000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_3_END		0xffffffff
+
+#define SBI_SSE_EVENT_GLOBAL_BIT		(1 << 15)
+#define SBI_SSE_EVENT_PLATFORM_BIT		(1 << 14)
+
+/* SBI function IDs for MPXY extension */
+#define SBI_EXT_MPXY_SET_SHMEM			0x0
+#define SBI_EXT_MPXY_GET_CHANNEL_IDS		0x1
+#define SBI_EXT_MPXY_READ_ATTRS			0x2
+#define SBI_EXT_MPXY_WRITE_ATTRS		0x3
+#define SBI_EXT_MPXY_SEND_MSG_WITH_RESP		0x4
+#define SBI_EXT_MPXY_SEND_MSG_NO_RESP		0x5
+#define SBI_EXT_MPXY_GET_NOTIFICATION_EVENTS	0x6
+
 /* SBI base specification related macros */
 #define SBI_SPEC_VERSION_MAJOR_OFFSET		24
 #define SBI_SPEC_VERSION_MAJOR_MASK		0x7f
@@ -313,8 +451,12 @@ enum sbi_cppc_reg_id {
 #define SBI_ERR_ALREADY_STARTED			-7
 #define SBI_ERR_ALREADY_STOPPED			-8
 #define SBI_ERR_NO_SHMEM			-9
+#define SBI_ERR_INVALID_STATE			-10
+#define SBI_ERR_BAD_RANGE			-11
+#define SBI_ERR_TIMEOUT				-12
+#define SBI_ERR_IO				-13
 
-#define SBI_LAST_ERR				SBI_ERR_NO_SHMEM
+#define SBI_LAST_ERR				SBI_ERR_BAD_RANGE
 
 /* clang-format on */
 

include/sbi/sbi_error.h

@@ -24,16 +24,19 @@
 #define SBI_EALREADY_STARTED	SBI_ERR_ALREADY_STARTED
 #define SBI_EALREADY_STOPPED	SBI_ERR_ALREADY_STOPPED
 #define SBI_ENO_SHMEM		SBI_ERR_NO_SHMEM
+#define SBI_EINVALID_STATE	SBI_ERR_INVALID_STATE
+#define SBI_EBAD_RANGE		SBI_ERR_BAD_RANGE
+#define SBI_ETIMEOUT		SBI_ERR_TIMEOUT
+#define SBI_ETIMEDOUT		SBI_ERR_TIMEOUT
+#define SBI_EIO			SBI_ERR_IO
 
 #define SBI_ENODEV		-1000
 #define SBI_ENOSYS		-1001
-#define SBI_ETIMEDOUT		-1002
-#define SBI_EIO			-1003
-#define SBI_EILL		-1004
-#define SBI_ENOSPC		-1005
-#define SBI_ENOMEM		-1006
-#define SBI_EUNKNOWN		-1007
-#define SBI_ENOENT		-1008
+#define SBI_EILL		-1002
+#define SBI_ENOSPC		-1003
+#define SBI_ENOMEM		-1004
+#define SBI_EUNKNOWN		-1005
+#define SBI_ENOENT		-1006
 
 /* clang-format on */
 

include/sbi/sbi_fifo.h

@@ -23,6 +23,18 @@ struct sbi_fifo {
 	u16 tail;
 };
 
+#define SBI_FIFO_INITIALIZER(__queue_mem, __entries, __entry_size)	\
+{	.queue = __queue_mem,						\
+	.qlock = SPIN_LOCK_INITIALIZER,					\
+	.num_entries = __entries,					\
+	.entry_size = __entry_size,					\
+	.avail = 0,							\
+	.tail = 0,							\
+}
+
+#define SBI_FIFO_DEFINE(__name, __queue_mem, __entries, __entry_size)	\
+struct sbi_fifo __name = SBI_FIFO_INITIALIZER(__queue_mem, __entries, __entry_size)
+
 enum sbi_fifo_inplace_update_types {
 	SBI_FIFO_SKIP,
 	SBI_FIFO_UPDATED,
@@ -30,7 +42,7 @@ enum sbi_fifo_inplace_update_types {
 };
 
 int sbi_fifo_dequeue(struct sbi_fifo *fifo, void *data);
-int sbi_fifo_enqueue(struct sbi_fifo *fifo, void *data);
+int sbi_fifo_enqueue(struct sbi_fifo *fifo, void *data, bool force);
 void sbi_fifo_init(struct sbi_fifo *fifo, void *queue_mem, u16 entries,
 		   u16 entry_size);
 int sbi_fifo_is_empty(struct sbi_fifo *fifo);

include/sbi/sbi_fwft.h

@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Rivos Inc.
+ *
+ * Authors:
+ *   Clément Léger <cleger@rivosinc.com>
+ */
+
+#ifndef __SBI_FW_FEATURE_H__
+#define __SBI_FW_FEATURE_H__
+
+#include <sbi/sbi_ecall_interface.h>
+
+struct sbi_scratch;
+
+int sbi_fwft_set(enum sbi_fwft_feature_t feature, unsigned long value,
+		 unsigned long flags);
+int sbi_fwft_get(enum sbi_fwft_feature_t feature, unsigned long *out_val);
+
+int sbi_fwft_init(struct sbi_scratch *scratch, bool cold_boot);
+
+#endif

include/sbi/sbi_hart.h

@@ -53,6 +53,28 @@ enum sbi_hart_extensions {
 	SBI_HART_EXT_ZICBOM,
 	/** Hart has Svpbmt extension */
 	SBI_HART_EXT_SVPBMT,
+	/** Hart has debug trigger extension */
+	SBI_HART_EXT_SDTRIG,
+	/** Hart has Smcsrind extension */
+	SBI_HART_EXT_SMCSRIND,
+	/** Hart has Smcdeleg extension */
+	SBI_HART_EXT_SMCDELEG,
+	/** Hart has Sscsrind extension */
+	SBI_HART_EXT_SSCSRIND,
+	/** Hart has Ssccfg extension */
+	SBI_HART_EXT_SSCCFG,
+	/** Hart has Svade extension */
+	SBI_HART_EXT_SVADE,
+	/** Hart has Svadu extension */
+	SBI_HART_EXT_SVADU,
+	/** Hart has Smnpm extension */
+	SBI_HART_EXT_SMNPM,
+	/** HART has zicfilp extension */
+	SBI_HART_EXT_ZICFILP,
+	/** HART has zicfiss extension */
+	SBI_HART_EXT_ZICFISS,
+	/** Hart has Ssdbltrp extension */
+	SBI_HART_EXT_SSDBLTRP,
 
 	/** Maximum index of Hart extension */
 	SBI_HART_EXT_MAX,

include/sbi/sbi_hartmask.h

@@ -127,6 +127,18 @@ static inline void sbi_hartmask_clear_all(struct sbi_hartmask *dstp)
 	bitmap_zero(sbi_hartmask_bits(dstp), SBI_HARTMASK_MAX_BITS);
 }
 
+/**
+ * *dstp = *srcp
+ * @param dstp the hartmask destination
+ * @param srcp the hartmask source
+ */
+static inline void sbi_hartmask_copy(struct sbi_hartmask *dstp,
+				     const struct sbi_hartmask *srcp)
+{
+	bitmap_copy(sbi_hartmask_bits(dstp), sbi_hartmask_bits(srcp),
+		    SBI_HARTMASK_MAX_BITS);
+}
+
 /**
  * *dstp = *src1p & *src2p
  * @param dstp the hartmask result

include/sbi/sbi_heap.h

@@ -12,16 +12,41 @@
 
 #include <sbi/sbi_types.h>
 
+/* Opaque declaration of heap control struct */
+struct sbi_heap_control;
+
+/* Global heap control structure */
+extern struct sbi_heap_control global_hpctrl;
+
 /* Alignment of heap base address and size */
 #define HEAP_BASE_ALIGN			1024
 
 struct sbi_scratch;
 
 /** Allocate from heap area */
-void *sbi_malloc(size_t size);
+void *sbi_malloc_from(struct sbi_heap_control *hpctrl, size_t size);
+
+static inline void *sbi_malloc(size_t size)
+{
+	return sbi_malloc_from(&global_hpctrl, size);
+}
+
+/** Allocate aligned from heap area */
+void *sbi_aligned_alloc_from(struct sbi_heap_control *hpctrl,
+			     size_t alignment,size_t size);
+
+static inline void *sbi_aligned_alloc(size_t alignment, size_t size)
+{
+	return sbi_aligned_alloc_from(&global_hpctrl, alignment, size);
+}
 
 /** Zero allocate from heap area */
-void *sbi_zalloc(size_t size);
+void *sbi_zalloc_from(struct sbi_heap_control *hpctrl, size_t size);
+
+static inline void *sbi_zalloc(size_t size)
+{
+	return sbi_zalloc_from(&global_hpctrl, size);
+}
 
 /** Allocate array from heap area */
 static inline void *sbi_calloc(size_t nitems, size_t size)
@@ -29,19 +54,48 @@ static inline void *sbi_calloc(size_t nitems, size_t size)
 	return sbi_zalloc(nitems * size);
 }
 
+static inline void *sbi_calloc_from(struct sbi_heap_control *hpctrl,
+				    size_t nitems, size_t size)
+{
+	return sbi_zalloc_from(hpctrl, nitems * size);
+}
+
 /** Free-up to heap area */
-void sbi_free(void *ptr);
+void sbi_free_from(struct sbi_heap_control *hpctrl, void *ptr);
+
+static inline void sbi_free(void *ptr)
+{
+	return sbi_free_from(&global_hpctrl, ptr);
+}
 
 /** Amount (in bytes) of free space in the heap area */
-unsigned long sbi_heap_free_space(void);
+unsigned long sbi_heap_free_space_from(struct sbi_heap_control *hpctrl);
+
+static inline unsigned long sbi_heap_free_space(void)
+{
+	return sbi_heap_free_space_from(&global_hpctrl);
+}
 
 /** Amount (in bytes) of used space in the heap area */
-unsigned long sbi_heap_used_space(void);
+unsigned long sbi_heap_used_space_from(struct sbi_heap_control *hpctrl);
+
+static inline unsigned long sbi_heap_used_space(void)
+{
+	return sbi_heap_used_space_from(&global_hpctrl);
+}
 
 /** Amount (in bytes) of reserved space in the heap area */
-unsigned long sbi_heap_reserved_space(void);
+unsigned long sbi_heap_reserved_space_from(struct sbi_heap_control *hpctrl);
+
+static inline unsigned long sbi_heap_reserved_space(void)
+{
+	return sbi_heap_reserved_space_from(&global_hpctrl);
+}
 
 /** Initialize heap area */
 int sbi_heap_init(struct sbi_scratch *scratch);
+int sbi_heap_init_new(struct sbi_heap_control *hpctrl, unsigned long base,
+		       unsigned long size);
+int sbi_heap_alloc_new(struct sbi_heap_control **hpctrl);
 
 #endif

include/sbi/sbi_hsm.h

@@ -10,6 +10,7 @@
 #ifndef __SBI_HSM_H__
 #define __SBI_HSM_H__
 
+#include <sbi/sbi_hartmask.h>
 #include <sbi/sbi_types.h>
 
 /** Hart state managment device */
@@ -39,8 +40,12 @@ struct sbi_hsm_device {
 	 *
 	 * For successful non-retentive suspend, the hart will resume from
 	 * the warm boot entry point.
+	 *
+	 * NOTE: mmode_resume_addr(resume address) is optional hence it
+	 * may or may not be honored by the platform. If its not honored
+	 * then platform must ensure to resume from the warmboot address.
 	 */
-	int (*hart_suspend)(u32 suspend_type);
+	int (*hart_suspend)(u32 suspend_type, ulong mmode_resume_addr);
 
 	/**
 	 * Perform platform-specific actions to resume from a suspended state.
@@ -58,7 +63,7 @@ const struct sbi_hsm_device *sbi_hsm_get_device(void);
 
 void sbi_hsm_set_device(const struct sbi_hsm_device *dev);
 
-int sbi_hsm_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot);
+int sbi_hsm_init(struct sbi_scratch *scratch, bool cold_boot);
 void __noreturn sbi_hsm_exit(struct sbi_scratch *scratch);
 
 int sbi_hsm_hart_start(struct sbi_scratch *scratch,
@@ -72,10 +77,10 @@ int sbi_hsm_hart_suspend(struct sbi_scratch *scratch, u32 suspend_type,
 			 ulong raddr, ulong rmode, ulong arg1);
 bool sbi_hsm_hart_change_state(struct sbi_scratch *scratch, long oldstate,
 			       long newstate);
-int __sbi_hsm_hart_get_state(u32 hartid);
+int __sbi_hsm_hart_get_state(u32 hartindex);
 int sbi_hsm_hart_get_state(const struct sbi_domain *dom, u32 hartid);
 int sbi_hsm_hart_interruptible_mask(const struct sbi_domain *dom,
-				    ulong hbase, ulong *out_hmask);
+				    struct sbi_hartmask *mask);
 void __sbi_hsm_suspend_non_ret_save(struct sbi_scratch *scratch);
 void __noreturn sbi_hsm_hart_start_finish(struct sbi_scratch *scratch,
 					  u32 hartid);

include/sbi/sbi_illegal_insn.h

@@ -12,8 +12,8 @@
 
 #include <sbi/sbi_types.h>
 
-struct sbi_trap_regs;
+struct sbi_trap_context;
 
-int sbi_illegal_insn_handler(ulong insn, struct sbi_trap_regs *regs);
+int sbi_illegal_insn_handler(struct sbi_trap_context *tcntx);
 
 #endif

include/sbi/sbi_init.h

@@ -16,9 +16,9 @@ struct sbi_scratch;
 
 void __noreturn sbi_init(struct sbi_scratch *scratch);
 
-unsigned long sbi_entry_count(u32 hartid);
+unsigned long sbi_entry_count(u32 hartindex);
 
-unsigned long sbi_init_count(u32 hartid);
+unsigned long sbi_init_count(u32 hartindex);
 
 void __noreturn sbi_exit(struct sbi_scratch *scratch);
 

include/sbi/sbi_ipi.h

@@ -26,8 +26,8 @@ struct sbi_ipi_device {
 	/** Send IPI to a target HART index */
 	void (*ipi_send)(u32 hart_index);
 
-	/** Clear IPI for a target HART index */
-	void (*ipi_clear)(u32 hart_index);
+	/** Clear IPI for the current hart */
+	void (*ipi_clear)(void);
 };
 
 enum sbi_ipi_update_type {
@@ -87,7 +87,7 @@ void sbi_ipi_process(void);
 
 int sbi_ipi_raw_send(u32 hartindex);
 
-void sbi_ipi_raw_clear(u32 hartindex);
+void sbi_ipi_raw_clear(void);
 
 const struct sbi_ipi_device *sbi_ipi_get_device(void);
 

include/sbi/sbi_irqchip.h

@@ -10,20 +10,22 @@
 #ifndef __SBI_IRQCHIP_H__
 #define __SBI_IRQCHIP_H__
 
+#include <sbi/sbi_list.h>
 #include <sbi/sbi_types.h>
 
 struct sbi_scratch;
-struct sbi_trap_regs;
 
-/**
- * Set external interrupt handling function
- *
- * This function is called by OpenSBI platform code to set a handler for
- * external interrupts
- *
- * @param fn function pointer for handling external irqs
- */
-void sbi_irqchip_set_irqfn(int (*fn)(struct sbi_trap_regs *regs));
+/** irqchip hardware device */
+struct sbi_irqchip_device {
+	/** Node in the list of irqchip devices */
+	struct sbi_dlist node;
+
+	/** Initialize per-hart state for the current hart */
+	int (*warm_init)(struct sbi_irqchip_device *dev);
+
+	/** Handle an IRQ from this irqchip */
+	int (*irq_handle)(void);
+};
 
 /**
  * Process external interrupts
@@ -33,7 +35,10 @@ void sbi_irqchip_set_irqfn(int (*fn)(struct sbi_trap_regs *regs));
  *
  * @param regs pointer for trap registers
  */
-int sbi_irqchip_process(struct sbi_trap_regs *regs);
+int sbi_irqchip_process(void);
+
+/** Register an irqchip device to receive callbacks */
+void sbi_irqchip_add_device(struct sbi_irqchip_device *dev);
 
 /** Initialize interrupt controllers */
 int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot);

include/sbi/sbi_misaligned_ldst.h

@@ -1,23 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2019 Western Digital Corporation or its affiliates.
- *
- * Authors:
- *   Anup Patel <anup.patel@wdc.com>
- */
-
-#ifndef __SBI_MISALIGNED_LDST_H__
-#define __SBI_MISALIGNED_LDST_H__
-
-#include <sbi/sbi_types.h>
-
-struct sbi_trap_regs;
-
-int sbi_misaligned_load_handler(ulong addr, ulong tval2, ulong tinst,
-				struct sbi_trap_regs *regs);
-
-int sbi_misaligned_store_handler(ulong addr, ulong tval2, ulong tinst,
-				 struct sbi_trap_regs *regs);
-
-#endif

include/sbi/sbi_mpxy.h

@@ -0,0 +1,183 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Rahul Pathak <rpathak@ventanamicro.com>
+ */
+
+#ifndef __SBI_MPXY_H__
+#define __SBI_MPXY_H__
+
+#include <sbi/sbi_list.h>
+
+struct sbi_scratch;
+
+#define SBI_MPXY_MSGPROTO_VERSION(Major, Minor) ((Major << 16) | Minor)
+
+enum sbi_mpxy_attr_id {
+	/* Standard channel attributes managed by MPXY framework */
+	SBI_MPXY_ATTR_MSG_PROT_ID		= 0x00000000,
+	SBI_MPXY_ATTR_MSG_PROT_VER		= 0x00000001,
+	SBI_MPXY_ATTR_MSG_MAX_LEN		= 0x00000002,
+	SBI_MPXY_ATTR_MSG_SEND_TIMEOUT		= 0x00000003,
+	SBI_MPXY_ATTR_MSG_COMPLETION_TIMEOUT	= 0x00000004,
+	SBI_MPXY_ATTR_CHANNEL_CAPABILITY	= 0x00000005,
+	SBI_MPXY_ATTR_SSE_EVENT_ID		= 0x00000006,
+	SBI_MPXY_ATTR_MSI_CONTROL		= 0x00000007,
+	SBI_MPXY_ATTR_MSI_ADDR_LO		= 0x00000008,
+	SBI_MPXY_ATTR_MSI_ADDR_HI		= 0x00000009,
+	SBI_MPXY_ATTR_MSI_DATA			= 0x0000000A,
+	SBI_MPXY_ATTR_EVENTS_STATE_CONTROL	= 0x0000000B,
+	SBI_MPXY_ATTR_STD_ATTR_MAX_IDX,
+	/* Message protocol specific attributes, managed by
+	 * message protocol driver */
+	SBI_MPXY_ATTR_MSGPROTO_ATTR_START	= 0x80000000,
+	SBI_MPXY_ATTR_MSGPROTO_ATTR_END		= 0xffffffff
+};
+
+/**
+ * SBI MPXY Message Protocol IDs
+ */
+enum sbi_mpxy_msgproto_id {
+	SBI_MPXY_MSGPROTO_RPMI_ID = 0x00000000,
+	SBI_MPXY_MSGPROTO_MAX_IDX,
+	/** Vendor specific message protocol IDs */
+	SBI_MPXY_MSGPROTO_VENDOR_START	= 0x80000000,
+	SBI_MPXY_MSGPROTO_VENDOR_END	= 0xffffffff
+};
+
+enum SBI_EXT_MPXY_SHMEM_FLAGS {
+	SBI_EXT_MPXY_SHMEM_FLAG_OVERWRITE		= 0b00,
+	SBI_EXT_MPXY_SHMEM_FLAG_OVERWRITE_RETURN	= 0b01,
+	SBI_EXT_MPXY_SHMEM_FLAG_MAX_IDX
+};
+
+struct sbi_mpxy_msi_info {
+	/* MSI target address low 32-bit */
+	u32 msi_addr_lo;
+	/* MSI target address high 32-bit */
+	u32 msi_addr_hi;
+	/* MSI data */
+	u32 msi_data;
+};
+
+/**
+ * Channel attributes.
+ * NOTE: The sequence of attribute fields are as per the
+ * defined sequence in the attribute table in spec(or as
+ * per the enum sbi_mpxy_attr_id).
+ */
+struct sbi_mpxy_channel_attrs {
+	/* Message protocol ID */
+	u32 msg_proto_id;
+	/* Message protocol Version */
+	u32 msg_proto_version;
+	/* Message protocol maximum message data length(bytes) */
+	u32 msg_data_maxlen;
+	/* Message protocol message send timeout
+	 * in microseconds */
+	u32 msg_send_timeout;
+	/* Message protocol message response timeout in
+	 * microseconds. Its the aggregate of msg_send_timeout
+	 * and the timeout in receiving the response */
+	u32 msg_completion_timeout;
+	/* Bit array for channel capabilities */
+	u32 capability;
+	u32 sse_event_id;
+	u32 msi_control;
+	struct sbi_mpxy_msi_info msi_info;
+	/* Events State Control */
+	u32 eventsstate_ctrl;
+};
+
+/** A Message proxy channel accessible through SBI interface */
+struct sbi_mpxy_channel {
+	/** List head to a set of channels */
+	struct sbi_dlist head;
+	u32 channel_id;
+	struct sbi_mpxy_channel_attrs attrs;
+
+	/**
+	 * Read message protocol attributes
+	 * NOTE: inmem requires little-endian byte-ordering
+	 */
+	int (*read_attributes)(struct sbi_mpxy_channel *channel,
+				u32 *outmem,
+				u32 base_attr_id,
+				u32 attr_count);
+
+	/**
+	 * Write message protocol attributes
+	 * NOTE: outmem requires little-endian byte-ordering
+	 */
+	int (*write_attributes)(struct sbi_mpxy_channel *channel,
+				u32 *inmem,
+				u32 base_attr_id,
+				u32 attr_count);
+	/**
+	 * Send a message and wait for response
+	 * NOTE: msgbuf requires little-endian byte-ordering
+	 */
+	int (*send_message_with_response)(struct sbi_mpxy_channel *channel,
+			    u32 msg_id, void *msgbuf, u32 msg_len,
+			    void *respbuf, u32 resp_max_len,
+			    unsigned long *resp_len);
+
+	/** Send message without response */
+	int (*send_message_without_response)(struct sbi_mpxy_channel *channel,
+			    u32 msg_id, void *msgbuf, u32 msg_len);
+
+	/**
+	 * Get notifications events if supported on a channel
+	 * NOTE: eventsbuf requires little-endian byte-ordering
+	 */
+	int (*get_notification_events)(struct sbi_mpxy_channel *channel,
+					void *eventsbuf, u32 bufsize,
+					unsigned long *events_len);
+
+	/**
+	 * Callback to enable the events state reporting
+	 * in the message protocol implementation
+	 */
+	void (*switch_eventsstate)(u32 enable);
+};
+
+/** Register a Message proxy channel */
+int sbi_mpxy_register_channel(struct sbi_mpxy_channel *channel);
+
+/** Initialize Message proxy subsystem */
+int sbi_mpxy_init(struct sbi_scratch *scratch);
+
+/** Check if some Message proxy channel is available */
+bool sbi_mpxy_channel_available(void);
+
+/** Set Message proxy shared memory on the calling HART */
+int sbi_mpxy_set_shmem(unsigned long shmem_size,
+			unsigned long shmem_phys_lo,
+			unsigned long shmem_phys_hi,
+			unsigned long flags);
+
+/** Get channel IDs list */
+int sbi_mpxy_get_channel_ids(u32 start_index);
+
+/** Read MPXY channel attributes */
+int sbi_mpxy_read_attrs(u32 channel_id, u32 base_attr_id, u32 attr_count);
+
+/** Write MPXY channel attributes */
+int sbi_mpxy_write_attrs(u32 channel_id, u32 base_attr_id, u32 attr_count);
+
+/**
+ * Send a message over a MPXY channel.
+ * In case if response is not expected, resp_data_len will be NULL.
+ */
+int sbi_mpxy_send_message(u32 channel_id, u8 msg_id,
+					unsigned long msg_data_len,
+					unsigned long *resp_data_len);
+
+/** Get Message proxy notification events */
+int sbi_mpxy_get_notification_events(u32 channel_id,
+					unsigned long *events_len);
+
+#endif

include/sbi/sbi_platform.h

@@ -48,11 +48,13 @@
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_version.h>
+#include <sbi/sbi_trap_ldst.h>
 
 struct sbi_domain_memregion;
 struct sbi_ecall_return;
 struct sbi_trap_regs;
 struct sbi_hart_features;
+union sbi_ldst_data;
 
 /** Possible feature flags of a platform */
 enum sbi_platform_features {
@@ -109,18 +111,11 @@ struct sbi_platform_operations {
 	/** Get platform specific mhpmevent value */
 	uint64_t (*pmu_xlate_to_mhpmevent)(uint32_t event_idx, uint64_t data);
 
-	/** Initialize the platform console */
-	int (*console_init)(void);
+	/** Initialize the platform interrupt controller during cold boot */
+	int (*irqchip_init)(void);
 
-	/** Initialize the platform interrupt controller for current HART */
-	int (*irqchip_init)(bool cold_boot);
-	/** Exit the platform interrupt controller for current HART */
-	void (*irqchip_exit)(void);
-
-	/** Initialize IPI for current HART */
-	int (*ipi_init)(bool cold_boot);
-	/** Exit IPI for current HART */
-	void (*ipi_exit)(void);
+	/** Initialize IPI during cold boot */
+	int (*ipi_init)(void);
 
 	/** Get tlb flush limit value **/
 	u64 (*get_tlbr_flush_limit)(void);
@@ -128,10 +123,11 @@ struct sbi_platform_operations {
 	/** Get tlb fifo num entries*/
 	u32 (*get_tlb_num_entries)(void);
 
-	/** Initialize platform timer for current HART */
-	int (*timer_init)(bool cold_boot);
-	/** Exit platform timer for current HART */
-	void (*timer_exit)(void);
+	/** Initialize platform timer during cold boot */
+	int (*timer_init)(void);
+
+	/** Initialize the platform Message Proxy(MPXY) driver */
+	int (*mpxy_init)(void);
 
 	/** Check if SBI vendor extension is implemented or not */
 	bool (*vendor_ext_check)(void);
@@ -139,6 +135,13 @@ struct sbi_platform_operations {
 	int (*vendor_ext_provider)(long funcid,
 				   struct sbi_trap_regs *regs,
 				   struct sbi_ecall_return *out);
+
+	/** platform specific handler to fixup load fault */
+	int (*emulate_load)(int rlen, unsigned long addr,
+			    union sbi_ldst_data *out_val);
+	/** platform specific handler to fixup store fault */
+	int (*emulate_store)(int wlen, unsigned long addr,
+			     union sbi_ldst_data in_val);
 };
 
 /** Platform default per-HART stack size for exception/interrupt handling */
@@ -146,7 +149,7 @@ struct sbi_platform_operations {
 
 /** Platform default heap size */
 #define SBI_PLATFORM_DEFAULT_HEAP_SIZE(__num_hart)	\
-					(0x8000 + 0x800 * (__num_hart))
+					(0x8000 + 0x1000 * (__num_hart))
 
 /** Representation of a platform */
 struct sbi_platform {
@@ -541,98 +544,59 @@ static inline uint64_t sbi_platform_pmu_xlate_to_mhpmevent(const struct sbi_plat
 }
 
 /**
- * Initialize the platform console
+ * Initialize the platform interrupt controller during cold boot
  *
  * @param plat pointer to struct sbi_platform
  *
  * @return 0 on success and negative error code on failure
  */
-static inline int sbi_platform_console_init(const struct sbi_platform *plat)
-{
-	if (plat && sbi_platform_ops(plat)->console_init)
-		return sbi_platform_ops(plat)->console_init();
-	return 0;
-}
-
-/**
- * Initialize the platform interrupt controller for current HART
- *
- * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (true) or warm_boot (false)
- *
- * @return 0 on success and negative error code on failure
- */
-static inline int sbi_platform_irqchip_init(const struct sbi_platform *plat,
-					    bool cold_boot)
+static inline int sbi_platform_irqchip_init(const struct sbi_platform *plat)
 {
 	if (plat && sbi_platform_ops(plat)->irqchip_init)
-		return sbi_platform_ops(plat)->irqchip_init(cold_boot);
+		return sbi_platform_ops(plat)->irqchip_init();
 	return 0;
 }
 
 /**
- * Exit the platform interrupt controller for current HART
+ * Initialize the platform IPI support during cold boot
  *
  * @param plat pointer to struct sbi_platform
- */
-static inline void sbi_platform_irqchip_exit(const struct sbi_platform *plat)
-{
-	if (plat && sbi_platform_ops(plat)->irqchip_exit)
-		sbi_platform_ops(plat)->irqchip_exit();
-}
-
-/**
- * Initialize the platform IPI support for current HART
- *
- * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (true) or warm_boot (false)
  *
  * @return 0 on success and negative error code on failure
  */
-static inline int sbi_platform_ipi_init(const struct sbi_platform *plat,
-					bool cold_boot)
+static inline int sbi_platform_ipi_init(const struct sbi_platform *plat)
 {
 	if (plat && sbi_platform_ops(plat)->ipi_init)
-		return sbi_platform_ops(plat)->ipi_init(cold_boot);
+		return sbi_platform_ops(plat)->ipi_init();
 	return 0;
 }
 
 /**
- * Exit the platform IPI support for current HART
+ * Initialize the platform timer during cold boot
  *
  * @param plat pointer to struct sbi_platform
- */
-static inline void sbi_platform_ipi_exit(const struct sbi_platform *plat)
-{
-	if (plat && sbi_platform_ops(plat)->ipi_exit)
-		sbi_platform_ops(plat)->ipi_exit();
-}
-
-/**
- * Initialize the platform timer for current HART
- *
- * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (true) or warm_boot (false)
  *
  * @return 0 on success and negative error code on failure
  */
-static inline int sbi_platform_timer_init(const struct sbi_platform *plat,
-					  bool cold_boot)
+static inline int sbi_platform_timer_init(const struct sbi_platform *plat)
 {
 	if (plat && sbi_platform_ops(plat)->timer_init)
-		return sbi_platform_ops(plat)->timer_init(cold_boot);
+		return sbi_platform_ops(plat)->timer_init();
 	return 0;
 }
 
 /**
- * Exit the platform timer for current HART
+ * Initialize the platform Message Proxy drivers
  *
  * @param plat pointer to struct sbi_platform
+ *
+ * @return 0 on success and negative error code on failure
  */
-static inline void sbi_platform_timer_exit(const struct sbi_platform *plat)
+static inline int sbi_platform_mpxy_init(const struct sbi_platform *plat)
 {
-	if (plat && sbi_platform_ops(plat)->timer_exit)
-		sbi_platform_ops(plat)->timer_exit();
+	if (plat && sbi_platform_ops(plat)->mpxy_init)
+		return sbi_platform_ops(plat)->mpxy_init();
+	return 0;
 }
 
 /**
@@ -675,6 +639,50 @@ static inline int sbi_platform_vendor_ext_provider(
 	return SBI_ENOTSUPP;
 }
 
+/**
+ * Ask platform to emulate the trapped load
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param rlen length of the load: 1/2/4/8...
+ * @param addr virtual address of the load. Platform needs to page-walk and
+ *        find the physical address if necessary
+ * @param out_val value loaded
+ *
+ * @return 0 on success and negative error code on failure
+ */
+static inline int sbi_platform_emulate_load(const struct sbi_platform *plat,
+					    int rlen, unsigned long addr,
+					    union sbi_ldst_data *out_val)
+{
+	if (plat && sbi_platform_ops(plat)->emulate_load) {
+		return sbi_platform_ops(plat)->emulate_load(rlen, addr,
+							    out_val);
+	}
+	return SBI_ENOTSUPP;
+}
+
+/**
+ * Ask platform to emulate the trapped store
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param wlen length of the store: 1/2/4/8...
+ * @param addr virtual address of the store. Platform needs to page-walk and
+ *        find the physical address if necessary
+ * @param in_val value to store
+ *
+ * @return 0 on success and negative error code on failure
+ */
+static inline int sbi_platform_emulate_store(const struct sbi_platform *plat,
+					     int wlen, unsigned long addr,
+					     union sbi_ldst_data in_val)
+{
+	if (plat && sbi_platform_ops(plat)->emulate_store) {
+		return sbi_platform_ops(plat)->emulate_store(wlen, addr,
+							     in_val);
+	}
+	return SBI_ENOTSUPP;
+}
+
 #endif
 
 #endif

include/sbi/sbi_pmu.h

@@ -11,6 +11,7 @@
 #define __SBI_PMU_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_trap.h>
 
 struct sbi_scratch;
 
@@ -141,6 +142,8 @@ int sbi_pmu_ctr_start(unsigned long cidx_base, unsigned long cidx_mask,
 		      unsigned long flags, uint64_t ival);
 
 int sbi_pmu_ctr_get_info(uint32_t cidx, unsigned long *ctr_info);
+int sbi_pmu_event_get_info(unsigned long shmem_lo, unsigned long shmem_high,
+						   unsigned long num_events, unsigned long flags);
 
 unsigned long sbi_pmu_num_ctr(void);
 
@@ -150,4 +153,6 @@ int sbi_pmu_ctr_cfg_match(unsigned long cidx_base, unsigned long cidx_mask,
 
 int sbi_pmu_ctr_incr_fw(enum sbi_pmu_fw_event_code_id fw_id);
 
+void sbi_pmu_ovf_irq();
+
 #endif

include/sbi/sbi_scratch.h

@@ -22,7 +22,7 @@
 #define SBI_SCRATCH_FW_RW_OFFSET		(2 * __SIZEOF_POINTER__)
 /** Offset of fw_heap_offset member in sbi_scratch */
 #define SBI_SCRATCH_FW_HEAP_OFFSET		(3 * __SIZEOF_POINTER__)
-/** Offset of fw_heap_size_offset member in sbi_scratch */
+/** Offset of fw_heap_size member in sbi_scratch */
 #define SBI_SCRATCH_FW_HEAP_SIZE_OFFSET		(4 * __SIZEOF_POINTER__)
 /** Offset of next_arg1 member in sbi_scratch */
 #define SBI_SCRATCH_NEXT_ARG1_OFFSET		(5 * __SIZEOF_POINTER__)
@@ -36,14 +36,16 @@
 #define SBI_SCRATCH_PLATFORM_ADDR_OFFSET	(9 * __SIZEOF_POINTER__)
 /** Offset of hartid_to_scratch member in sbi_scratch */
 #define SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET	(10 * __SIZEOF_POINTER__)
-/** Offset of trap_exit member in sbi_scratch */
-#define SBI_SCRATCH_TRAP_EXIT_OFFSET		(11 * __SIZEOF_POINTER__)
+/** Offset of trap_context member in sbi_scratch */
+#define SBI_SCRATCH_TRAP_CONTEXT_OFFSET		(11 * __SIZEOF_POINTER__)
 /** Offset of tmp0 member in sbi_scratch */
 #define SBI_SCRATCH_TMP0_OFFSET			(12 * __SIZEOF_POINTER__)
 /** Offset of options member in sbi_scratch */
 #define SBI_SCRATCH_OPTIONS_OFFSET		(13 * __SIZEOF_POINTER__)
+/** Offset of hartindex member in sbi_scratch */
+#define SBI_SCRATCH_HARTINDEX_OFFSET		(14 * __SIZEOF_POINTER__)
 /** Offset of extra space in sbi_scratch */
-#define SBI_SCRATCH_EXTRA_SPACE_OFFSET		(14 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_EXTRA_SPACE_OFFSET		(15 * __SIZEOF_POINTER__)
 /** Maximum size of sbi_scratch (4KB) */
 #define SBI_SCRATCH_SIZE			(0x1000)
 
@@ -77,12 +79,14 @@ struct sbi_scratch {
 	unsigned long platform_addr;
 	/** Address of HART ID to sbi_scratch conversion function */
 	unsigned long hartid_to_scratch;
-	/** Address of trap exit function */
-	unsigned long trap_exit;
+	/** Address of current trap context */
+	unsigned long trap_context;
 	/** Temporary storage */
 	unsigned long tmp0;
 	/** Options for OpenSBI library */
 	unsigned long options;
+	/** Index of the hart */
+	unsigned long hartindex;
 };
 
 /**
@@ -130,10 +134,10 @@ _Static_assert(
 	"struct sbi_scratch definition has changed, please redefine "
 	"SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET");
 _Static_assert(
-	offsetof(struct sbi_scratch, trap_exit)
-		== SBI_SCRATCH_TRAP_EXIT_OFFSET,
+	offsetof(struct sbi_scratch, trap_context)
+		== SBI_SCRATCH_TRAP_CONTEXT_OFFSET,
 	"struct sbi_scratch definition has changed, please redefine "
-	"SBI_SCRATCH_TRAP_EXIT_OFFSET");
+	"SBI_SCRATCH_TRAP_CONTEXT_OFFSET");
 _Static_assert(
 	offsetof(struct sbi_scratch, tmp0)
 		== SBI_SCRATCH_TMP0_OFFSET,
@@ -155,7 +159,7 @@ enum sbi_scratch_options {
 
 /** Get pointer to sbi_scratch for current HART */
 #define sbi_scratch_thishart_ptr() \
-	((struct sbi_scratch *)csr_read(CSR_MSCRATCH))
+	((struct sbi_scratch *)csr_read_relaxed(CSR_MSCRATCH))
 
 /** Get Arg1 of next booting stage for current HART */
 #define sbi_scratch_thishart_arg1_ptr() \
@@ -202,6 +206,10 @@ do {									\
 					= (__type)(__ptr);		\
 } while (0)
 
+/** Get the hart index of the current hart */
+#define current_hartindex() \
+	(sbi_scratch_thishart_ptr()->hartindex)
+
 /** Last HART index having a sbi_scratch pointer */
 extern u32 last_hartindex_having_scratch;
 

include/sbi/sbi_sse.h

@@ -0,0 +1,95 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Rivos Systems.
+ */
+
+#ifndef __SBI_SSE_H__
+#define __SBI_SSE_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_list.h>
+#include <sbi/riscv_locks.h>
+
+struct sbi_scratch;
+struct sbi_trap_regs;
+struct sbi_ecall_return;
+
+#define EXC_MODE_PP_SHIFT		0
+#define EXC_MODE_PP			BIT(EXC_MODE_PP_SHIFT)
+#define EXC_MODE_PV_SHIFT		1
+#define EXC_MODE_PV			BIT(EXC_MODE_PV_SHIFT)
+#define EXC_MODE_SSTATUS_SPIE_SHIFT	2
+#define EXC_MODE_SSTATUS_SPIE		BIT(EXC_MODE_SSTATUS_SPIE_SHIFT)
+
+struct sbi_sse_cb_ops {
+	/**
+	 * Called when hart_id is changed on the event.
+	 */
+	void (*set_hartid_cb)(uint32_t event_id, unsigned long hart_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_COMPLETE is invoked on the event.
+	 */
+	void (*complete_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_REGISTER is invoked on the event.
+	 */
+	void (*register_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_UNREGISTER is invoked on the event.
+	 */
+	void (*unregister_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_ENABLE is invoked on the event.
+	 */
+	void (*enable_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_DISABLE is invoked on the event.
+	 */
+	void (*disable_cb)(uint32_t event_id);
+};
+
+/* Set the callback operations for an event
+ * @param event_id Event identifier (SBI_SSE_EVENT_*)
+ * @param cb_ops Callback operations
+ * @return 0 on success, error otherwise
+ */
+int sbi_sse_set_cb_ops(uint32_t event_id, const struct sbi_sse_cb_ops *cb_ops);
+
+/* Inject an event to the current hard
+ * @param event_id Event identifier (SBI_SSE_EVENT_*)
+ * @param regs Registers that were used on SBI entry
+ * @return 0 on success, error otherwise
+ */
+int sbi_sse_inject_event(uint32_t event_id);
+
+void sbi_sse_process_pending_events(struct sbi_trap_regs *regs);
+
+
+int sbi_sse_init(struct sbi_scratch *scratch, bool cold_boot);
+void sbi_sse_exit(struct sbi_scratch *scratch);
+
+/* Interface called from sbi_ecall_sse.c */
+int sbi_sse_register(uint32_t event_id, unsigned long handler_entry_pc,
+		     unsigned long handler_entry_arg);
+int sbi_sse_unregister(uint32_t event_id);
+int sbi_sse_hart_mask(void);
+int sbi_sse_hart_unmask(void);
+int sbi_sse_enable(uint32_t event_id);
+int sbi_sse_disable(uint32_t event_id);
+int sbi_sse_complete(struct sbi_trap_regs *regs, struct sbi_ecall_return *out);
+int sbi_sse_inject_from_ecall(uint32_t event_id, unsigned long hart_id,
+			      struct sbi_ecall_return *out);
+int sbi_sse_read_attrs(uint32_t event_id, uint32_t base_attr_id,
+		       uint32_t attr_count, unsigned long output_phys_lo,
+		       unsigned long output_phys_hi);
+int sbi_sse_write_attrs(uint32_t event_id, uint32_t base_attr_id,
+			uint32_t attr_count, unsigned long input_phys_lo,
+			unsigned long input_phys_hi);
+
+#endif

include/sbi/sbi_timer.h

@@ -28,6 +28,9 @@ struct sbi_timer_device {
 
 	/** Stop timer event for current HART */
 	void (*timer_event_stop)(void);
+
+	/** Initialize timer device for current HART */
+	int (*warm_init)(void);
 };
 
 struct sbi_scratch;
@@ -78,8 +81,10 @@ u64 sbi_timer_get_delta(void);
 /** Set timer delta value for current HART */
 void sbi_timer_set_delta(ulong delta);
 
+#if __riscv_xlen == 32
 /** Set upper 32-bits of timer delta value for current HART */
 void sbi_timer_set_delta_upper(ulong delta_upper);
+#endif
 
 /** Start timer event for current HART */
 void sbi_timer_event_start(u64 next_event);

include/sbi/sbi_trap.h

@@ -87,20 +87,18 @@
 /** Last member index in sbi_trap_regs */
 #define SBI_TRAP_REGS_last			35
 
-/** Index of epc member in sbi_trap_info */
-#define SBI_TRAP_INFO_epc			0
 /** Index of cause member in sbi_trap_info */
-#define SBI_TRAP_INFO_cause			1
+#define SBI_TRAP_INFO_cause			0
 /** Index of tval member in sbi_trap_info */
-#define SBI_TRAP_INFO_tval			2
+#define SBI_TRAP_INFO_tval			1
 /** Index of tval2 member in sbi_trap_info */
-#define SBI_TRAP_INFO_tval2			3
+#define SBI_TRAP_INFO_tval2			2
 /** Index of tinst member in sbi_trap_info */
-#define SBI_TRAP_INFO_tinst			4
+#define SBI_TRAP_INFO_tinst			3
 /** Index of gva member in sbi_trap_info */
-#define SBI_TRAP_INFO_gva			5
+#define SBI_TRAP_INFO_gva			4
 /** Last member index in sbi_trap_info */
-#define SBI_TRAP_INFO_last			6
+#define SBI_TRAP_INFO_last			5
 
 /* clang-format on */
 
@@ -114,9 +112,15 @@
 /** Size (in bytes) of sbi_trap_info */
 #define SBI_TRAP_INFO_SIZE SBI_TRAP_INFO_OFFSET(last)
 
+/** Size (in bytes) of sbi_trap_context */
+#define SBI_TRAP_CONTEXT_SIZE (SBI_TRAP_REGS_SIZE + \
+			       SBI_TRAP_INFO_SIZE + \
+			       __SIZEOF_POINTER__)
+
 #ifndef __ASSEMBLER__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_scratch.h>
 
 /** Representation of register state at time of trap/interrupt */
 struct sbi_trap_regs {
@@ -194,8 +198,6 @@ struct sbi_trap_regs {
 
 /** Representation of trap details */
 struct sbi_trap_info {
-	/** epc Trap program counter */
-	unsigned long epc;
 	/** cause Trap exception cause */
 	unsigned long cause;
 	/** tval Trap value */
@@ -208,6 +210,16 @@ struct sbi_trap_info {
 	unsigned long gva;
 };
 
+/** Representation of trap context saved on stack */
+struct sbi_trap_context {
+	/** Register state */
+	struct sbi_trap_regs regs;
+	/** Trap details */
+	struct sbi_trap_info trap;
+	/** Pointer to previous trap context */
+	struct sbi_trap_context *prev_context;
+};
+
 static inline unsigned long sbi_regs_gva(const struct sbi_trap_regs *regs)
 {
 	/*
@@ -224,12 +236,35 @@ static inline unsigned long sbi_regs_gva(const struct sbi_trap_regs *regs)
 #endif
 }
 
+static inline bool sbi_regs_from_virt(const struct sbi_trap_regs *regs)
+{
+#if __riscv_xlen == 32
+	return (regs->mstatusH & MSTATUSH_MPV) ? true : false;
+#else
+	return (regs->mstatus & MSTATUS_MPV) ? true : false;
+#endif
+}
+
+static inline int sbi_mstatus_prev_mode(unsigned long mstatus)
+{
+	return (mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT;
+}
+
 int sbi_trap_redirect(struct sbi_trap_regs *regs,
-		      struct sbi_trap_info *trap);
+		      const struct sbi_trap_info *trap);
 
-struct sbi_trap_regs *sbi_trap_handler(struct sbi_trap_regs *regs);
+static inline struct sbi_trap_context *sbi_trap_get_context(struct sbi_scratch *scratch)
+{
+	return (scratch) ? (void *)scratch->trap_context : NULL;
+}
 
-void __noreturn sbi_trap_exit(const struct sbi_trap_regs *regs);
+static inline void sbi_trap_set_context(struct sbi_scratch *scratch,
+					struct sbi_trap_context *tcntx)
+{
+	scratch->trap_context = (unsigned long)tcntx;
+}
+
+struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx);
 
 #endif
 

include/sbi/sbi_trap_ldst.h

@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __SBI_TRAP_LDST_H__
+#define __SBI_TRAP_LDST_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_trap.h>
+
+union sbi_ldst_data {
+	u64 data_u64;
+	u32 data_u32;
+	u8 data_bytes[8];
+	ulong data_ulong;
+};
+
+int sbi_misaligned_load_handler(struct sbi_trap_context *tcntx);
+
+int sbi_misaligned_store_handler(struct sbi_trap_context *tcntx);
+
+int sbi_load_access_handler(struct sbi_trap_context *tcntx);
+
+int sbi_store_access_handler(struct sbi_trap_context *tcntx);
+
+int sbi_double_trap_handler(struct sbi_trap_context *tcntx);
+
+ulong sbi_misaligned_tinst_fixup(ulong orig_tinst, ulong new_tinst,
+				 ulong addr_offset);
+
+int sbi_misaligned_v_ld_emulator(int rlen, union sbi_ldst_data *out_val,
+				 struct sbi_trap_context *tcntx);
+
+int sbi_misaligned_v_st_emulator(int wlen, union sbi_ldst_data in_val,
+				 struct sbi_trap_context *tcntx);
+
+#endif

include/sbi/sbi_types.h

@@ -44,7 +44,12 @@ typedef unsigned long long	uint64_t;
 #error "Unexpected __riscv_xlen"
 #endif
 
-typedef int			bool;
+#if __STDC_VERSION__ < 202000L
+typedef _Bool			bool;
+#define true			1
+#define false			0
+#endif
+
 typedef unsigned long		ulong;
 typedef unsigned long		uintptr_t;
 typedef unsigned long		size_t;
@@ -61,15 +66,15 @@ typedef uint32_t		be32_t;
 typedef uint64_t		le64_t;
 typedef uint64_t		be64_t;
 
-#define true			1
-#define false			0
-
 #define NULL			((void *)0)
 
 #define __packed		__attribute__((packed))
 #define __noreturn		__attribute__((noreturn))
 #define __aligned(x)		__attribute__((aligned(x)))
+
+#ifndef __always_inline
 #define __always_inline	inline __attribute__((always_inline))
+#endif
 
 #define likely(x) __builtin_expect((x), 1)
 #define unlikely(x) __builtin_expect((x), 0)

include/sbi/sbi_unit_test.h

@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Author: Ivan Orlov <ivan.orlov0322@gmail.com>
+ */
+#ifdef CONFIG_SBIUNIT
+#ifndef __SBI_UNIT_H__
+#define __SBI_UNIT_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_string.h>
+
+struct sbiunit_test_case {
+	const char *name;
+	bool failed;
+	void (*test_func)(struct sbiunit_test_case *test);
+};
+
+struct sbiunit_test_suite {
+	const char *name;
+	void (*init)(void);
+	struct sbiunit_test_case *cases;
+};
+
+#define SBIUNIT_TEST_CASE(func)		\
+	{				\
+		.name = #func,		\
+		.failed = false,	\
+		.test_func = (func)	\
+	}
+
+#define SBIUNIT_END_CASE { }
+
+#define SBIUNIT_TEST_SUITE(suite_name, cases_arr)		\
+	struct sbiunit_test_suite suite_name = {		\
+		.name = #suite_name,				\
+		.init = NULL,					\
+		.cases = cases_arr				\
+	}
+
+#define _sbiunit_msg(test, msg) "[SBIUnit] [%s:%d]: %s: %s", __FILE__,	\
+				__LINE__, test->name, msg
+
+#define SBIUNIT_INFO(test, msg) sbi_printf(_sbiunit_msg(test, msg))
+#define SBIUNIT_PANIC(test, msg) sbi_panic(_sbiunit_msg(test, msg))
+
+#define SBIUNIT_EXPECT(test, cond) do {							\
+	if (!(cond)) {									\
+		test->failed = true;							\
+		SBIUNIT_INFO(test, "Condition \"" #cond "\" expected to be true!\n");	\
+	}										\
+} while (0)
+
+#define SBIUNIT_ASSERT(test, cond) do {						\
+	if (!(cond))								\
+		SBIUNIT_PANIC(test, "Condition \"" #cond "\" must be true!\n");	\
+} while (0)
+
+#define SBIUNIT_EXPECT_EQ(test, a, b) SBIUNIT_EXPECT(test, (a) == (b))
+#define SBIUNIT_ASSERT_EQ(test, a, b) SBIUNIT_ASSERT(test, (a) == (b))
+#define SBIUNIT_EXPECT_NE(test, a, b) SBIUNIT_EXPECT(test, (a) != (b))
+#define SBIUNIT_ASSERT_NE(test, a, b) SBIUNIT_ASSERT(test, (a) != (b))
+#define SBIUNIT_EXPECT_MEMEQ(test, a, b, len) SBIUNIT_EXPECT(test, !sbi_memcmp(a, b, len))
+#define SBIUNIT_ASSERT_MEMEQ(test, a, b, len) SBIUNIT_ASSERT(test, !sbi_memcmp(a, b, len))
+#define SBIUNIT_EXPECT_STREQ(test, a, b, len) SBIUNIT_EXPECT(test, !sbi_strncmp(a, b, len))
+#define SBIUNIT_ASSERT_STREQ(test, a, b, len) SBIUNIT_ASSERT(test, !sbi_strncmp(a, b, len))
+
+void run_all_tests(void);
+#endif
+#else
+#define run_all_tests()
+#endif

include/sbi/sbi_version.h

@@ -11,7 +11,7 @@
 #define __SBI_VERSION_H__
 
 #define OPENSBI_VERSION_MAJOR 1
-#define OPENSBI_VERSION_MINOR 4
+#define OPENSBI_VERSION_MINOR 6
 
 /**
  * OpenSBI 32-bit version with:

include/sbi_utils/cppc/fdt_cppc.h

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __FDT_CPPC_H__
+#define __FDT_CPPC_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi_utils/fdt/fdt_driver.h>
+
+#ifdef CONFIG_FDT_CPPC
+
+void fdt_cppc_init(const void *fdt);
+
+#else
+
+static inline void fdt_cppc_init(const void *fdt) { }
+
+#endif
+
+#endif

include/sbi_utils/fdt/fdt_domain.h

@@ -70,12 +70,12 @@ void fdt_domain_fixup(void *fdt);
  *
  * @return 0 on success and negative error code on failure
  */
-int fdt_domains_populate(void *fdt);
+int fdt_domains_populate(const void *fdt);
 
 #else
 
 static inline void fdt_domain_fixup(void *fdt) { }
-static inline int fdt_domains_populate(void *fdt) { return 0; }
+static inline int fdt_domains_populate(const void *fdt) { return 0; }
 
 #endif
 

include/sbi_utils/fdt/fdt_driver.h

@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * fdt_driver.h - Generic support for initializing drivers from DT nodes.
+ *
+ * Copyright (c) 2024 SiFive
+ */
+
+#ifndef __FDT_DRIVER_H__
+#define __FDT_DRIVER_H__
+
+#include <sbi_utils/fdt/fdt_helper.h>
+
+struct fdt_driver {
+	const struct fdt_match *match_table;
+	int (*init)(const void *fdt, int nodeoff,
+		    const struct fdt_match *match);
+	bool experimental;
+};
+
+/**
+ * Initialize a driver instance for a specific DT node
+ *
+ * @param fdt devicetree blob
+ * @param nodeoff offset of a node in the devicetree blob
+ * @param drivers NULL-terminated array of drivers to match against this node
+ *
+ * @return 0 if a driver was matched and successfully initialized or a negative
+ * error code on failure
+ */
+int fdt_driver_init_by_offset(const void *fdt, int nodeoff,
+			      const struct fdt_driver *const *drivers);
+
+/**
+ * Initialize a driver instance for each DT node that matches any of the
+ * provided drivers
+ *
+ * @param fdt devicetree blob
+ * @param drivers NULL-terminated array of drivers to match against each node
+ *
+ * @return 0 if drivers for all matches (if any) were successfully initialized
+ * or a negative error code on failure
+ */
+int fdt_driver_init_all(const void *fdt,
+			const struct fdt_driver *const *drivers);
+
+/**
+ * Initialize a driver instance for the first DT node that matches any of the
+ * provided drivers
+ *
+ * @param fdt devicetree blob
+ * @param drivers NULL-terminated array of drivers to match against each node
+ *
+ * @return 0 if a driver was matched and successfully initialized or a negative
+ * error code on failure
+ */
+int fdt_driver_init_one(const void *fdt,
+			const struct fdt_driver *const *drivers);
+
+#endif /* __FDT_DRIVER_H__ */

include/sbi_utils/fdt/fdt_fixup.h

@@ -9,6 +9,8 @@
 #ifndef __FDT_FIXUP_H__
 #define __FDT_FIXUP_H__
 
+#include <sbi/sbi_list.h>
+
 struct sbi_cpu_idle_state {
 	const char *name;
 	uint32_t suspend_param;
@@ -30,7 +32,7 @@ struct sbi_cpu_idle_state {
  * @param states: array of idle state descriptions, ending with empty element
  * @return zero on success and -ve on failure
  */
-int fdt_add_cpu_idle_states(void *dtb, const struct sbi_cpu_idle_state *state);
+int fdt_add_cpu_idle_states(void *fdt, const struct sbi_cpu_idle_state *state);
 
 /**
  * Fix up the CPU node in the device tree
@@ -93,6 +95,19 @@ void fdt_plic_fixup(void *fdt);
  */
 int fdt_reserved_memory_fixup(void *fdt);
 
+/** Representation of a general fixup */
+struct fdt_general_fixup {
+	struct sbi_dlist head;
+	const char *name;
+	void (*do_fixup)(struct fdt_general_fixup *f, void *fdt);
+};
+
+/** Register a general fixup */
+int fdt_register_general_fixup(struct fdt_general_fixup *fixup);
+
+/** UnRegister a general fixup */
+void fdt_unregister_general_fixup(struct fdt_general_fixup *fixup);
+
 /**
  * General device tree fix-up
  *

include/sbi_utils/fdt/fdt_helper.h

@@ -34,87 +34,92 @@ struct platform_uart_data {
 	unsigned long reg_offset;
 };
 
-const struct fdt_match *fdt_match_node(void *fdt, int nodeoff,
+const struct fdt_match *fdt_match_node(const void *fdt, int nodeoff,
 				       const struct fdt_match *match_table);
 
-int fdt_find_match(void *fdt, int startoff,
+int fdt_find_match(const void *fdt, int startoff,
 		   const struct fdt_match *match_table,
 		   const struct fdt_match **out_match);
 
-int fdt_parse_phandle_with_args(void *fdt, int nodeoff,
+int fdt_parse_phandle_with_args(const void *fdt, int nodeoff,
 				const char *prop, const char *cells_prop,
 				int index, struct fdt_phandle_args *out_args);
 
-int fdt_get_node_addr_size(void *fdt, int node, int index,
+int fdt_get_node_addr_size(const void *fdt, int node, int index,
 			   uint64_t *addr, uint64_t *size);
 
-int fdt_get_node_addr_size_by_name(void *fdt, int node, const char *name,
+int fdt_get_node_addr_size_by_name(const void *fdt, int node, const char *name,
 				   uint64_t *addr, uint64_t *size);
 
-bool fdt_node_is_enabled(void *fdt, int nodeoff);
+bool fdt_node_is_enabled(const void *fdt, int nodeoff);
 
-int fdt_parse_hart_id(void *fdt, int cpu_offset, u32 *hartid);
+int fdt_parse_hart_id(const void *fdt, int cpu_offset, u32 *hartid);
 
-int fdt_parse_max_enabled_hart_id(void *fdt, u32 *max_hartid);
+int fdt_parse_max_enabled_hart_id(const void *fdt, u32 *max_hartid);
 
-int fdt_parse_timebase_frequency(void *fdt, unsigned long *freq);
+int fdt_parse_timebase_frequency(const void *fdt, unsigned long *freq);
 
-int fdt_parse_isa_extensions(void *fdt, unsigned int hard_id,
+int fdt_parse_isa_extensions(const void *fdt, unsigned int hard_id,
 			     unsigned long *extensions);
 
-int fdt_parse_gaisler_uart_node(void *fdt, int nodeoffset,
+int fdt_parse_gaisler_uart_node(const void *fdt, int nodeoffset,
 				struct platform_uart_data *uart);
 
-int fdt_parse_renesas_scif_node(void *fdt, int nodeoffset,
+int fdt_parse_renesas_scif_node(const void *fdt, int nodeoffset,
 				struct platform_uart_data *uart);
 
-int fdt_parse_shakti_uart_node(void *fdt, int nodeoffset,
+int fdt_parse_shakti_uart_node(const void *fdt, int nodeoffset,
 			       struct platform_uart_data *uart);
 
-int fdt_parse_sifive_uart_node(void *fdt, int nodeoffset,
+int fdt_parse_sifive_uart_node(const void *fdt, int nodeoffset,
 			       struct platform_uart_data *uart);
 
-int fdt_parse_uart_node(void *fdt, int nodeoffset,
+int fdt_parse_uart_node(const void *fdt, int nodeoffset,
 			struct platform_uart_data *uart);
 
-int fdt_parse_uart8250(void *fdt, struct platform_uart_data *uart,
+int fdt_parse_uart8250(const void *fdt, struct platform_uart_data *uart,
 		       const char *compatible);
 
-int fdt_parse_xlnx_uartlite_node(void *fdt, int nodeoffset,
+int fdt_parse_xlnx_uartlite_node(const void *fdt, int nodeoffset,
 			       struct platform_uart_data *uart);
 
 struct aplic_data;
 
-int fdt_parse_aplic_node(void *fdt, int nodeoff, struct aplic_data *aplic);
+int fdt_parse_aplic_node(const void *fdt, int nodeoff, struct aplic_data *aplic);
 
 struct imsic_data;
 
-bool fdt_check_imsic_mlevel(void *fdt);
+bool fdt_check_imsic_mlevel(const void *fdt);
 
-int fdt_parse_imsic_node(void *fdt, int nodeoff, struct imsic_data *imsic);
+int fdt_parse_imsic_node(const void *fdt, int nodeoff, struct imsic_data *imsic);
 
 struct plic_data;
 
-int fdt_parse_plic_node(void *fdt, int nodeoffset, struct plic_data *plic);
+int fdt_parse_plic_node(const void *fdt, int nodeoffset, struct plic_data *plic);
 
-int fdt_parse_plic(void *fdt, struct plic_data *plic, const char *compat);
+int fdt_parse_plic(const void *fdt, struct plic_data *plic, const char *compat);
 
-int fdt_parse_aclint_node(void *fdt, int nodeoffset,
+int fdt_parse_aclint_node(const void *fdt, int nodeoffset,
 			  bool for_timer, bool allow_regname,
 			  unsigned long *out_addr1, unsigned long *out_size1,
 			  unsigned long *out_addr2, unsigned long *out_size2,
 			  u32 *out_first_hartid, u32 *out_hart_count);
 
-int fdt_parse_plmt_node(void *fdt, int nodeoffset, unsigned long *plmt_base,
+int fdt_parse_plmt_node(const void *fdt, int nodeoffset, unsigned long *plmt_base,
 			unsigned long *plmt_size, u32 *hart_count);
 
-int fdt_parse_plicsw_node(void *fdt, int nodeoffset, unsigned long *plicsw_base,
+int fdt_parse_plicsw_node(const void *fdt, int nodeoffset, unsigned long *plicsw_base,
 			  unsigned long *size, u32 *hart_count);
 
-int fdt_parse_compat_addr(void *fdt, uint64_t *addr,
+int fdt_parse_compat_addr(const void *fdt, uint64_t *addr,
 			  const char *compatible);
 
-static inline void *fdt_get_address(void)
+static inline const void *fdt_get_address(void)
+{
+	return (const void *)root.next_arg1;
+}
+
+static inline void *fdt_get_address_rw(void)
 {
 	return (void *)root.next_arg1;
 }

include/sbi_utils/fdt/fdt_pmu.h

@@ -52,7 +52,7 @@ int fdt_pmu_fixup(void *fdt);
  *
  * @return 0 on success and negative error code on failure
  */
-int fdt_pmu_setup(void *fdt);
+int fdt_pmu_setup(const void *fdt);
 
 /**
  * Get the mhpmevent select value read from DT for a given event
@@ -70,7 +70,7 @@ extern uint32_t hw_event_count;
 #else
 
 static inline void fdt_pmu_fixup(void *fdt) { }
-static inline int fdt_pmu_setup(void *fdt) { return 0; }
+static inline int fdt_pmu_setup(const void *fdt) { return 0; }
 static inline uint64_t fdt_pmu_get_select_value(uint32_t event_idx) { return 0; }
 
 #endif

include/sbi_utils/gpio/fdt_gpio.h

@@ -10,22 +10,21 @@
 #ifndef __FDT_GPIO_H__
 #define __FDT_GPIO_H__
 
+#include <sbi_utils/fdt/fdt_driver.h>
 #include <sbi_utils/gpio/gpio.h>
 
 struct fdt_phandle_args;
 
 /** FDT based GPIO driver */
 struct fdt_gpio {
-	const struct fdt_match *match_table;
+	struct fdt_driver driver;
 	int (*xlate)(struct gpio_chip *chip,
 		     const struct fdt_phandle_args *pargs,
 		     struct gpio_pin *out_pin);
-	int (*init)(void *fdt, int nodeoff, u32 phandle,
-		    const struct fdt_match *match);
 };
 
 /** Get a GPIO pin using "gpios" DT property of client DT node */
-int fdt_gpio_pin_get(void *fdt, int nodeoff, int index,
+int fdt_gpio_pin_get(const void *fdt, int nodeoff, int index,
 		     struct gpio_pin *out_pin);
 
 /** Simple xlate function to convert two GPIO FDT cells into GPIO pin */

include/sbi_utils/gpio/gpio.h

@@ -40,7 +40,7 @@ struct gpio_pin {
 /** Representation of a GPIO chip */
 struct gpio_chip {
 	/** Pointer to GPIO driver owning this GPIO chip */
-	void *driver;
+	const void *driver;
 	/** Uniquie ID of the GPIO chip assigned by the driver */
 	unsigned int id;
 	/** Number of GPIOs supported by the GPIO chip */

include/sbi_utils/hsm/fdt_hsm.h

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __FDT_HSM_H__
+#define __FDT_HSM_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi_utils/fdt/fdt_driver.h>
+
+#ifdef CONFIG_FDT_HSM
+
+void fdt_hsm_init(const void *fdt);
+
+#else
+
+static inline void fdt_hsm_init(const void *fdt) { }
+
+#endif
+
+#endif

include/sbi_utils/i2c/fdt_i2c.h

@@ -10,17 +10,11 @@
 #ifndef __FDT_I2C_H__
 #define __FDT_I2C_H__
 
+#include <sbi_utils/fdt/fdt_driver.h>
 #include <sbi_utils/i2c/i2c.h>
 
-/** FDT based I2C adapter driver */
-struct fdt_i2c_adapter {
-	const struct fdt_match *match_table;
-	int (*init)(void *fdt, int nodeoff,
-		    const struct fdt_match *match);
-};
-
 /** Get I2C adapter identified by nodeoff */
-int fdt_i2c_adapter_get(void *fdt, int nodeoff,
+int fdt_i2c_adapter_get(const void *fdt, int nodeoff,
 			struct i2c_adapter **out_adapter);
 
 #endif

include/sbi_utils/ipi/aclint_mswi.h

@@ -26,8 +26,6 @@ struct aclint_mswi_data {
 	u32 hart_count;
 };
 
-int aclint_mswi_warm_init(void);
-
 int aclint_mswi_cold_init(struct aclint_mswi_data *mswi);
 
 #endif

include/sbi_utils/ipi/andes_plicsw.h

@@ -32,8 +32,6 @@ struct plicsw_data {
 	uint32_t hart_count;
 };
 
-int plicsw_warm_ipi_init(void);
-
 int plicsw_cold_ipi_init(struct plicsw_data *plicsw);
 
 #endif /* _IPI_ANDES_PLICSW_H_ */

include/sbi_utils/ipi/fdt_ipi.h

@@ -11,24 +11,15 @@
 #define __FDT_IPI_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi_utils/fdt/fdt_driver.h>
 
 #ifdef CONFIG_FDT_IPI
 
-struct fdt_ipi {
-	const struct fdt_match *match_table;
-	int (*cold_init)(void *fdt, int nodeoff, const struct fdt_match *match);
-	int (*warm_init)(void);
-	void (*exit)(void);
-};
-
-void fdt_ipi_exit(void);
-
-int fdt_ipi_init(bool cold_boot);
+int fdt_ipi_init(void);
 
 #else
 
-static inline void fdt_ipi_exit(void) { }
-static inline int fdt_ipi_init(bool cold_boot) { return 0; }
+static inline int fdt_ipi_init(void) { return 0; }
 
 #endif
 

include/sbi_utils/irqchip/aplic.h

@@ -12,6 +12,7 @@
 #define __IRQCHIP_APLIC_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_irqchip.h>
 
 #define APLIC_MAX_DELEGATE	16
 
@@ -30,6 +31,9 @@ struct aplic_delegate_data {
 };
 
 struct aplic_data {
+	/* Private members */
+	struct sbi_irqchip_device irqchip;
+	/* Public members */
 	unsigned long addr;
 	unsigned long size;
 	unsigned long num_idc;

include/sbi_utils/irqchip/fdt_irqchip.h

@@ -16,20 +16,14 @@
 
 struct fdt_irqchip {
 	const struct fdt_match *match_table;
-	int (*cold_init)(void *fdt, int nodeoff, const struct fdt_match *match);
-	int (*warm_init)(void);
-	void (*exit)(void);
+	int (*cold_init)(const void *fdt, int nodeoff, const struct fdt_match *match);
 };
 
-void fdt_irqchip_exit(void);
-
-int fdt_irqchip_init(bool cold_boot);
+int fdt_irqchip_init(void);
 
 #else
 
-static inline void fdt_irqchip_exit(void) { }
-
-static inline int fdt_irqchip_init(bool cold_boot) { return 0; }
+static inline int fdt_irqchip_init(void) { return 0; }
 
 #endif
 

include/sbi_utils/irqchip/fdt_irqchip_plic.h

@@ -1,33 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2022 Samuel Holland <samuel@sholland.org>
- */
-
-#ifndef __IRQCHIP_FDT_IRQCHIP_PLIC_H__
-#define __IRQCHIP_FDT_IRQCHIP_PLIC_H__
-
-#include <sbi/sbi_types.h>
-
-/**
- * Save the PLIC priority state
- * @param priority pointer to the memory region for the saved priority
- * @param num size of the memory region including interrupt source 0
- */
-void fdt_plic_priority_save(u8 *priority, u32 num);
-
-/**
- * Restore the PLIC priority state
- * @param priority pointer to the memory region for the saved priority
- * @param num size of the memory region including interrupt source 0
- */
-void fdt_plic_priority_restore(const u8 *priority, u32 num);
-
-void fdt_plic_context_save(bool smode, u32 *enable, u32 *threshold, u32 num);
-
-void fdt_plic_context_restore(bool smode, const u32 *enable, u32 threshold,
-			      u32 num);
-
-void thead_plic_restore(void);
-
-#endif

include/sbi_utils/irqchip/imsic.h

@@ -37,14 +37,12 @@ struct imsic_data {
 
 int imsic_map_hartid_to_data(u32 hartid, struct imsic_data *imsic, int file);
 
-struct imsic_data *imsic_get_data(u32 hartid);
+struct imsic_data *imsic_get_data(u32 hartindex);
 
-int imsic_get_target_file(u32 hartid);
+int imsic_get_target_file(u32 hartindex);
 
 void imsic_local_irqchip_init(void);
 
-int imsic_warm_irqchip_init(void);
-
 int imsic_data_check(struct imsic_data *imsic);
 
 int imsic_cold_irqchip_init(struct imsic_data *imsic);

include/sbi_utils/irqchip/plic.h

@@ -11,31 +11,41 @@
 #define __IRQCHIP_PLIC_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_irqchip.h>
 
 struct plic_data {
+	/* Private members */
+	struct sbi_irqchip_device irqchip;
+	/* Public members */
 	unsigned long addr;
 	unsigned long size;
 	unsigned long num_src;
+	unsigned long flags;
+	void *pm_data;
+	s16 context_map[][2];
 };
 
-/* So far, priorities on all consumers of these functions fit in 8 bits. */
-void plic_priority_save(const struct plic_data *plic, u8 *priority, u32 num);
+/** Work around a bug on Ariane that requires enabling interrupts at boot */
+#define PLIC_FLAG_ARIANE_BUG		BIT(0)
+/** PLIC must be delegated to S-mode like T-HEAD C906 and C910 */
+#define PLIC_FLAG_THEAD_DELEGATION	BIT(1)
+/** Allocate space for power management save/restore operations */
+#define PLIC_FLAG_ENABLE_PM		BIT(2)
 
-void plic_priority_restore(const struct plic_data *plic, const u8 *priority,
-			   u32 num);
+#define PLIC_M_CONTEXT			0
+#define PLIC_S_CONTEXT			1
 
-void plic_context_save(const struct plic_data *plic, int context_id,
-		       u32 *enable, u32 *threshold, u32 num);
+#define PLIC_DATA_SIZE(__hart_count)	(sizeof(struct plic_data) + \
+					 (__hart_count) * 2 * sizeof(s16))
 
-void plic_context_restore(const struct plic_data *plic, int context_id,
-			  const u32 *enable, u32 threshold, u32 num);
+#define PLIC_IE_WORDS(__p)		((__p)->num_src / 32 + 1)
 
-int plic_context_init(const struct plic_data *plic, int context_id,
-		      bool enable, u32 threshold);
+struct plic_data *plic_get(void);
 
-int plic_warm_irqchip_init(const struct plic_data *plic,
-			   int m_cntx_id, int s_cntx_id);
+void plic_suspend(void);
 
-int plic_cold_irqchip_init(const struct plic_data *plic);
+void plic_resume(void);
+
+int plic_cold_irqchip_init(struct plic_data *plic);
 
 #endif

include/sbi_utils/mailbox/fdt_mailbox.h

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __FDT_MAILBOX_H__
+#define __FDT_MAILBOX_H__
+
+#include <sbi_utils/fdt/fdt_driver.h>
+#include <sbi_utils/mailbox/mailbox.h>
+
+struct fdt_phandle_args;
+
+/** FDT based mailbox driver */
+struct fdt_mailbox {
+	struct fdt_driver driver;
+	int (*xlate)(struct mbox_controller *mbox,
+		     const struct fdt_phandle_args *pargs,
+		     u32 *out_chan_args);
+};
+
+/** Request a mailbox channel using "mboxes" DT property of client DT node */
+int fdt_mailbox_request_chan(const void *fdt, int nodeoff, int index,
+			     struct mbox_chan **out_chan);
+
+/** Simple xlate function to convert one mailbox FDT cell into channel args */
+int fdt_mailbox_simple_xlate(struct mbox_controller *mbox,
+			     const struct fdt_phandle_args *pargs,
+			     u32 *out_chan_args);
+
+#endif

include/sbi_utils/mailbox/mailbox.h

@@ -0,0 +1,180 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __MAILBOX_H__
+#define __MAILBOX_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_list.h>
+#include <sbi/riscv_atomic.h>
+
+/** Representation of a mailbox channel */
+struct mbox_chan {
+	/** List head */
+	struct sbi_dlist node;
+	/** Pointer to the mailbox controller */
+	struct mbox_controller *mbox;
+	/**
+	 * Arguments (or parameters) to identify a mailbox channel
+	 * within a mailbox controller.
+	 */
+#define MBOX_CHAN_MAX_ARGS	2
+	u32 chan_args[MBOX_CHAN_MAX_ARGS];
+};
+
+#define to_mbox_chan(__node)	\
+	container_of((__node), struct mbox_chan, node)
+
+/**
+ * Representation of a mailbox data transfer
+ *
+ * NOTE: If both "tx" and "rx" are non-NULL then Tx is done before Rx.
+ */
+struct mbox_xfer {
+#define MBOX_XFER_SEQ			(1UL << 0)
+	/** Transfer flags */
+	unsigned long flags;
+	/** Transfer arguments (or parameters) */
+	void *args;
+	/**
+	 * Sequence number
+	 *
+	 * If MBOX_XFER_SEQ is not set in flags then mbox_chan_xfer()
+	 * will generate a unique sequence number and update this field
+	 * else mbox_chan_xfer() will blindly use the sequence number
+	 * specified by this field.
+	 */
+	long seq;
+	/** Send data pointer */
+	void *tx;
+	/** Send data length (valid only if tx != NULL) */
+	unsigned long tx_len;
+	/**
+	 * Send timeout milliseconds (valid only if tx != NULL)
+	 *
+	 * If this field is non-zero along with tx != NULL then the
+	 * mailbox controller driver will wait specified milliseconds
+	 * for send data transfer to complete else the mailbox controller
+	 * driver will not wait.
+	 */
+	unsigned long tx_timeout;
+	/** Receive data pointer */
+	void *rx;
+	/** Receive data length (valid only if rx != NULL) */
+	unsigned long rx_len;
+	/**
+	 * Receive timeout milliseconds (valid only if rx != NULL)
+	 *
+	 * If this field is non-zero along with rx != NULL then the
+	 * mailbox controller driver will wait specified milliseconds
+	 * for receive data transfer to complete else the mailbox
+	 * controller driver will not wait.
+	 */
+	unsigned long rx_timeout;
+};
+
+#define mbox_xfer_init_tx(__p, __a, __t, __t_len, __t_tim)		\
+do {									\
+	(__p)->flags = 0;						\
+	(__p)->args = (__a);						\
+	(__p)->tx = (__t);						\
+	(__p)->tx_len = (__t_len);					\
+	(__p)->tx_timeout = (__t_tim);					\
+	(__p)->rx = NULL;						\
+	(__p)->rx_len = 0;						\
+	(__p)->rx_timeout = 0;						\
+} while (0)
+
+#define mbox_xfer_init_rx(__p, __a, __r, __r_len, __r_tim)		\
+do {									\
+	(__p)->flags = 0;						\
+	(__p)->args = (__a);						\
+	(__p)->tx = NULL;						\
+	(__p)->tx_len = 0;						\
+	(__p)->tx_timeout = 0;						\
+	(__p)->rx = (__r);						\
+	(__p)->rx_len = (__r_len);					\
+	(__p)->rx_timeout = (__r_tim);					\
+} while (0)
+
+#define mbox_xfer_init_txrx(__p, __a, __t, __t_len, __t_tim, __r, __r_len, __r_tim)\
+do {									\
+	(__p)->flags = 0;						\
+	(__p)->args = (__a);						\
+	(__p)->tx = (__t);						\
+	(__p)->tx_len = (__t_len);					\
+	(__p)->tx_timeout = (__t_tim);					\
+	(__p)->rx = (__r);						\
+	(__p)->rx_len = (__r_len);					\
+	(__p)->rx_timeout = (__r_tim);					\
+} while (0)
+
+#define mbox_xfer_set_sequence(__p, __seq)				\
+do {									\
+	(__p)->flags |= MBOX_XFER_SEQ;					\
+	(__p)->seq = (__seq);						\
+} while (0)
+
+/** Representation of a mailbox controller */
+struct mbox_controller {
+	/** List head */
+	struct sbi_dlist node;
+	/** Next sequence atomic counter */
+	atomic_t xfer_next_seq;
+	/* List of mailbox channels */
+	struct sbi_dlist chan_list;
+	/** Unique ID of the mailbox controller assigned by the driver */
+	unsigned int id;
+	/** Maximum length of transfer supported by the mailbox controller */
+	unsigned int max_xfer_len;
+	/** Pointer to mailbox driver owning this mailbox controller */
+	void *driver;
+	/** Request a mailbox channel from the mailbox controller */
+	struct mbox_chan *(*request_chan)(struct mbox_controller *mbox,
+					  u32 *chan_args);
+	/** Free a mailbox channel from the mailbox controller */
+	void (*free_chan)(struct mbox_controller *mbox,
+			  struct mbox_chan *chan);
+	/** Transfer data over mailbox channel */
+	int (*xfer)(struct mbox_chan *chan, struct mbox_xfer *xfer);
+	/** Get an attribute of mailbox channel */
+	int (*get_attribute)(struct mbox_chan *chan, int attr_id, void *out_value);
+	/** Set an attribute of mailbox channel */
+	int (*set_attribute)(struct mbox_chan *chan, int attr_id, void *new_value);
+};
+
+#define to_mbox_controller(__node)	\
+	container_of((__node), struct mbox_controller, node)
+
+/** Find a registered mailbox controller */
+struct mbox_controller *mbox_controller_find(unsigned int id);
+
+/** Register mailbox controller */
+int mbox_controller_add(struct mbox_controller *mbox);
+
+/** Un-register mailbox controller */
+void mbox_controller_remove(struct mbox_controller *mbox);
+
+/** Request a mailbox channel */
+struct mbox_chan *mbox_controller_request_chan(struct mbox_controller *mbox,
+					       u32 *chan_args);
+
+/** Free a mailbox channel */
+void mbox_controller_free_chan(struct mbox_chan *chan);
+
+/** Data transfer over mailbox channel */
+int mbox_chan_xfer(struct mbox_chan *chan, struct mbox_xfer *xfer);
+
+/** Get an attribute of mailbox channel */
+int mbox_chan_get_attribute(struct mbox_chan *chan, int attr_id, void *out_value);
+
+/** Set an attribute of mailbox channel */
+int mbox_chan_set_attribute(struct mbox_chan *chan, int attr_id, void *new_value);
+
+#endif

include/sbi_utils/mailbox/rpmi_mailbox.h

@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __RPMI_MAILBOX_H__
+#define __RPMI_MAILBOX_H__
+
+#include <sbi/sbi_error.h>
+#include <sbi_utils/mailbox/rpmi_msgprot.h>
+
+#define rpmi_u32_count(__var)	(sizeof(__var) / sizeof(u32))
+
+/** Convert RPMI error to SBI error */
+int rpmi_xlate_error(enum rpmi_error error);
+
+/** Typical RPMI normal request with at least status code in response */
+int rpmi_normal_request_with_status(
+			struct mbox_chan *chan, u32 service_id,
+			void *req, u32 req_words, u32 req_endian_words,
+			void *resp, u32 resp_words, u32 resp_endian_words);
+
+/* RPMI posted request which is without any response*/
+int rpmi_posted_request(
+		struct mbox_chan *chan, u32 service_id,
+		void *req, u32 req_words, u32 req_endian_words);
+
+#endif /* !__RPMI_MAILBOX_H__ */

include/sbi_utils/mailbox/rpmi_msgprot.h

@@ -0,0 +1,607 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Rahul Pathak <rpathak@ventanamicro.com>
+ *   Subrahmanya Lingappa <slingappa@ventanamicro.com>
+ */
+
+#ifndef __RPMI_MSGPROT_H__
+#define __RPMI_MSGPROT_H__
+
+#include <sbi/sbi_byteorder.h>
+#include <sbi/sbi_error.h>
+
+/*
+ * 31                                            0
+ * +---------------------+-----------------------+
+ * | FLAGS | SERVICE_ID  |   SERVICEGROUP_ID     |
+ * +---------------------+-----------------------+
+ * |        TOKEN        |     DATA LENGTH       |
+ * +---------------------+-----------------------+
+ * |                 DATA/PAYLOAD                |
+ * +---------------------------------------------+
+ */
+
+/** Message Header byte offset */
+#define RPMI_MSG_HDR_OFFSET			(0x0)
+/** Message Header Size in bytes */
+#define RPMI_MSG_HDR_SIZE			(8)
+
+/** ServiceGroup ID field byte offset */
+#define RPMI_MSG_SERVICEGROUP_ID_OFFSET		(0x0)
+/** ServiceGroup ID field size in bytes */
+#define RPMI_MSG_SERVICEGROUP_ID_SIZE		(2)
+
+/** Service ID field byte offset */
+#define RPMI_MSG_SERVICE_ID_OFFSET		(0x2)
+/** Service ID field size in bytes */
+#define RPMI_MSG_SERVICE_ID_SIZE		(1)
+
+/** Flags field byte offset */
+#define RPMI_MSG_FLAGS_OFFSET			(0x3)
+/** Flags field size in bytes */
+#define RPMI_MSG_FLAGS_SIZE			(1)
+
+#define RPMI_MSG_FLAGS_TYPE_POS			(0U)
+#define RPMI_MSG_FLAGS_TYPE_MASK		0x7
+#define RPMI_MSG_FLAGS_TYPE			\
+	((0x7) << RPMI_MSG_FLAGS_TYPE_POS)
+
+#define RPMI_MSG_FLAGS_DOORBELL_POS		(3U)
+#define RPMI_MSG_FLAGS_DOORBELL_MASK		0x1
+#define RPMI_MSG_FLAGS_DOORBELL			\
+	((0x1) << RPMI_MSG_FLAGS_DOORBELL_POS)
+
+/** Data length field byte offset */
+#define RPMI_MSG_DATALEN_OFFSET			(0x4)
+/** Data length field size in bytes */
+#define RPMI_MSG_DATALEN_SIZE			(2)
+
+/** Token field byte offset */
+#define RPMI_MSG_TOKEN_OFFSET			(0x6)
+/** Token field size in bytes */
+#define RPMI_MSG_TOKEN_SIZE			(2)
+/** Token field mask */
+#define RPMI_MSG_TOKEN_MASK			(0xffffU)
+
+/** Data field byte offset */
+#define RPMI_MSG_DATA_OFFSET			(RPMI_MSG_HDR_SIZE)
+/** Data field size in bytes */
+#define RPMI_MSG_DATA_SIZE(__slot_size)		((__slot_size) - RPMI_MSG_HDR_SIZE)
+
+/** Minimum slot size in bytes */
+#define RPMI_SLOT_SIZE_MIN			(64)
+
+/** Name length of 16 characters */
+#define RPMI_NAME_CHARS_MAX			(16)
+
+/** Queue layout */
+#define RPMI_QUEUE_HEAD_SLOT		0
+#define RPMI_QUEUE_TAIL_SLOT		1
+#define RPMI_QUEUE_HEADER_SLOTS		2
+
+/** Default timeout values */
+#define RPMI_DEF_TX_TIMEOUT			20
+#define RPMI_DEF_RX_TIMEOUT			20
+
+/**
+ * Common macro to generate composite version from major
+ * and minor version numbers.
+ *
+ * RPMI has Specification version, Implementation version
+ * Service group versions which follow the same versioning
+ * encoding as below.
+ */
+#define RPMI_VERSION(__major, __minor) (((__major) << 16) | (__minor))
+
+/** RPMI Message Header */
+struct rpmi_message_header {
+	le16_t servicegroup_id;
+	uint8_t service_id;
+	uint8_t flags;
+	le16_t datalen;
+	le16_t token;
+} __packed;
+
+/** RPMI Message */
+struct rpmi_message {
+	struct rpmi_message_header header;
+	u8 data[0];
+} __packed;
+
+/** RPMI Messages Types */
+enum rpmi_message_type {
+	/* Normal request backed with ack */
+	RPMI_MSG_NORMAL_REQUEST = 0x0,
+	/* Request without any ack */
+	RPMI_MSG_POSTED_REQUEST = 0x1,
+	/* Acknowledgment for normal request message */
+	RPMI_MSG_ACKNOWLDGEMENT = 0x2,
+	/* Notification message */
+	RPMI_MSG_NOTIFICATION = 0x3,
+};
+
+/** RPMI Error Types */
+enum rpmi_error {
+	/* Success */
+	RPMI_SUCCESS		= 0,
+	/* General failure  */
+	RPMI_ERR_FAILED		= -1,
+	/* Service or feature not supported */
+	RPMI_ERR_NOTSUPP	= -2,
+	/* Invalid Parameter  */
+	RPMI_ERR_INVALID_PARAM    = -3,
+	/*
+	 * Denied to insufficient permissions
+	 * or due to unmet prerequisite
+	 */
+	RPMI_ERR_DENIED		= -4,
+	/* Invalid address or offset */
+	RPMI_ERR_INVALID_ADDR	= -5,
+	/*
+	 * Operation failed as it was already in
+	 * progress or the state has changed already
+	 * for which the operation was carried out.
+	 */
+	RPMI_ERR_ALREADY	= -6,
+	/*
+	 * Error in implementation which violates
+	 * the specification version
+	 */
+	RPMI_ERR_EXTENSION	= -7,
+	/* Operation failed due to hardware issues */
+	RPMI_ERR_HW_FAULT	= -8,
+	/* System, device or resource is busy */
+	RPMI_ERR_BUSY		= -9,
+	/* System or device or resource in invalid state */
+	RPMI_ERR_INVALID_STATE	= -10,
+	/* Index, offset or address is out of range */
+	RPMI_ERR_BAD_RANGE	= -11,
+	/* Operation timed out */
+	RPMI_ERR_TIMEOUT	= -12,
+	/*
+	 * Error in input or output or
+	 * error in sending or receiving data
+	 * through communication medium
+	 */
+	RPMI_ERR_IO		= -13,
+	/* No data available */
+	RPMI_ERR_NO_DATA	= -14,
+	RPMI_ERR_RESERVED_START	= -15,
+	RPMI_ERR_RESERVED_END	= -127,
+	RPMI_ERR_VENDOR_START	= -128,
+};
+
+/** RPMI Message Arguments */
+struct rpmi_message_args {
+	u32 flags;
+#define RPMI_MSG_FLAGS_NO_TX		(1U << 0)
+#define RPMI_MSG_FLAGS_NO_RX		(1U << 1)
+#define RPMI_MSG_FLAGS_NO_RX_TOKEN	(1U << 2)
+	enum rpmi_message_type type;
+	u8 service_id;
+	u32 tx_endian_words;
+	u32 rx_endian_words;
+	u16 rx_token;
+	u32 rx_data_len;
+};
+
+/*
+ * RPMI SERVICEGROUPS AND SERVICES
+ */
+
+/** RPMI ServiceGroups IDs */
+enum rpmi_servicegroup_id {
+	RPMI_SRVGRP_ID_MIN = 0,
+	RPMI_SRVGRP_BASE = 0x0001,
+	RPMI_SRVGRP_SYSTEM_RESET = 0x0002,
+	RPMI_SRVGRP_SYSTEM_SUSPEND = 0x0003,
+	RPMI_SRVGRP_HSM = 0x0004,
+	RPMI_SRVGRP_CPPC = 0x0005,
+	RPMI_SRVGRP_CLOCK = 0x0007,
+	RPMI_SRVGRP_ID_MAX_COUNT,
+
+	/* Reserved range for service groups */
+	RPMI_SRVGRP_RESERVE_START = RPMI_SRVGRP_ID_MAX_COUNT,
+	RPMI_SRVGRP_RESERVE_END = 0x7FFF,
+
+	/* Vendor/Implementation-specific service groups range */
+	RPMI_SRVGRP_VENDOR_START = 0x8000,
+	RPMI_SRVGRP_VENDOR_END = 0xFFFF,
+};
+
+/** RPMI enable notification request */
+struct rpmi_enable_notification_req {
+	u32 eventid;
+};
+
+/** RPMI enable notification response */
+struct rpmi_enable_notification_resp {
+	s32 status;
+};
+
+/** RPMI Base ServiceGroup Service IDs */
+enum rpmi_base_service_id {
+	RPMI_BASE_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_BASE_SRV_GET_IMPLEMENTATION_VERSION = 0x02,
+	RPMI_BASE_SRV_GET_IMPLEMENTATION_IDN = 0x03,
+	RPMI_BASE_SRV_GET_SPEC_VERSION = 0x04,
+	RPMI_BASE_SRV_GET_PLATFORM_INFO = 0x05,
+	RPMI_BASE_SRV_PROBE_SERVICE_GROUP = 0x06,
+	RPMI_BASE_SRV_GET_ATTRIBUTES = 0x07,
+	RPMI_BASE_SRV_SET_MSI = 0x08,
+};
+
+#define RPMI_BASE_FLAGS_F0_PRIVILEGE		(1U << 2)
+#define RPMI_BASE_FLAGS_F0_EV_NOTIFY		(1U << 1)
+#define RPMI_BASE_FLAGS_F0_MSI_EN		(1U)
+
+enum rpmi_base_context_priv_level {
+	RPMI_BASE_CONTEXT_PRIV_S_MODE,
+	RPMI_BASE_CONTEXT_PRIV_M_MODE,
+};
+
+struct rpmi_base_get_attributes_resp {
+	s32 status_code;
+	u32 f0;
+	u32 f1;
+	u32 f2;
+	u32 f3;
+};
+
+struct rpmi_base_get_platform_info_resp {
+	s32 status;
+	u32 plat_info_len;
+	char plat_info[];
+};
+
+/** RPMI System Reset ServiceGroup Service IDs */
+enum rpmi_system_reset_service_id {
+	RPMI_SYSRST_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_SYSRST_SRV_GET_ATTRIBUTES = 0x02,
+	RPMI_SYSRST_SRV_SYSTEM_RESET = 0x03,
+	RPMI_SYSRST_SRV_ID_MAX_COUNT,
+};
+
+/** RPMI System Reset types */
+enum rpmi_sysrst_reset_type {
+	RPMI_SYSRST_TYPE_SHUTDOWN = 0x0,
+	RPMI_SYSRST_TYPE_COLD_REBOOT = 0x1,
+	RPMI_SYSRST_TYPE_WARM_REBOOT = 0x2,
+	RPMI_SYSRST_TYPE_MAX,
+};
+
+#define RPMI_SYSRST_ATTRS_FLAGS_RESETTYPE_POS		(1)
+#define RPMI_SYSRST_ATTRS_FLAGS_RESETTYPE_MASK		\
+			(1U << RPMI_SYSRST_ATTRS_FLAGS_RESETTYPE_POS)
+
+/** Response for system reset attributes */
+struct rpmi_sysrst_get_reset_attributes_resp {
+	s32 status;
+	u32 flags;
+};
+
+/** RPMI System Suspend ServiceGroup Service IDs */
+enum rpmi_system_suspend_service_id {
+	RPMI_SYSSUSP_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_SYSSUSP_SRV_GET_ATTRIBUTES = 0x02,
+	RPMI_SYSSUSP_SRV_SYSTEM_SUSPEND = 0x03,
+	RPMI_SYSSUSP_SRV_ID_MAX_COUNT,
+};
+
+/** Request for system suspend attributes */
+struct rpmi_syssusp_get_attr_req {
+	u32 susp_type;
+};
+
+#define RPMI_SYSSUSP_ATTRS_FLAGS_RESUMEADDR	(1U << 1)
+#define RPMI_SYSSUSP_ATTRS_FLAGS_SUSPENDTYPE	1U
+
+/** Response for system suspend attributes */
+struct rpmi_syssusp_get_attr_resp {
+	s32 status;
+	u32 flags;
+};
+
+struct rpmi_syssusp_suspend_req {
+	u32 hartid;
+	u32 suspend_type;
+	u32 resume_addr_lo;
+	u32 resume_addr_hi;
+};
+
+struct rpmi_syssusp_suspend_resp {
+	s32 status;
+};
+
+/** RPMI HSM State Management ServiceGroup Service IDs */
+enum rpmi_hsm_service_id {
+	RPMI_HSM_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_HSM_SRV_GET_HART_STATUS = 0x02,
+	RPMI_HSM_SRV_GET_HART_LIST = 0x03,
+	RPMI_HSM_SRV_GET_SUSPEND_TYPES = 0x04,
+	RPMI_HSM_SRV_GET_SUSPEND_INFO = 0x05,
+	RPMI_HSM_SRV_HART_START = 0x06,
+	RPMI_HSM_SRV_HART_STOP = 0x07,
+	RPMI_HSM_SRV_HART_SUSPEND = 0x08,
+	RPMI_HSM_SRV_ID_MAX = 0x09,
+};
+
+/* HSM service group request and response structs */
+struct rpmi_hsm_hart_start_req {
+	u32 hartid;
+	u32 start_addr_lo;
+	u32 start_addr_hi;
+};
+
+struct rpmi_hsm_hart_start_resp {
+	s32 status;
+};
+
+struct rpmi_hsm_hart_stop_req {
+	u32 hartid;
+};
+
+struct rpmi_hsm_hart_stop_resp {
+	s32 status;
+};
+
+struct rpmi_hsm_hart_susp_req {
+	u32 hartid;
+	u32 suspend_type;
+	u32 resume_addr_lo;
+	u32 resume_addr_hi;
+};
+
+struct rpmi_hsm_hart_susp_resp {
+	s32 status;
+};
+
+struct rpmi_hsm_get_hart_status_req {
+	u32 hartid;
+};
+
+struct rpmi_hsm_get_hart_status_resp {
+	s32 status;
+	u32 hart_status;
+};
+
+struct rpmi_hsm_get_hart_list_req {
+	u32 start_index;
+};
+
+struct rpmi_hsm_get_hart_list_resp {
+	s32 status;
+	u32 remaining;
+	u32 returned;
+	/* remaining space need to be adjusted for the above 3 u32's */
+	u32 hartid[(RPMI_MSG_DATA_SIZE(RPMI_SLOT_SIZE_MIN) - (sizeof(u32) * 3)) / sizeof(u32)];
+};
+
+struct rpmi_hsm_get_susp_types_req {
+	u32 start_index;
+};
+
+struct rpmi_hsm_get_susp_types_resp {
+	s32 status;
+	u32 remaining;
+	u32 returned;
+	/* remaining space need to be adjusted for the above 3 u32's */
+	u32 types[(RPMI_MSG_DATA_SIZE(RPMI_SLOT_SIZE_MIN) - (sizeof(u32) * 3)) / sizeof(u32)];
+};
+
+struct rpmi_hsm_get_susp_info_req {
+	u32 suspend_type;
+};
+
+#define RPMI_HSM_SUSPEND_INFO_FLAGS_TIMER_STOP		1U
+
+struct rpmi_hsm_get_susp_info_resp {
+	s32 status;
+	u32 flags;
+	u32 entry_latency_us;
+	u32 exit_latency_us;
+	u32 wakeup_latency_us;
+	u32 min_residency_us;
+};
+
+/** RPMI CPPC ServiceGroup Service IDs */
+enum rpmi_cppc_service_id {
+	RPMI_CPPC_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_CPPC_SRV_PROBE_REG = 0x02,
+	RPMI_CPPC_SRV_READ_REG = 0x03,
+	RPMI_CPPC_SRV_WRITE_REG = 0x04,
+	RPMI_CPPC_SRV_GET_FAST_CHANNEL_REGION = 0x05,
+	RPMI_CPPC_SRV_GET_FAST_CHANNEL_OFFSET = 0x06,
+	RPMI_CPPC_SRV_GET_HART_LIST = 0x07,
+	RPMI_CPPC_SRV_MAX_COUNT,
+};
+
+struct rpmi_cppc_probe_req {
+	u32 hart_id;
+	u32 reg_id;
+};
+
+struct rpmi_cppc_probe_resp {
+	s32 status;
+	u32 reg_len;
+};
+
+struct rpmi_cppc_read_reg_req {
+	u32 hart_id;
+	u32 reg_id;
+};
+
+struct rpmi_cppc_read_reg_resp {
+	s32 status;
+	u32 data_lo;
+	u32 data_hi;
+};
+
+struct rpmi_cppc_write_reg_req {
+	u32 hart_id;
+	u32 reg_id;
+	u32 data_lo;
+	u32 data_hi;
+};
+
+struct rpmi_cppc_write_reg_resp {
+	s32 status;
+};
+
+struct rpmi_cppc_get_fastchan_offset_req {
+	u32 hart_id;
+};
+
+struct rpmi_cppc_get_fastchan_offset_resp {
+	s32 status;
+	u32 fc_perf_request_offset_lo;
+	u32 fc_perf_request_offset_hi;
+	u32 fc_perf_feedback_offset_lo;
+	u32 fc_perf_feedback_offset_hi;
+};
+
+#define RPMI_CPPC_FAST_CHANNEL_CPPC_MODE_POS		3
+#define RPMI_CPPC_FAST_CHANNEL_CPPC_MODE_MASK		\
+			(3U << RPMI_CPPC_FAST_CHANNEL_CPPC_MODE_POS)
+#define RPMI_CPPC_FAST_CHANNEL_FLAGS_DB_WIDTH_POS	1
+#define RPMI_CPPC_FAST_CHANNEL_FLAGS_DB_WIDTH_MASK	\
+			(3U << RPMI_CPPC_FAST_CHANNEL_FLAGS_DB_WIDTH_POS)
+#define RPMI_CPPC_FAST_CHANNEL_FLAGS_DB_SUPPORTED	(1U << 0)
+
+struct rpmi_cppc_get_fastchan_region_resp {
+	s32 status;
+	u32 flags;
+	u32 region_addr_lo;
+	u32 region_addr_hi;
+	u32 region_size_lo;
+	u32 region_size_hi;
+	u32 db_addr_lo;
+	u32 db_addr_hi;
+	u32 db_setmask_lo;
+	u32 db_setmask_hi;
+	u32 db_preservemask_lo;
+	u32 db_preservemask_hi;
+};
+
+enum rpmi_cppc_fast_channel_db_width {
+	RPMI_CPPC_FAST_CHANNEL_DB_WIDTH_8 = 0x0,
+	RPMI_CPPC_FAST_CHANNEL_DB_WIDTH_16 = 0x1,
+	RPMI_CPPC_FAST_CHANNEL_DB_WIDTH_32 = 0x2,
+	RPMI_CPPC_FAST_CHANNEL_DB_WIDTH_64 = 0x3,
+};
+
+enum rpmi_cppc_fast_channel_cppc_mode {
+	RPMI_CPPC_FAST_CHANNEL_CPPC_MODE_PASSIVE = 0x0,
+	RPMI_CPPC_FAST_CHANNEL_CPPC_MODE_ACTIVE = 0x1,
+	RPMI_CPPC_FAST_CHANNEL_CPPC_MODE_MAX_IDX,
+};
+
+struct rpmi_cppc_hart_list_req {
+	u32 start_index;
+};
+
+struct rpmi_cppc_hart_list_resp {
+	s32 status;
+	u32 remaining;
+	u32 returned;
+	/* remaining space need to be adjusted for the above 3 u32's */
+	u32 hartid[(RPMI_MSG_DATA_SIZE(RPMI_SLOT_SIZE_MIN) - (sizeof(u32) * 3)) / sizeof(u32)];
+};
+
+/** RPMI Clock ServiceGroup Service IDs */
+enum rpmi_clock_service_id {
+	RPMI_CLOCK_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_CLOCK_SRV_GET_NUM_CLOCKS = 0x02,
+	RPMI_CLOCK_SRV_GET_ATTRIBUTES = 0x03,
+	RPMI_CLOCK_SRV_GET_SUPPORTED_RATES = 0x04,
+	RPMI_CLOCK_SRV_SET_CONFIG = 0x05,
+	RPMI_CLOCK_SRV_GET_CONFIG = 0x06,
+	RPMI_CLOCK_SRV_SET_RATE = 0x07,
+	RPMI_CLOCK_SRV_GET_RATE = 0x08,
+	RPMI_CLOCK_SRV_MAX_COUNT,
+};
+
+struct rpmi_clock_get_num_clocks_resp {
+	s32 status;
+	u32 num_clocks;
+};
+
+struct rpmi_clock_get_attributes_req {
+	u32 clock_id;
+};
+
+struct rpmi_clock_get_attributes_resp {
+	s32 status;
+#define RPMI_CLOCK_FLAGS_FORMAT_POS		30
+#define RPMI_CLOCK_FLAGS_FORMAT_MASK		\
+			(3U << RPMI_CLOCK_FLAGS_CLOCK_FORMAT_POS)
+#define RPMI_CLOCK_FLAGS_FORMAT_DISCRETE	0
+#define RPMI_CLOCK_FLAGS_FORMAT_LINEAR		1
+	u32 flags;
+	u32 num_rates;
+	u32 transition_latency;
+	u8 name[16];
+};
+
+struct rpmi_clock_get_supported_rates_req {
+	u32 clock_id;
+	u32 clock_rate_index;
+};
+
+struct rpmi_clock_get_supported_rates_resp {
+	s32 status;
+	u32 flags;
+	u32 remaining;
+	u32 returned;
+	u32 clock_rate[0];
+};
+
+struct rpmi_clock_set_config_req {
+	u32 clock_id;
+#define RPMI_CLOCK_CONFIG_ENABLE		(1U << 0)
+	u32 config;
+};
+
+struct rpmi_clock_set_config_resp {
+	s32 status;
+};
+
+struct rpmi_clock_get_config_req {
+	u32 clock_id;
+};
+
+struct rpmi_clock_get_config_resp {
+	s32 status;
+	u32 config;
+};
+
+struct rpmi_clock_set_rate_req {
+	u32 clock_id;
+#define RPMI_CLOCK_SET_RATE_FLAGS_MASK		(3U << 0)
+#define RPMI_CLOCK_SET_RATE_FLAGS_ROUND_DOWN	0
+#define RPMI_CLOCK_SET_RATE_FLAGS_ROUND_UP	1
+#define RPMI_CLOCK_SET_RATE_FLAGS_ROUND_PLAT	2
+	u32 flags;
+	u32 clock_rate_low;
+	u32 clock_rate_high;
+};
+
+struct rpmi_clock_set_rate_resp {
+	s32 status;
+};
+
+struct rpmi_clock_get_rate_req {
+	u32 clock_id;
+};
+
+struct rpmi_clock_get_rate_resp {
+	s32 status;
+	u32 clock_rate_low;
+	u32 clock_rate_high;
+};
+
+#endif /* !__RPMI_MSGPROT_H__ */

include/sbi_utils/mpxy/fdt_mpxy.h

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __FDT_MPXY_H__
+#define __FDT_MPXY_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi_utils/fdt/fdt_driver.h>
+
+#ifdef CONFIG_FDT_MPXY
+
+void fdt_mpxy_init(const void *fdt);
+
+#else
+
+static inline void fdt_mpxy_init(const void *fdt) { }
+
+#endif
+
+#endif

include/sbi_utils/regmap/fdt_regmap.h

@@ -10,22 +10,16 @@
 #ifndef __FDT_REGMAP_H__
 #define __FDT_REGMAP_H__
 
+#include <sbi_utils/fdt/fdt_driver.h>
 #include <sbi_utils/regmap/regmap.h>
 
 struct fdt_phandle_args;
 
-/** FDT based regmap driver */
-struct fdt_regmap {
-	const struct fdt_match *match_table;
-	int (*init)(void *fdt, int nodeoff, u32 phandle,
-		    const struct fdt_match *match);
-};
-
 /** Get regmap instance based on phandle */
-int fdt_regmap_get_by_phandle(void *fdt, u32 phandle,
+int fdt_regmap_get_by_phandle(const void *fdt, u32 phandle,
 			      struct regmap **out_rmap);
 
-/** Get regmap instance based on "regmap' property of the specified DT node */
-int fdt_regmap_get(void *fdt, int nodeoff, struct regmap **out_rmap);
+/** Get regmap instance based on "regmap" property of the specified DT node */
+int fdt_regmap_get(const void *fdt, int nodeoff, struct regmap **out_rmap);
 
 #endif

include/sbi_utils/reset/fdt_reset.h

@@ -11,33 +11,20 @@
 #define __FDT_RESET_H__
 
 #include <sbi/sbi_types.h>
-
-struct fdt_reset {
-	const struct fdt_match *match_table;
-	int (*init)(void *fdt, int nodeoff, const struct fdt_match *match);
-};
+#include <sbi_utils/fdt/fdt_driver.h>
 
 #ifdef CONFIG_FDT_RESET
 
-/**
- * fdt_reset_driver_init() - initialize reset driver based on the device-tree
- */
-int fdt_reset_driver_init(void *fdt, struct fdt_reset *drv);
-
 /**
  * fdt_reset_init() - initialize reset drivers based on the device-tree
  *
  * This function shall be invoked in final init.
  */
-void fdt_reset_init(void);
+void fdt_reset_init(const void *fdt);
 
 #else
 
-static inline int fdt_reset_driver_init(void *fdt, struct fdt_reset *drv)
-{
-	return 0;
-}
-static inline void fdt_reset_init(void) { }
+static inline void fdt_reset_init(const void *fdt) { }
 
 #endif
 

include/sbi_utils/serial/fdt_serial.h

@@ -11,19 +11,15 @@
 #define __FDT_SERIAL_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi_utils/fdt/fdt_driver.h>
 
 #ifdef CONFIG_FDT_SERIAL
 
-struct fdt_serial {
-	const struct fdt_match *match_table;
-	int (*init)(void *fdt, int nodeoff, const struct fdt_match *match);
-};
-
-int fdt_serial_init(void);
+int fdt_serial_init(const void *fdt);
 
 #else
 
-static inline int fdt_serial_init(void) { return 0; }
+static inline int fdt_serial_init(const void *fdt) { return 0; }
 
 #endif
 

include/sbi_utils/serial/semihosting.h

@@ -38,10 +38,10 @@ enum semihosting_open_mode {
 
 #ifdef CONFIG_SERIAL_SEMIHOSTING
 int semihosting_init(void);
-int semihosting_enabled(void);
+bool semihosting_enabled(void);
 #else
 static inline int semihosting_init(void) { return SBI_ENODEV; }
-static inline int semihosting_enabled(void) { return 0; }
+static inline bool semihosting_enabled(void) { return false; }
 #endif
 
 #endif

include/sbi_utils/suspend/fdt_suspend.h

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __FDT_SUSPEND_H__
+#define __FDT_SUSPEND_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi_utils/fdt/fdt_driver.h>
+
+#ifdef CONFIG_FDT_SUSPEND
+
+void fdt_suspend_init(const void *fdt);
+
+#else
+
+static inline void fdt_suspend_init(const void *fdt) { }
+
+#endif
+
+#endif

include/sbi_utils/timer/aclint_mtimer.h

@@ -47,8 +47,6 @@ void aclint_mtimer_sync(struct aclint_mtimer_data *mt);
 void aclint_mtimer_set_reference(struct aclint_mtimer_data *mt,
 				 struct aclint_mtimer_data *ref);
 
-int aclint_mtimer_warm_init(void);
-
 int aclint_mtimer_cold_init(struct aclint_mtimer_data *mt,
 			    struct aclint_mtimer_data *reference);
 

include/sbi_utils/timer/andes_plmt.h

@@ -24,6 +24,5 @@ struct plmt_data {
 };
 
 int plmt_cold_timer_init(struct plmt_data *plmt);
-int plmt_warm_timer_init(void);
 
 #endif /* __TIMER_ANDES_PLMT_H__ */

include/sbi_utils/timer/fdt_timer.h

@@ -11,24 +11,15 @@
 #define __FDT_TIMER_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi_utils/fdt/fdt_driver.h>
 
 #ifdef CONFIG_FDT_TIMER
 
-struct fdt_timer {
-	const struct fdt_match *match_table;
-	int (*cold_init)(void *fdt, int nodeoff, const struct fdt_match *match);
-	int (*warm_init)(void);
-	void (*exit)(void);
-};
-
-void fdt_timer_exit(void);
-
-int fdt_timer_init(bool cold_boot);
+int fdt_timer_init(void);
 
 #else
 
-static inline void fdt_timer_exit(void) { }
-static inline int fdt_timer_init(bool cold_boot) { return 0; }
+static inline int fdt_timer_init(void) { return 0; }
 
 #endif
 

lib/sbi/Kconfig

@@ -1,6 +1,10 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
-menu "SBI Extension Support"
+menu "Generic SBI Support"
+
+config CONSOLE_EARLY_BUFFER_SIZE
+	int "Early console buffer size (bytes)"
+	default 256
 
 config SBI_ECALL_TIME
 	bool "Timer extension"
@@ -38,6 +42,10 @@ config SBI_ECALL_CPPC
 	bool "CPPC extension"
 	default y
 
+config SBI_ECALL_FWFT
+	bool "Firmware Feature extension"
+	default y
+
 config SBI_ECALL_LEGACY
 	bool "SBI v0.1 legacy extensions"
 	default y
@@ -46,4 +54,19 @@ config SBI_ECALL_VENDOR
 	bool "Platform-defined vendor extensions"
 	default y
 
+config SBI_ECALL_DBTR
+	bool "Debug Trigger Extension"
+	default y
+
+config SBIUNIT
+	bool "Enable SBIUNIT tests"
+	default n
+
+config SBI_ECALL_SSE
+	bool "SSE extension"
+	default y
+
+config SBI_ECALL_MPXY
+	bool "MPXY extension"
+	default y
 endmenu

lib/sbi/objects.mk

@@ -13,7 +13,7 @@ libsbi-objs-y += riscv_hardfp.o
 libsbi-objs-y += riscv_locks.o
 
 libsbi-objs-y += sbi_ecall.o
-libsbi-objs-y += sbi_ecall_exts.o
+libsbi-objs-y += sbi_ecall_exts.carray.o
 
 # The order of below extensions is performance optimized
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_TIME) += ecall_time
@@ -46,18 +46,34 @@ libsbi-objs-$(CONFIG_SBI_ECALL_DBCN) += sbi_ecall_dbcn.o
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_CPPC) += ecall_cppc
 libsbi-objs-$(CONFIG_SBI_ECALL_CPPC) += sbi_ecall_cppc.o
 
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_FWFT) += ecall_fwft
+libsbi-objs-$(CONFIG_SBI_ECALL_FWFT) += sbi_ecall_fwft.o
+
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_LEGACY) += ecall_legacy
 libsbi-objs-$(CONFIG_SBI_ECALL_LEGACY) += sbi_ecall_legacy.o
 
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_VENDOR) += ecall_vendor
 libsbi-objs-$(CONFIG_SBI_ECALL_VENDOR) += sbi_ecall_vendor.o
 
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_DBTR) += ecall_dbtr
+libsbi-objs-$(CONFIG_SBI_ECALL_DBTR) += sbi_ecall_dbtr.o
+
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_SSE) += ecall_sse
+libsbi-objs-$(CONFIG_SBI_ECALL_SSE) += sbi_ecall_sse.o
+
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_MPXY) += ecall_mpxy
+libsbi-objs-$(CONFIG_SBI_ECALL_MPXY) += sbi_ecall_mpxy.o
+
 libsbi-objs-y += sbi_bitmap.o
 libsbi-objs-y += sbi_bitops.o
 libsbi-objs-y += sbi_console.o
+libsbi-objs-y += sbi_domain_context.o
+libsbi-objs-y += sbi_domain_data.o
 libsbi-objs-y += sbi_domain.o
+libsbi-objs-y += sbi_double_trap.o
 libsbi-objs-y += sbi_emulate_csr.o
 libsbi-objs-y += sbi_fifo.o
+libsbi-objs-y += sbi_fwft.o
 libsbi-objs-y += sbi_hart.o
 libsbi-objs-y += sbi_heap.o
 libsbi-objs-y += sbi_math.o
@@ -67,15 +83,19 @@ libsbi-objs-y += sbi_illegal_insn.o
 libsbi-objs-y += sbi_init.o
 libsbi-objs-y += sbi_ipi.o
 libsbi-objs-y += sbi_irqchip.o
-libsbi-objs-y += sbi_misaligned_ldst.o
 libsbi-objs-y += sbi_platform.o
 libsbi-objs-y += sbi_pmu.o
+libsbi-objs-y += sbi_dbtr.o
+libsbi-objs-y += sbi_mpxy.o
 libsbi-objs-y += sbi_scratch.o
+libsbi-objs-y += sbi_sse.o
 libsbi-objs-y += sbi_string.o
 libsbi-objs-y += sbi_system.o
 libsbi-objs-y += sbi_timer.o
 libsbi-objs-y += sbi_tlb.o
 libsbi-objs-y += sbi_trap.o
+libsbi-objs-y += sbi_trap_ldst.o
+libsbi-objs-y += sbi_trap_v_ldst.o
 libsbi-objs-y += sbi_unpriv.o
 libsbi-objs-y += sbi_expected_trap.o
 libsbi-objs-y += sbi_cppc.o

lib/sbi/riscv_asm.c

@@ -291,7 +291,8 @@ int is_pmp_entry_mapped(unsigned long entry)
 	unsigned long addr;
 	unsigned long log2len;
 
-	pmp_get(entry, &prot, &addr, &log2len);
+	if (pmp_get(entry, &prot, &addr, &log2len) != 0)
+		return false;
 
 	/* If address matching bits are non-zero, the entry is enable */
 	if (prot & PMP_A)

lib/sbi/sbi_console.c

@@ -9,6 +9,7 @@
 
 #include <sbi/riscv_locks.h>
 #include <sbi/sbi_console.h>
+#include <sbi/sbi_fifo.h>
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_scratch.h>
@@ -21,6 +22,15 @@ static char console_tbuf[CONSOLE_TBUF_MAX];
 static u32 console_tbuf_len;
 static spinlock_t console_out_lock	       = SPIN_LOCK_INITIALIZER;
 
+#ifdef CONFIG_CONSOLE_EARLY_BUFFER_SIZE
+#define CONSOLE_EARLY_BUFFER_SIZE	CONFIG_CONSOLE_EARLY_BUFFER_SIZE
+#else
+#define CONSOLE_EARLY_BUFFER_SIZE	256
+#endif
+static char console_early_buffer[CONSOLE_EARLY_BUFFER_SIZE] = { 0 };
+static SBI_FIFO_DEFINE(console_early_fifo, console_early_buffer, \
+		       CONSOLE_EARLY_BUFFER_SIZE, sizeof(char));
+
 bool sbi_isprintable(char c)
 {
 	if (((31 < c) && (c < 127)) || (c == '\f') || (c == '\r') ||
@@ -39,6 +49,7 @@ int sbi_getc(void)
 
 static unsigned long nputs(const char *str, unsigned long len)
 {
+	char ch;
 	unsigned long i;
 
 	if (console_dev) {
@@ -51,6 +62,11 @@ static unsigned long nputs(const char *str, unsigned long len)
 				console_dev->console_putc(str[i]);
 			}
 		}
+	} else {
+		for (i = 0; i < len; i++) {
+			ch = str[i];
+			sbi_fifo_enqueue(&console_early_fifo, &ch, true);
+		}
 	}
 	return len;
 }
@@ -472,19 +488,19 @@ const struct sbi_console_device *sbi_console_get_device(void)
 
 void sbi_console_set_device(const struct sbi_console_device *dev)
 {
-	if (!dev || console_dev)
+	char ch;
+	bool flush_early_fifo = false;
+
+	if (!dev)
 		return;
 
+	if (!console_dev)
+		flush_early_fifo = true;
+
 	console_dev = dev;
+
+	if (flush_early_fifo) {
+		while (!sbi_fifo_dequeue(&console_early_fifo, &ch))
+			sbi_putc(ch);
 	}
-
-int sbi_console_init(struct sbi_scratch *scratch)
-{
-	int rc = sbi_platform_console_init(sbi_platform_ptr(scratch));
-
-	/* console is not a necessary device */
-	if (rc == SBI_ENODEV)
-		return 0;
-
-	return rc;
 }

lib/sbi/sbi_dbtr.c

@@ -0,0 +1,710 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems, Inc.
+ *
+ * Author(s):
+ *   Himanshu Chauhan <hchauhan@ventanamicro.com>
+ */
+
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_csr_detect.h>
+#include <sbi/sbi_platform.h>
+#include <sbi/sbi_byteorder.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_dbtr.h>
+#include <sbi/sbi_heap.h>
+#include <sbi/riscv_encoding.h>
+#include <sbi/riscv_asm.h>
+
+
+/** Offset of pointer to HART's debug triggers info in scratch space */
+static unsigned long hart_state_ptr_offset;
+
+#define dbtr_get_hart_state_ptr(__scratch)				\
+	sbi_scratch_read_type((__scratch), void *, hart_state_ptr_offset)
+
+#define dbtr_thishart_state_ptr()				\
+	dbtr_get_hart_state_ptr(sbi_scratch_thishart_ptr())
+
+#define dbtr_set_hart_state_ptr(__scratch, __hart_state)		\
+	sbi_scratch_write_type((__scratch), void *, hart_state_ptr_offset, \
+			       (__hart_state))
+
+#define INDEX_TO_TRIGGER(_index)					\
+	({								\
+		struct sbi_dbtr_trigger *__trg = NULL;			\
+		struct sbi_dbtr_hart_triggers_state *__hs = NULL;	\
+		__hs = dbtr_get_hart_state_ptr(sbi_scratch_thishart_ptr()); \
+		__trg = &__hs->triggers[_index];			\
+		(__trg);						\
+	})
+
+#define for_each_trig_entry(_base, _max, _etype, _entry)		\
+	for (int _idx = 0; _entry = ((_etype *)_base + _idx),		\
+	     _idx < _max;						\
+	     _idx++, _entry = ((_etype *)_base + _idx))
+
+#define DBTR_SHMEM_MAKE_PHYS(_p_hi, _p_lo) (_p_lo)
+
+/* must call with hs != NULL */
+static inline bool sbi_dbtr_shmem_disabled(
+	struct sbi_dbtr_hart_triggers_state *hs)
+{
+	return (hs->shmem.phys_lo == SBI_DBTR_SHMEM_INVALID_ADDR &&
+		hs->shmem.phys_hi == SBI_DBTR_SHMEM_INVALID_ADDR
+		? true : false);
+}
+
+/* must call with hs != NULL */
+static inline void sbi_dbtr_disable_shmem(
+	struct sbi_dbtr_hart_triggers_state *hs)
+{
+	hs->shmem.phys_lo = SBI_DBTR_SHMEM_INVALID_ADDR;
+	hs->shmem.phys_hi = SBI_DBTR_SHMEM_INVALID_ADDR;
+}
+
+/* must call with hs which is not disabled */
+static inline void *hart_shmem_base(
+	struct sbi_dbtr_hart_triggers_state *hs)
+{
+	return ((void *)(unsigned long)DBTR_SHMEM_MAKE_PHYS(
+			hs->shmem.phys_hi, hs->shmem.phys_lo));
+}
+
+static void sbi_trigger_init(struct sbi_dbtr_trigger *trig,
+			     unsigned long type_mask, unsigned long idx)
+{
+	trig->type_mask = type_mask;
+	trig->state = 0;
+	trig->tdata1 = 0;
+	trig->tdata2 = 0;
+	trig->tdata3 = 0;
+	trig->index = idx;
+}
+
+static inline struct sbi_dbtr_trigger *sbi_alloc_trigger(void)
+{
+	int i;
+	struct sbi_dbtr_trigger *f_trig = NULL;
+	struct sbi_dbtr_hart_triggers_state *hart_state;
+
+	hart_state = dbtr_thishart_state_ptr();
+	if (!hart_state)
+		return NULL;
+
+	if (hart_state->available_trigs <= 0)
+		return NULL;
+
+	for (i = 0; i < hart_state->total_trigs; i++) {
+		f_trig = INDEX_TO_TRIGGER(i);
+		if (f_trig->state & RV_DBTR_BIT_MASK(TS, MAPPED))
+			continue;
+		hart_state->available_trigs--;
+		break;
+	}
+
+	if (i == hart_state->total_trigs)
+		return NULL;
+
+	__set_bit(RV_DBTR_BIT(TS, MAPPED), &f_trig->state);
+
+	return f_trig;
+}
+
+static inline void sbi_free_trigger(struct sbi_dbtr_trigger *trig)
+{
+	struct sbi_dbtr_hart_triggers_state *hart_state;
+
+	if (trig == NULL)
+		return;
+
+	hart_state = dbtr_thishart_state_ptr();
+	if (!hart_state)
+		return;
+
+	trig->state = 0;
+	trig->tdata1 = 0;
+	trig->tdata2 = 0;
+	trig->tdata3 = 0;
+
+	hart_state->available_trigs++;
+}
+
+int sbi_dbtr_init(struct sbi_scratch *scratch, bool coldboot)
+{
+	struct sbi_trap_info trap = {0};
+	unsigned long tdata1;
+	unsigned long val;
+	int i;
+	struct sbi_dbtr_hart_triggers_state *hart_state = NULL;
+
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SDTRIG))
+		return SBI_SUCCESS;
+
+	if (coldboot) {
+		hart_state_ptr_offset = sbi_scratch_alloc_type_offset(void *);
+		if (!hart_state_ptr_offset)
+			return SBI_ENOMEM;
+	}
+
+	hart_state = dbtr_get_hart_state_ptr(scratch);
+	if (!hart_state) {
+		hart_state = sbi_zalloc(sizeof(*hart_state));
+		if (!hart_state)
+			return SBI_ENOMEM;
+		hart_state->hartid = current_hartid();
+		dbtr_set_hart_state_ptr(scratch, hart_state);
+	}
+
+	/* disable the shared memory */
+	sbi_dbtr_disable_shmem(hart_state);
+
+	/* Skip probing triggers if already probed */
+	if (hart_state->probed)
+		goto _probed;
+
+	for (i = 0; i < RV_MAX_TRIGGERS; i++) {
+		csr_write_allowed(CSR_TSELECT, &trap, i);
+		if (trap.cause)
+			break;
+
+		val = csr_read_allowed(CSR_TSELECT, &trap);
+		if (trap.cause)
+			break;
+
+		/*
+		 * Read back tselect and check that it contains the
+		 * written value
+		 */
+		if (val != i)
+			break;
+
+		val = csr_read_allowed(CSR_TINFO, &trap);
+		if (trap.cause) {
+			/*
+			 * If reading tinfo caused an exception, the
+			 * debugger must read tdata1 to discover the
+			 * type.
+			 */
+			tdata1 = csr_read_allowed(CSR_TDATA1,
+						  &trap);
+			if (trap.cause)
+				break;
+
+			if (TDATA1_GET_TYPE(tdata1) == 0)
+				break;
+
+			sbi_trigger_init(INDEX_TO_TRIGGER(i),
+					 BIT(TDATA1_GET_TYPE(tdata1)),
+					 i);
+			hart_state->total_trigs++;
+		} else {
+			if (val == 1)
+				break;
+
+			sbi_trigger_init(INDEX_TO_TRIGGER(i), val, i);
+			hart_state->total_trigs++;
+		}
+	}
+
+	hart_state->probed = 1;
+
+ _probed:
+	hart_state->available_trigs = hart_state->total_trigs;
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_get_total_triggers(void)
+{
+	struct sbi_dbtr_hart_triggers_state *hs;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	/*
+	 * This function may be used during ecall registration.
+	 * By that time the debug trigger module might not be
+	 * initialized. If the extension is not supported, report
+	 * number of triggers as 0.
+	 */
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SDTRIG))
+		return 0;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return 0;
+
+	return hs->total_trigs;
+}
+
+int sbi_dbtr_setup_shmem(const struct sbi_domain *dom, unsigned long smode,
+			 unsigned long shmem_phys_lo,
+			 unsigned long shmem_phys_hi)
+{
+	struct sbi_dbtr_hart_triggers_state *hart_state;
+
+	if (dom && !sbi_domain_is_assigned_hart(dom, current_hartindex())) {
+		sbi_dprintf("%s: calling hart not assigned to this domain\n",
+			   __func__);
+		return SBI_ERR_DENIED;
+	}
+
+	hart_state = dbtr_thishart_state_ptr();
+	if (!hart_state)
+		return SBI_ERR_FAILED;
+
+	/* call is to disable shared memory */
+	if (shmem_phys_lo == SBI_DBTR_SHMEM_INVALID_ADDR
+	    && shmem_phys_hi == SBI_DBTR_SHMEM_INVALID_ADDR) {
+		sbi_dbtr_disable_shmem(hart_state);
+		return SBI_SUCCESS;
+	}
+
+	/* the shared memory must be disabled on this hart */
+	if (!sbi_dbtr_shmem_disabled(hart_state))
+		return SBI_ERR_ALREADY_AVAILABLE;
+
+	/* lower physical address must be XLEN/8 bytes aligned */
+	if (shmem_phys_lo & SBI_DBTR_SHMEM_ALIGN_MASK)
+		return SBI_ERR_INVALID_PARAM;
+
+	/*
+	* On RV32, the M-mode can only access the first 4GB of
+	* the physical address space because M-mode does not have
+	* MMU to access full 34-bit physical address space.
+	* So fail if the upper 32 bits of the physical address
+	* is non-zero on RV32.
+	*
+	* On RV64, kernel sets upper 64bit address part to zero.
+	* So fail if the upper 64bit of the physical address
+	* is non-zero on RV64.
+	*/
+	if (shmem_phys_hi)
+		return SBI_EINVALID_ADDR;
+
+	if (dom && !sbi_domain_check_addr(dom,
+		  DBTR_SHMEM_MAKE_PHYS(shmem_phys_hi, shmem_phys_lo), smode,
+		  SBI_DOMAIN_READ | SBI_DOMAIN_WRITE))
+		return SBI_ERR_INVALID_ADDRESS;
+
+	hart_state->shmem.phys_lo = shmem_phys_lo;
+	hart_state->shmem.phys_hi = shmem_phys_hi;
+
+	return SBI_SUCCESS;
+}
+
+static void dbtr_trigger_setup(struct sbi_dbtr_trigger *trig,
+			       struct sbi_dbtr_data_msg *recv)
+{
+	unsigned long tdata1;
+
+	if (!trig)
+		return;
+
+	trig->tdata1 = lle_to_cpu(recv->tdata1);
+	trig->tdata2 = lle_to_cpu(recv->tdata2);
+	trig->tdata3 = lle_to_cpu(recv->tdata3);
+
+	tdata1 = lle_to_cpu(recv->tdata1);
+
+	trig->state = 0;
+
+	__set_bit(RV_DBTR_BIT(TS, MAPPED), &trig->state);
+
+	SET_TRIG_HW_INDEX(trig->state, trig->index);
+
+	switch (TDATA1_GET_TYPE(tdata1)) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		if (__test_bit(RV_DBTR_BIT(MC, U), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, U), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC, S), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, S), &trig->state);
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		if (__test_bit(RV_DBTR_BIT(MC6, U), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, U), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC6, S), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, S), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC6, VU), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, VU), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC6, VS), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, VS), &trig->state);
+		break;
+	default:
+		sbi_dprintf("%s: Unknown type (tdata1: 0x%lx Type: %ld)\n",
+			    __func__, tdata1, TDATA1_GET_TYPE(tdata1));
+		break;
+	}
+}
+
+static inline void update_bit(unsigned long new, int nr, volatile unsigned long *addr)
+{
+	if (new)
+		__set_bit(nr, addr);
+	else
+		__clear_bit(nr, addr);
+}
+
+static void dbtr_trigger_enable(struct sbi_dbtr_trigger *trig)
+{
+	unsigned long state;
+	unsigned long tdata1;
+
+	if (!trig || !(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+		return;
+
+	state = trig->state;
+	tdata1 = trig->tdata1;
+
+	switch (TDATA1_GET_TYPE(tdata1)) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		update_bit(state & RV_DBTR_BIT_MASK(TS, U),
+			   RV_DBTR_BIT(MC, U), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, S),
+			   RV_DBTR_BIT(MC, S), &trig->tdata1);
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		update_bit(state & RV_DBTR_BIT_MASK(TS, VU),
+			   RV_DBTR_BIT(MC6, VU), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, VS),
+			   RV_DBTR_BIT(MC6, VS), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, U),
+			   RV_DBTR_BIT(MC6, U), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, S),
+			   RV_DBTR_BIT(MC6, S), &trig->tdata1);
+		break;
+	default:
+		break;
+	}
+
+	/*
+	 * RISC-V Debug Support v1.0.0 section 5.5:
+	 * Debugger cannot simply set a trigger by writing tdata1, then tdata2,
+	 * etc. The current value of tdata2 might not be legal with the new
+	 * value of tdata1. To help with this situation, it is guaranteed that
+	 * writing 0 to tdata1 disables the trigger, and leaves it in a state
+	 * where tdata2 and tdata3 can be written with any value that makes
+	 * sense for any trigger type supported by this trigger.
+	 */
+	csr_write(CSR_TSELECT, trig->index);
+	csr_write(CSR_TDATA1, 0x0);
+	csr_write(CSR_TDATA2, trig->tdata2);
+	csr_write(CSR_TDATA1, trig->tdata1);
+}
+
+static void dbtr_trigger_disable(struct sbi_dbtr_trigger *trig)
+{
+	unsigned long tdata1;
+
+	if (!trig || !(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+		return;
+
+	tdata1 = trig->tdata1;
+
+	switch (TDATA1_GET_TYPE(tdata1)) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		__clear_bit(RV_DBTR_BIT(MC, U), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC, S), &trig->tdata1);
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		__clear_bit(RV_DBTR_BIT(MC6, VU), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC6, VS), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC6, U), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC6, S), &trig->tdata1);
+		break;
+	default:
+		break;
+	}
+
+	csr_write(CSR_TSELECT, trig->index);
+	csr_write(CSR_TDATA1, trig->tdata1);
+}
+
+static void dbtr_trigger_clear(struct sbi_dbtr_trigger *trig)
+{
+	if (!trig || !(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+		return;
+
+	csr_write(CSR_TSELECT, trig->index);
+	csr_write(CSR_TDATA1, 0x0);
+	csr_write(CSR_TDATA2, 0x0);
+}
+
+static int dbtr_trigger_supported(unsigned long type)
+{
+	switch (type) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		return 1;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int dbtr_trigger_valid(unsigned long type, unsigned long tdata)
+{
+	switch (type) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		if (!(tdata & RV_DBTR_BIT_MASK(MC, DMODE)) &&
+		    !(tdata & RV_DBTR_BIT_MASK(MC, M)))
+			return 1;
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		if (!(tdata & RV_DBTR_BIT_MASK(MC6, DMODE)) &&
+		    !(tdata & RV_DBTR_BIT_MASK(MC6, M)))
+			return 1;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+int sbi_dbtr_num_trig(unsigned long data, unsigned long *out)
+{
+	unsigned long type = TDATA1_GET_TYPE(data);
+	u32 hartid = current_hartid();
+	unsigned long total = 0;
+	struct sbi_dbtr_trigger *trig;
+	int i;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (data == 0) {
+		*out = hs->total_trigs;
+		return SBI_SUCCESS;
+	}
+
+	for (i = 0; i < hs->total_trigs; i++) {
+		trig = INDEX_TO_TRIGGER(i);
+
+		if (__test_bit(type, &trig->type_mask))
+			total++;
+	}
+
+	sbi_dprintf("%s: hart%d: total triggers of type %lu: %lu\n",
+		    __func__, hartid, type, total);
+
+	*out = total;
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_read_trig(unsigned long smode,
+		       unsigned long trig_idx_base, unsigned long trig_count)
+{
+	struct sbi_dbtr_data_msg *xmit;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_shmem_entry *entry;
+	void *shmem_base = NULL;
+	struct sbi_dbtr_hart_triggers_state *hs = NULL;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (trig_idx_base >= hs->total_trigs ||
+	    trig_idx_base + trig_count >= hs->total_trigs)
+		return SBI_ERR_INVALID_PARAM;
+
+	if (sbi_dbtr_shmem_disabled(hs))
+		return SBI_ERR_NO_SHMEM;
+
+	shmem_base = hart_shmem_base(hs);
+
+	for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) {
+		sbi_hart_map_saddr((unsigned long)entry, sizeof(*entry));
+		xmit = &entry->data;
+		trig = INDEX_TO_TRIGGER((_idx + trig_idx_base));
+		xmit->tstate = cpu_to_lle(trig->state);
+		xmit->tdata1 = cpu_to_lle(trig->tdata1);
+		xmit->tdata2 = cpu_to_lle(trig->tdata2);
+		xmit->tdata3 = cpu_to_lle(trig->tdata3);
+		sbi_hart_unmap_saddr();
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_install_trig(unsigned long smode,
+			  unsigned long trig_count, unsigned long *out)
+{
+	void *shmem_base = NULL;
+	struct sbi_dbtr_shmem_entry *entry;
+	struct sbi_dbtr_data_msg *recv;
+	struct sbi_dbtr_id_msg *xmit;
+	unsigned long ctrl;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs = NULL;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (sbi_dbtr_shmem_disabled(hs))
+		return SBI_ERR_NO_SHMEM;
+
+	shmem_base = hart_shmem_base(hs);
+
+	/* Check requested triggers configuration */
+	for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) {
+		sbi_hart_map_saddr((unsigned long)entry, sizeof(*entry));
+		recv = (struct sbi_dbtr_data_msg *)(&entry->data);
+		ctrl = recv->tdata1;
+
+		if (!dbtr_trigger_supported(TDATA1_GET_TYPE(ctrl))) {
+			*out = _idx;
+			sbi_hart_unmap_saddr();
+			return SBI_ERR_FAILED;
+		}
+
+		if (!dbtr_trigger_valid(TDATA1_GET_TYPE(ctrl), ctrl)) {
+			*out = _idx;
+			sbi_hart_unmap_saddr();
+			return SBI_ERR_FAILED;
+		}
+		sbi_hart_unmap_saddr();
+	}
+
+	if (hs->available_trigs < trig_count) {
+		*out = hs->available_trigs;
+		return SBI_ERR_FAILED;
+	}
+
+	/* Install triggers */
+	for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) {
+		/*
+		 * Since we have already checked if enough triggers are
+		 * available, trigger allocation must succeed.
+		 */
+		trig = sbi_alloc_trigger();
+
+		sbi_hart_map_saddr((unsigned long)entry, sizeof(*entry));
+
+		recv = (struct sbi_dbtr_data_msg *)(&entry->data);
+		xmit = (struct sbi_dbtr_id_msg *)(&entry->id);
+
+		dbtr_trigger_setup(trig,  recv);
+		dbtr_trigger_enable(trig);
+		xmit->idx = cpu_to_lle(trig->index);
+		sbi_hart_unmap_saddr();
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_uninstall_trig(unsigned long trig_idx_base,
+			    unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+		if (!(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+			return SBI_ERR_INVALID_PARAM;
+
+		dbtr_trigger_clear(trig);
+
+		sbi_free_trigger(trig);
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_enable_trig(unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+		sbi_dprintf("%s: enable trigger %lu\n", __func__, idx);
+		dbtr_trigger_enable(trig);
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_update_trig(unsigned long smode,
+			 unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_data_msg *recv;
+	unsigned long uidx = 0;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_shmem_entry *entry;
+	void *shmem_base = NULL;
+	struct sbi_dbtr_hart_triggers_state *hs = NULL;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (sbi_dbtr_shmem_disabled(hs))
+		return SBI_ERR_NO_SHMEM;
+
+	shmem_base = hart_shmem_base(hs);
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+
+		if (!(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+			return SBI_ERR_INVALID_PARAM;
+
+		entry = (shmem_base + uidx * sizeof(*entry));
+		recv = &entry->data;
+
+		trig->tdata2 = lle_to_cpu(recv->tdata2);
+		dbtr_trigger_enable(trig);
+		uidx++;
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_disable_trig(unsigned long trig_idx_base,
+			  unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+		dbtr_trigger_disable(trig);
+	}
+
+	return SBI_SUCCESS;
+}

lib/sbi/sbi_domain.c

@@ -13,20 +13,18 @@
 #include <sbi/sbi_hartmask.h>
 #include <sbi/sbi_heap.h>
 #include <sbi/sbi_hsm.h>
+#include <sbi/sbi_list.h>
 #include <sbi/sbi_math.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
 
-/*
- * We allocate an extra element because sbi_domain_for_each() expects
- * the array to be null-terminated.
- */
-struct sbi_domain *domidx_to_domain_table[SBI_DOMAIN_MAX_INDEX + 1] = { 0 };
+SBI_LIST_HEAD(domain_list);
+
 static u32 domain_count = 0;
 static bool domain_finalized = false;
 
-#define ROOT_REGION_MAX	16
+#define ROOT_REGION_MAX	32
 static u32 root_memregs_count = 0;
 
 struct sbi_domain root = {
@@ -51,7 +49,7 @@ struct sbi_domain *sbi_hartindex_to_domain(u32 hartindex)
 	return sbi_scratch_read_type(scratch, void *, domain_hart_ptr_offset);
 }
 
-static void update_hartindex_to_domain(u32 hartindex, struct sbi_domain *dom)
+void sbi_update_hartindex_to_domain(u32 hartindex, struct sbi_domain *dom)
 {
 	struct sbi_scratch *scratch;
 
@@ -62,23 +60,36 @@ static void update_hartindex_to_domain(u32 hartindex, struct sbi_domain *dom)
 	sbi_scratch_write_type(scratch, void *, domain_hart_ptr_offset, dom);
 }
 
-bool sbi_domain_is_assigned_hart(const struct sbi_domain *dom, u32 hartid)
+bool sbi_domain_is_assigned_hart(const struct sbi_domain *dom, u32 hartindex)
 {
-	if (dom)
-		return sbi_hartmask_test_hartid(hartid, &dom->assigned_harts);
+	bool ret;
+	struct sbi_domain *tdom = (struct sbi_domain *)dom;
 
+	if (!dom)
 		return false;
+
+	spin_lock(&tdom->assigned_harts_lock);
+	ret = sbi_hartmask_test_hartindex(hartindex, &tdom->assigned_harts);
+	spin_unlock(&tdom->assigned_harts_lock);
+
+	return ret;
 }
 
-ulong sbi_domain_get_assigned_hartmask(const struct sbi_domain *dom,
-				       ulong hbase)
+int sbi_domain_get_assigned_hartmask(const struct sbi_domain *dom,
+				     struct sbi_hartmask *mask)
 {
 	ulong ret = 0;
-	for (int i = 0; i < 8 * sizeof(ret); i++) {
-		if (sbi_domain_is_assigned_hart(dom, hbase + i))
-			ret |= 1UL << i;
+	struct sbi_domain *tdom = (struct sbi_domain *)dom;
+
+	if (!dom) {
+		sbi_hartmask_clear_all(mask);
+		return 0;
 	}
 
+	spin_lock(&tdom->assigned_harts_lock);
+	sbi_hartmask_copy(mask, &tdom->assigned_harts);
+	spin_unlock(&tdom->assigned_harts_lock);
+
 	return ret;
 }
 
@@ -435,7 +446,7 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
 		j = sbi_hartindex_to_hartid(i);
 		sbi_printf("%s%d%s", (k++) ? "," : "",
-			   j, sbi_domain_is_assigned_hart(dom, j) ? "*" : "");
+			   j, sbi_domain_is_assigned_hart(dom, i) ? "*" : "");
 	}
 	sbi_printf("\n");
 
@@ -505,10 +516,9 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 
 void sbi_domain_dump_all(const char *suffix)
 {
-	u32 i;
 	const struct sbi_domain *dom;
 
-	sbi_domain_for_each(i, dom) {
+	sbi_domain_for_each(dom) {
 		sbi_domain_dump(dom, suffix);
 		sbi_printf("\n");
 	}
@@ -527,21 +537,11 @@ int sbi_domain_register(struct sbi_domain *dom,
 		return SBI_EINVAL;
 
 	/* Check if domain already discovered */
-	sbi_domain_for_each(i, tdom) {
+	sbi_domain_for_each(tdom) {
 		if (tdom == dom)
 			return SBI_EALREADY;
 	}
 
-	/*
-	 * Ensure that we have room for Domain Index to
-	 * HART ID mapping
-	 */
-	if (SBI_DOMAIN_MAX_INDEX <= domain_count) {
-		sbi_printf("%s: No room for %s\n",
-			   __func__, dom->name);
-		return SBI_ENOSPC;
-	}
-
 	/* Sanitize discovered domain */
 	rc = sanitize_domain(dom);
 	if (rc) {
@@ -551,9 +551,13 @@ int sbi_domain_register(struct sbi_domain *dom,
 		return rc;
 	}
 
+	sbi_list_add_tail(&dom->node, &domain_list);
+
 	/* Assign index to domain */
 	dom->index = domain_count++;
-	domidx_to_domain_table[dom->index] = dom;
+
+	/* Initialize spinlock for dom->assigned_harts */
+	SPIN_LOCK_INIT(dom->assigned_harts_lock);
 
 	/* Clear assigned HARTs of domain */
 	sbi_hartmask_clear_all(&dom->assigned_harts);
@@ -567,7 +571,7 @@ int sbi_domain_register(struct sbi_domain *dom,
 		if (tdom)
 			sbi_hartmask_clear_hartindex(i,
 					&tdom->assigned_harts);
-		update_hartindex_to_domain(i, dom);
+		sbi_update_hartindex_to_domain(i, dom);
 		sbi_hartmask_set_hartindex(i, &dom->assigned_harts);
 
 		/*
@@ -582,10 +586,19 @@ int sbi_domain_register(struct sbi_domain *dom,
 		}
 	}
 
+	/* Setup data for the discovered domain */
+	rc = sbi_domain_setup_data(dom);
+	if (rc) {
+		sbi_printf("%s: domain data setup failed for %s (error %d)\n",
+			   __func__, dom->name, rc);
+		sbi_list_del(&dom->node);
+		return rc;
+	}
+
 	return 0;
 }
 
-int sbi_domain_root_add_memregion(const struct sbi_domain_memregion *reg)
+static int root_add_memregion(const struct sbi_domain_memregion *reg)
 {
 	int rc;
 	bool reg_merged;
@@ -663,7 +676,7 @@ int sbi_domain_root_add_memrange(unsigned long addr, unsigned long size,
 				(end - pos) : align;
 
 		sbi_domain_memregion_init(pos, rsize, region_flags, &reg);
-		rc = sbi_domain_root_add_memregion(&reg);
+		rc = root_add_memregion(&reg);
 		if (rc)
 			return rc;
 		pos += rsize;
@@ -675,7 +688,7 @@ int sbi_domain_root_add_memrange(unsigned long addr, unsigned long size,
 int sbi_domain_finalize(struct sbi_scratch *scratch, u32 cold_hartid)
 {
 	int rc;
-	u32 i, dhart;
+	u32 dhart;
 	struct sbi_domain *dom;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
@@ -688,7 +701,7 @@ int sbi_domain_finalize(struct sbi_scratch *scratch, u32 cold_hartid)
 	}
 
 	/* Startup boot HART of domains */
-	sbi_domain_for_each(i, dom) {
+	sbi_domain_for_each(dom) {
 		/* Domain boot HART index */
 		dhart = sbi_hartid_to_hartindex(dom->boot_hartid);
 
@@ -701,8 +714,14 @@ int sbi_domain_finalize(struct sbi_scratch *scratch, u32 cold_hartid)
 			continue;
 
 		/* Ignore if boot HART assigned different domain */
-		if (sbi_hartindex_to_domain(dhart) != dom ||
-		    !sbi_hartmask_test_hartindex(dhart, &dom->assigned_harts))
+		if (sbi_hartindex_to_domain(dhart) != dom)
+			continue;
+
+		/* Ignore if boot HART is not part of the assigned HARTs */
+		spin_lock(&dom->assigned_harts_lock);
+		rc = sbi_hartmask_test_hartindex(dhart, &dom->assigned_harts);
+		spin_unlock(&dom->assigned_harts_lock);
+		if (!rc)
 			continue;
 
 		/* Startup boot HART of domain */
@@ -742,6 +761,8 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid)
 	struct sbi_domain_memregion *root_memregs;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
+	SBI_INIT_LIST_HEAD(&domain_list);
+
 	if (scratch->fw_rw_offset == 0 ||
 	    (scratch->fw_rw_offset & (scratch->fw_rw_offset - 1)) != 0) {
 		sbi_printf("%s: fw_rw_offset is not a power of 2 (0x%lx)\n",
@@ -759,11 +780,16 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid)
 	if (!domain_hart_ptr_offset)
 		return SBI_ENOMEM;
 
+	/* Initialize domain context support */
+	rc = sbi_domain_context_init();
+	if (rc)
+		goto fail_free_domain_hart_ptr_offset;
+
 	root_memregs = sbi_calloc(sizeof(*root_memregs), ROOT_REGION_MAX + 1);
 	if (!root_memregs) {
 		sbi_printf("%s: no memory for root regions\n", __func__);
 		rc = SBI_ENOMEM;
-		goto fail_free_domain_hart_ptr_offset;
+		goto fail_deinit_context;
 	}
 	root.regions = root_memregs;
 
@@ -828,6 +854,8 @@ fail_free_root_hmask:
 	sbi_free(root_hmask);
 fail_free_root_memregs:
 	sbi_free(root_memregs);
+fail_deinit_context:
+	sbi_domain_context_deinit();
 fail_free_domain_hart_ptr_offset:
 	sbi_scratch_free_offset(domain_hart_ptr_offset);
 	return rc;

lib/sbi/sbi_domain_context.c

@@ -0,0 +1,241 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) IPADS@SJTU 2023. All rights reserved.
+ */
+
+#include <sbi/sbi_error.h>
+#include <sbi/riscv_locks.h>
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_hsm.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_heap.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_domain_context.h>
+#include <sbi/sbi_trap.h>
+
+/** Context representation for a hart within a domain */
+struct hart_context {
+	/** Trap-related states such as GPRs, mepc, and mstatus */
+	struct sbi_trap_context trap_ctx;
+
+	/** Supervisor status register */
+	unsigned long sstatus;
+	/** Supervisor interrupt enable register */
+	unsigned long sie;
+	/** Supervisor trap vector base address register */
+	unsigned long stvec;
+	/** Supervisor scratch register for temporary storage */
+	unsigned long sscratch;
+	/** Supervisor exception program counter register */
+	unsigned long sepc;
+	/** Supervisor cause register */
+	unsigned long scause;
+	/** Supervisor trap value register */
+	unsigned long stval;
+	/** Supervisor interrupt pending register */
+	unsigned long sip;
+	/** Supervisor address translation and protection register */
+	unsigned long satp;
+	/** Counter-enable register */
+	unsigned long scounteren;
+	/** Supervisor environment configuration register */
+	unsigned long senvcfg;
+
+	/** Reference to the owning domain */
+	struct sbi_domain *dom;
+	/** Previous context (caller) to jump to during context exits */
+	struct hart_context *prev_ctx;
+	/** Is context initialized and runnable */
+	bool initialized;
+};
+
+struct domain_context_priv {
+	/** Contexts for possible HARTs indexed by hartindex */
+	struct hart_context *hartindex_to_context_table[SBI_HARTMASK_MAX_BITS];
+};
+
+static struct sbi_domain_data dcpriv = {
+	.data_size = sizeof(struct domain_context_priv),
+};
+
+static inline struct hart_context *hart_context_get(struct sbi_domain *dom,
+						    u32 hartindex)
+{
+	struct domain_context_priv *dcp = sbi_domain_data_ptr(dom, &dcpriv);
+
+	return (dcp && hartindex < SBI_HARTMASK_MAX_BITS) ?
+		dcp->hartindex_to_context_table[hartindex] : NULL;
+}
+
+static void hart_context_set(struct sbi_domain *dom, u32 hartindex,
+			     struct hart_context *hc)
+{
+	struct domain_context_priv *dcp = sbi_domain_data_ptr(dom, &dcpriv);
+
+	if (dcp && hartindex < SBI_HARTMASK_MAX_BITS)
+		dcp->hartindex_to_context_table[hartindex] = hc;
+}
+
+/** Macro to obtain the current hart's context pointer */
+#define hart_context_thishart_get()					\
+	hart_context_get(sbi_domain_thishart_ptr(),			\
+			 current_hartindex())
+
+/**
+ * Switches the HART context from the current domain to the target domain.
+ * This includes changing domain assignments and reconfiguring PMP, as well
+ * as saving and restoring CSRs and trap states.
+ *
+ * @param ctx pointer to the current HART context
+ * @param dom_ctx pointer to the target domain context
+ */
+static void switch_to_next_domain_context(struct hart_context *ctx,
+					  struct hart_context *dom_ctx)
+{
+	u32 hartindex = current_hartindex();
+	struct sbi_trap_context *trap_ctx;
+	struct sbi_domain *current_dom = ctx->dom;
+	struct sbi_domain *target_dom = dom_ctx->dom;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	unsigned int pmp_count = sbi_hart_pmp_count(scratch);
+
+	/* Assign current hart to target domain */
+	spin_lock(&current_dom->assigned_harts_lock);
+	sbi_hartmask_clear_hartindex(hartindex, &current_dom->assigned_harts);
+	spin_unlock(&current_dom->assigned_harts_lock);
+
+	sbi_update_hartindex_to_domain(hartindex, target_dom);
+
+	spin_lock(&target_dom->assigned_harts_lock);
+	sbi_hartmask_set_hartindex(hartindex, &target_dom->assigned_harts);
+	spin_unlock(&target_dom->assigned_harts_lock);
+
+	/* Reconfigure PMP settings for the new domain */
+	for (int i = 0; i < pmp_count; i++) {
+		pmp_disable(i);
+	}
+	sbi_hart_pmp_configure(scratch);
+
+	/* Save current CSR context and restore target domain's CSR context */
+	ctx->sstatus	= csr_swap(CSR_SSTATUS, dom_ctx->sstatus);
+	ctx->sie	= csr_swap(CSR_SIE, dom_ctx->sie);
+	ctx->stvec	= csr_swap(CSR_STVEC, dom_ctx->stvec);
+	ctx->sscratch	= csr_swap(CSR_SSCRATCH, dom_ctx->sscratch);
+	ctx->sepc	= csr_swap(CSR_SEPC, dom_ctx->sepc);
+	ctx->scause	= csr_swap(CSR_SCAUSE, dom_ctx->scause);
+	ctx->stval	= csr_swap(CSR_STVAL, dom_ctx->stval);
+	ctx->sip	= csr_swap(CSR_SIP, dom_ctx->sip);
+	ctx->satp	= csr_swap(CSR_SATP, dom_ctx->satp);
+	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_10)
+		ctx->scounteren = csr_swap(CSR_SCOUNTEREN, dom_ctx->scounteren);
+	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_12)
+		ctx->senvcfg	= csr_swap(CSR_SENVCFG, dom_ctx->senvcfg);
+
+	/* Save current trap state and restore target domain's trap state */
+	trap_ctx = sbi_trap_get_context(scratch);
+	sbi_memcpy(&ctx->trap_ctx, trap_ctx, sizeof(*trap_ctx));
+	sbi_memcpy(trap_ctx, &dom_ctx->trap_ctx, sizeof(*trap_ctx));
+
+	/* Mark current context structure initialized because context saved */
+	ctx->initialized = true;
+
+	/* If target domain context is not initialized or runnable */
+	if (!dom_ctx->initialized) {
+		/* Startup boot HART of target domain */
+		if (current_hartid() == target_dom->boot_hartid)
+			sbi_hart_switch_mode(target_dom->boot_hartid,
+					     target_dom->next_arg1,
+					     target_dom->next_addr,
+					     target_dom->next_mode,
+					     false);
+		else
+			sbi_hsm_hart_stop(scratch, true);
+	}
+}
+
+int sbi_domain_context_enter(struct sbi_domain *dom)
+{
+	struct hart_context *ctx = hart_context_thishart_get();
+	struct hart_context *dom_ctx = hart_context_get(dom, current_hartindex());
+
+	/* Validate the domain context existence */
+	if (!dom_ctx)
+		return SBI_EINVAL;
+
+	/* Update target context's previous context to indicate the caller */
+	dom_ctx->prev_ctx = ctx;
+
+	switch_to_next_domain_context(ctx, dom_ctx);
+
+	return 0;
+}
+
+int sbi_domain_context_exit(void)
+{
+	u32 hartindex = current_hartindex();
+	struct sbi_domain *dom;
+	struct hart_context *ctx = hart_context_thishart_get();
+	struct hart_context *dom_ctx, *tmp;
+
+	/*
+	 * If it's first time to call `exit` on the current hart, no
+	 * context allocated before. Loop through each domain to allocate
+	 * its context on the current hart if valid.
+	 */
+	if (!ctx) {
+		sbi_domain_for_each(dom) {
+			if (!sbi_hartmask_test_hartindex(hartindex,
+							 dom->possible_harts))
+				continue;
+
+			dom_ctx = sbi_zalloc(sizeof(struct hart_context));
+			if (!dom_ctx)
+				return SBI_ENOMEM;
+
+			/* Bind context and domain */
+			dom_ctx->dom = dom;
+			hart_context_set(dom, hartindex, dom_ctx);
+		}
+
+		ctx = hart_context_thishart_get();
+	}
+
+	dom_ctx = ctx->prev_ctx;
+
+	/* If no previous caller context */
+	if (!dom_ctx) {
+		/* Try to find next uninitialized user-defined domain's context */
+		sbi_domain_for_each(dom) {
+			if (dom == &root || dom == sbi_domain_thishart_ptr())
+				continue;
+
+			tmp = hart_context_get(dom, hartindex);
+			if (tmp && !tmp->initialized) {
+				dom_ctx = tmp;
+				break;
+			}
+		}
+	}
+
+	/* Take the root domain context if fail to find */
+	if (!dom_ctx)
+		dom_ctx = hart_context_get(&root, hartindex);
+
+	switch_to_next_domain_context(ctx, dom_ctx);
+
+	return 0;
+}
+
+int sbi_domain_context_init(void)
+{
+	return sbi_domain_register_data(&dcpriv);
+}
+
+void sbi_domain_context_deinit(void)
+{
+	sbi_domain_unregister_data(&dcpriv);
+}

lib/sbi/sbi_domain_data.c

@@ -0,0 +1,138 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Ventana Micro Systems Inc.
+ */
+
+#include <sbi/sbi_bitmap.h>
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_heap.h>
+
+static SBI_LIST_HEAD(data_list);
+static DECLARE_BITMAP(data_idx_bmap, SBI_DOMAIN_MAX_DATA_PTRS);
+
+void *sbi_domain_data_ptr(struct sbi_domain *dom, struct sbi_domain_data *data)
+{
+	if (dom && data && data->data_idx < SBI_DOMAIN_MAX_DATA_PTRS)
+		return dom->data_priv.idx_to_data_ptr[data->data_idx];
+
+	return NULL;
+}
+
+static int domain_setup_data_one(struct sbi_domain *dom,
+				 struct sbi_domain_data *data)
+{
+	struct sbi_domain_data_priv *priv = &dom->data_priv;
+	void *data_ptr;
+	int rc;
+
+	if (priv->idx_to_data_ptr[data->data_idx])
+		return SBI_EALREADY;
+
+	data_ptr = sbi_zalloc(data->data_size);
+	if (!data_ptr) {
+		sbi_domain_cleanup_data(dom);
+		return SBI_ENOMEM;
+	}
+
+	if (data->data_setup) {
+		rc = data->data_setup(dom, data, data_ptr);
+		if (rc) {
+			sbi_free(data_ptr);
+			return rc;
+		}
+	}
+
+	priv->idx_to_data_ptr[data->data_idx] = data_ptr;
+	return 0;
+}
+
+static void domain_cleanup_data_one(struct sbi_domain *dom,
+				    struct sbi_domain_data *data)
+{
+	struct sbi_domain_data_priv *priv = &dom->data_priv;
+	void *data_ptr;
+
+	data_ptr = priv->idx_to_data_ptr[data->data_idx];
+	if (!data_ptr)
+		return;
+
+	if (data->data_cleanup)
+		data->data_cleanup(dom, data, data_ptr);
+
+	sbi_free(data_ptr);
+	priv->idx_to_data_ptr[data->data_idx] = NULL;
+}
+
+int sbi_domain_setup_data(struct sbi_domain *dom)
+{
+	struct sbi_domain_data *data;
+	int rc;
+
+	if (!dom)
+		return SBI_EINVAL;
+
+	sbi_list_for_each_entry(data, &data_list, head) {
+		rc = domain_setup_data_one(dom, data);
+		if (rc) {
+			sbi_domain_cleanup_data(dom);
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+void sbi_domain_cleanup_data(struct sbi_domain *dom)
+{
+	struct sbi_domain_data *data;
+
+	if (!dom)
+		return;
+
+	sbi_list_for_each_entry(data, &data_list, head)
+		domain_cleanup_data_one(dom, data);
+}
+
+int sbi_domain_register_data(struct sbi_domain_data *data)
+{
+	struct sbi_domain *dom;
+	u32 data_idx;
+	int rc;
+
+	if (!data || !data->data_size)
+		return SBI_EINVAL;
+
+	for (data_idx = 0; data_idx < SBI_DOMAIN_MAX_DATA_PTRS; data_idx++) {
+		if (!bitmap_test(data_idx_bmap, data_idx))
+			break;
+	}
+	if (SBI_DOMAIN_MAX_DATA_PTRS <= data_idx)
+		return SBI_ENOSPC;
+	bitmap_set(data_idx_bmap, data_idx, 1);
+
+	data->data_idx = data_idx;
+	sbi_list_add_tail(&data->head, &data_list);
+
+	sbi_domain_for_each(dom) {
+		rc = domain_setup_data_one(dom, data);
+		if (rc) {
+			sbi_domain_unregister_data(data);
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+void sbi_domain_unregister_data(struct sbi_domain_data *data)
+{
+	struct sbi_domain *dom;
+
+	sbi_domain_for_each(dom)
+		domain_cleanup_data_one(dom, data);
+
+	sbi_list_del(&data->head);
+	bitmap_clear(data_idx_bmap, data->data_idx, 1);
+}

lib/sbi/sbi_double_trap.c

@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Rivos Inc.
+ *
+ * Authors:
+ *   Clément Léger <clement.leger@rivosinc.com>
+ */
+
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_sse.h>
+#include <sbi/sbi_trap.h>
+
+int sbi_double_trap_handler(struct sbi_trap_context *tcntx)
+{
+	struct sbi_trap_regs *regs = &tcntx->regs;
+	const struct sbi_trap_info *trap = &tcntx->trap;
+	bool prev_virt = sbi_regs_from_virt(regs);
+
+	if (sbi_mstatus_prev_mode(regs->mstatus) != PRV_S)
+		return SBI_ERR_INVALID_PARAM;
+
+	/* Exception was taken in VS-mode, redirect it to S-mode */
+	if (prev_virt)
+		return sbi_trap_redirect(regs, trap);
+
+	return sbi_sse_inject_event(SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP);
+}

lib/sbi/sbi_ecall.c

@@ -11,10 +11,10 @@
 #include <sbi/sbi_ecall.h>
 #include <sbi/sbi_ecall_interface.h>
 #include <sbi/sbi_error.h>
+#include <sbi/sbi_string.h>
 #include <sbi/sbi_trap.h>
 
-extern struct sbi_ecall_extension *sbi_ecall_exts[];
-extern unsigned long sbi_ecall_exts_size;
+extern struct sbi_ecall_extension *const sbi_ecall_exts[];
 
 u16 sbi_ecall_version_major(void)
 {
@@ -54,6 +54,29 @@ struct sbi_ecall_extension *sbi_ecall_find_extension(unsigned long extid)
 	return ret;
 }
 
+void sbi_ecall_get_extensions_str(char *exts_str, int exts_str_size, bool experimental)
+{
+	struct sbi_ecall_extension *t;
+	int offset = 0;
+
+	if (!exts_str || exts_str_size <= 0)
+		return;
+	sbi_memset(exts_str, 0, exts_str_size);
+
+	sbi_list_for_each_entry(t, &ecall_exts_list, head) {
+		if (experimental != t->experimental)
+			continue;
+		sbi_snprintf(exts_str + offset, exts_str_size - offset,
+			     "%s,", t->name);
+		offset = offset + sbi_strlen(t->name) + 1;
+	}
+
+	if (offset)
+		exts_str[offset - 1] = '\0';
+	else
+		sbi_strncpy(exts_str, "none", exts_str_size);
+}
+
 int sbi_ecall_register_extension(struct sbi_ecall_extension *ext)
 {
 	struct sbi_ecall_extension *t;
@@ -95,9 +118,10 @@ void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext)
 		sbi_list_del_init(&ext->head);
 }
 
-int sbi_ecall_handler(struct sbi_trap_regs *regs)
+int sbi_ecall_handler(struct sbi_trap_context *tcntx)
 {
 	int ret = 0;
+	struct sbi_trap_regs *regs = &tcntx->regs;
 	struct sbi_ecall_extension *ext;
 	unsigned long extension_id = regs->a7;
 	unsigned long func_id = regs->a6;
@@ -147,7 +171,7 @@ int sbi_ecall_init(void)
 	struct sbi_ecall_extension *ext;
 	unsigned long i;
 
-	for (i = 0; i < sbi_ecall_exts_size; i++) {
+	for (i = 0; sbi_ecall_exts[i]; i++) {
 		ext = sbi_ecall_exts[i];
 		ret = SBI_ENODEV;